]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
audio follouwp: pull out parsing into a conf_has_audio
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use POSIX;
7use IO::Handle;
8use IO::Select;
9use IO::File;
10use IO::Dir;
11use IO::Socket::UNIX;
12use File::Basename;
13use File::Path;
14use File::stat;
15use Getopt::Long;
16use Digest::SHA;
17use Fcntl ':flock';
18use Cwd 'abs_path';
19use IPC::Open3;
20use JSON;
21use Fcntl;
22use PVE::SafeSyslog;
23use Storable qw(dclone);
24use MIME::Base64;
25use PVE::Exception qw(raise raise_param_exc);
26use PVE::Storage;
27use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
28use PVE::JSONSchema qw(get_standard_option);
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30use PVE::INotify;
31use PVE::ProcFSTools;
32use PVE::QemuConfig;
33use PVE::QMPClient;
34use PVE::RPCEnvironment;
35use PVE::GuestHelpers;
36use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
37use PVE::QemuServer::Memory;
38use PVE::QemuServer::USB qw(parse_usb_device);
39use PVE::QemuServer::Cloudinit;
40use PVE::SysFSTools;
41use PVE::Systemd;
42use Time::HiRes qw(gettimeofday);
43use File::Copy qw(copy);
44use URI::Escape;
45
46my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
47my $OVMF = {
48 x86_64 => [
49 "$EDK2_FW_BASE/OVMF_CODE.fd",
50 "$EDK2_FW_BASE/OVMF_VARS.fd"
51 ],
52 aarch64 => [
53 "$EDK2_FW_BASE/AAVMF_CODE.fd",
54 "$EDK2_FW_BASE/AAVMF_VARS.fd"
55 ],
56};
57
58my $qemu_snap_storage = { rbd => 1 };
59
60my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
61
62my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
63
64# Note about locking: we use flock on the config file protect
65# against concurent actions.
66# Aditionaly, we have a 'lock' setting in the config file. This
67# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
68# allowed when such lock is set. But you can ignore this kind of
69# lock with the --skiplock flag.
70
71cfs_register_file('/qemu-server/',
72 \&parse_vm_config,
73 \&write_vm_config);
74
75PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
76 description => "Some command save/restore state from this location.",
77 type => 'string',
78 maxLength => 128,
79 optional => 1,
80});
81
82PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
83 type => 'string',
84 enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
85 description => "The drive's backing file's data format.",
86 optional => 1,
87});
88
89PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
90 description => "Specifies the Qemu machine type.",
91 type => 'string',
92 pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
93 maxLength => 40,
94 optional => 1,
95});
96
97#no warnings 'redefine';
98
99sub cgroups_write {
100 my ($controller, $vmid, $option, $value) = @_;
101
102 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
103 PVE::ProcFSTools::write_proc_entry($path, $value);
104
105}
106
107my $nodename = PVE::INotify::nodename();
108
109mkdir "/etc/pve/nodes/$nodename";
110my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
111mkdir $confdir;
112
113my $var_run_tmpdir = "/var/run/qemu-server";
114mkdir $var_run_tmpdir;
115
116my $lock_dir = "/var/lock/qemu-server";
117mkdir $lock_dir;
118
119my $cpu_vendor_list = {
120 # Intel CPUs
121 486 => 'GenuineIntel',
122 pentium => 'GenuineIntel',
123 pentium2 => 'GenuineIntel',
124 pentium3 => 'GenuineIntel',
125 coreduo => 'GenuineIntel',
126 core2duo => 'GenuineIntel',
127 Conroe => 'GenuineIntel',
128 Penryn => 'GenuineIntel',
129 Nehalem => 'GenuineIntel',
130 'Nehalem-IBRS' => 'GenuineIntel',
131 Westmere => 'GenuineIntel',
132 'Westmere-IBRS' => 'GenuineIntel',
133 SandyBridge => 'GenuineIntel',
134 'SandyBridge-IBRS' => 'GenuineIntel',
135 IvyBridge => 'GenuineIntel',
136 'IvyBridge-IBRS' => 'GenuineIntel',
137 Haswell => 'GenuineIntel',
138 'Haswell-IBRS' => 'GenuineIntel',
139 'Haswell-noTSX' => 'GenuineIntel',
140 'Haswell-noTSX-IBRS' => 'GenuineIntel',
141 Broadwell => 'GenuineIntel',
142 'Broadwell-IBRS' => 'GenuineIntel',
143 'Broadwell-noTSX' => 'GenuineIntel',
144 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
145 'Skylake-Client' => 'GenuineIntel',
146 'Skylake-Client-IBRS' => 'GenuineIntel',
147 'Skylake-Server' => 'GenuineIntel',
148 'Skylake-Server-IBRS' => 'GenuineIntel',
149
150 # AMD CPUs
151 athlon => 'AuthenticAMD',
152 phenom => 'AuthenticAMD',
153 Opteron_G1 => 'AuthenticAMD',
154 Opteron_G2 => 'AuthenticAMD',
155 Opteron_G3 => 'AuthenticAMD',
156 Opteron_G4 => 'AuthenticAMD',
157 Opteron_G5 => 'AuthenticAMD',
158 EPYC => 'AuthenticAMD',
159 'EPYC-IBPB' => 'AuthenticAMD',
160
161 # generic types, use vendor from host node
162 host => 'default',
163 kvm32 => 'default',
164 kvm64 => 'default',
165 qemu32 => 'default',
166 qemu64 => 'default',
167 max => 'default',
168};
169
170my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
171
172my $cpu_fmt = {
173 cputype => {
174 description => "Emulated CPU type.",
175 type => 'string',
176 enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
177 default => 'kvm64',
178 default_key => 1,
179 },
180 hidden => {
181 description => "Do not identify as a KVM virtual machine.",
182 type => 'boolean',
183 optional => 1,
184 default => 0
185 },
186 'hv-vendor-id' => {
187 type => 'string',
188 pattern => qr/[a-zA-Z0-9]{1,12}/,
189 format_description => 'vendor-id',
190 description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
191 optional => 1,
192 },
193 flags => {
194 description => "List of additional CPU flags separated by ';'."
195 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
196 . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
197 format_description => '+FLAG[;-FLAG...]',
198 type => 'string',
199 pattern => qr/$cpu_flag(;$cpu_flag)*/,
200 optional => 1,
201 },
202};
203
204my $watchdog_fmt = {
205 model => {
206 default_key => 1,
207 type => 'string',
208 enum => [qw(i6300esb ib700)],
209 description => "Watchdog type to emulate.",
210 default => 'i6300esb',
211 optional => 1,
212 },
213 action => {
214 type => 'string',
215 enum => [qw(reset shutdown poweroff pause debug none)],
216 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
217 optional => 1,
218 },
219};
220PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
221
222my $agent_fmt = {
223 enabled => {
224 description => "Enable/disable Qemu GuestAgent.",
225 type => 'boolean',
226 default => 0,
227 default_key => 1,
228 },
229 fstrim_cloned_disks => {
230 description => "Run fstrim after cloning/moving a disk.",
231 type => 'boolean',
232 optional => 1,
233 default => 0
234 },
235};
236
237my $vga_fmt = {
238 type => {
239 description => "Select the VGA type.",
240 type => 'string',
241 default => 'std',
242 optional => 1,
243 default_key => 1,
244 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
245 },
246 memory => {
247 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
248 type => 'integer',
249 optional => 1,
250 minimum => 4,
251 maximum => 512,
252 },
253};
254
255my $ivshmem_fmt = {
256 size => {
257 type => 'integer',
258 minimum => 1,
259 description => "The size of the file in MB.",
260 },
261 name => {
262 type => 'string',
263 pattern => '[a-zA-Z0-9\-]+',
264 optional => 1,
265 format_description => 'string',
266 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
267 },
268};
269
270my $audio_fmt = {
271 device => {
272 type => 'string',
273 enum => [qw(ich9-intel-hda intel-hda AC97)],
274 description => "Configure an audio device."
275 },
276 driver => {
277 type => 'string',
278 enum => ['spice'],
279 default => 'spice',
280 optional => 1,
281 description => "Driver backend for the audio device."
282 },
283};
284
285my $confdesc = {
286 onboot => {
287 optional => 1,
288 type => 'boolean',
289 description => "Specifies whether a VM will be started during system bootup.",
290 default => 0,
291 },
292 autostart => {
293 optional => 1,
294 type => 'boolean',
295 description => "Automatic restart after crash (currently ignored).",
296 default => 0,
297 },
298 hotplug => {
299 optional => 1,
300 type => 'string', format => 'pve-hotplug-features',
301 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
302 default => 'network,disk,usb',
303 },
304 reboot => {
305 optional => 1,
306 type => 'boolean',
307 description => "Allow reboot. If set to '0' the VM exit on reboot.",
308 default => 1,
309 },
310 lock => {
311 optional => 1,
312 type => 'string',
313 description => "Lock/unlock the VM.",
314 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
315 },
316 cpulimit => {
317 optional => 1,
318 type => 'number',
319 description => "Limit of CPU usage.",
320 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
321 minimum => 0,
322 maximum => 128,
323 default => 0,
324 },
325 cpuunits => {
326 optional => 1,
327 type => 'integer',
328 description => "CPU weight for a VM.",
329 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
330 minimum => 2,
331 maximum => 262144,
332 default => 1024,
333 },
334 memory => {
335 optional => 1,
336 type => 'integer',
337 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
338 minimum => 16,
339 default => 512,
340 },
341 balloon => {
342 optional => 1,
343 type => 'integer',
344 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
345 minimum => 0,
346 },
347 shares => {
348 optional => 1,
349 type => 'integer',
350 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
351 minimum => 0,
352 maximum => 50000,
353 default => 1000,
354 },
355 keyboard => {
356 optional => 1,
357 type => 'string',
358 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
359 "It should not be necessary to set it.",
360 enum => PVE::Tools::kvmkeymaplist(),
361 default => undef,
362 },
363 name => {
364 optional => 1,
365 type => 'string', format => 'dns-name',
366 description => "Set a name for the VM. Only used on the configuration web interface.",
367 },
368 scsihw => {
369 optional => 1,
370 type => 'string',
371 description => "SCSI controller model",
372 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
373 default => 'lsi',
374 },
375 description => {
376 optional => 1,
377 type => 'string',
378 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
379 },
380 ostype => {
381 optional => 1,
382 type => 'string',
383 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
384 description => "Specify guest operating system.",
385 verbose_description => <<EODESC,
386Specify guest operating system. This is used to enable special
387optimization/features for specific operating systems:
388
389[horizontal]
390other;; unspecified OS
391wxp;; Microsoft Windows XP
392w2k;; Microsoft Windows 2000
393w2k3;; Microsoft Windows 2003
394w2k8;; Microsoft Windows 2008
395wvista;; Microsoft Windows Vista
396win7;; Microsoft Windows 7
397win8;; Microsoft Windows 8/2012/2012r2
398win10;; Microsoft Windows 10/2016
399l24;; Linux 2.4 Kernel
400l26;; Linux 2.6/3.X Kernel
401solaris;; Solaris/OpenSolaris/OpenIndiania kernel
402EODESC
403 },
404 boot => {
405 optional => 1,
406 type => 'string',
407 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
408 pattern => '[acdn]{1,4}',
409 default => 'cdn',
410 },
411 bootdisk => {
412 optional => 1,
413 type => 'string', format => 'pve-qm-bootdisk',
414 description => "Enable booting from specified disk.",
415 pattern => '(ide|sata|scsi|virtio)\d+',
416 },
417 smp => {
418 optional => 1,
419 type => 'integer',
420 description => "The number of CPUs. Please use option -sockets instead.",
421 minimum => 1,
422 default => 1,
423 },
424 sockets => {
425 optional => 1,
426 type => 'integer',
427 description => "The number of CPU sockets.",
428 minimum => 1,
429 default => 1,
430 },
431 cores => {
432 optional => 1,
433 type => 'integer',
434 description => "The number of cores per socket.",
435 minimum => 1,
436 default => 1,
437 },
438 numa => {
439 optional => 1,
440 type => 'boolean',
441 description => "Enable/disable NUMA.",
442 default => 0,
443 },
444 hugepages => {
445 optional => 1,
446 type => 'string',
447 description => "Enable/disable hugepages memory.",
448 enum => [qw(any 2 1024)],
449 },
450 vcpus => {
451 optional => 1,
452 type => 'integer',
453 description => "Number of hotplugged vcpus.",
454 minimum => 1,
455 default => 0,
456 },
457 acpi => {
458 optional => 1,
459 type => 'boolean',
460 description => "Enable/disable ACPI.",
461 default => 1,
462 },
463 agent => {
464 optional => 1,
465 description => "Enable/disable Qemu GuestAgent and its properties.",
466 type => 'string',
467 format => $agent_fmt,
468 },
469 kvm => {
470 optional => 1,
471 type => 'boolean',
472 description => "Enable/disable KVM hardware virtualization.",
473 default => 1,
474 },
475 tdf => {
476 optional => 1,
477 type => 'boolean',
478 description => "Enable/disable time drift fix.",
479 default => 0,
480 },
481 localtime => {
482 optional => 1,
483 type => 'boolean',
484 description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
485 },
486 freeze => {
487 optional => 1,
488 type => 'boolean',
489 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
490 },
491 vga => {
492 optional => 1,
493 type => 'string', format => $vga_fmt,
494 description => "Configure the VGA hardware.",
495 verbose_description => "Configure the VGA Hardware. If you want to use ".
496 "high resolution modes (>= 1280x1024x16) you may need to increase " .
497 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
498 "is 'std' for all OS types besides some Windows versions (XP and " .
499 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
500 "display server. For win* OS you can select how many independent " .
501 "displays you want, Linux guests can add displays them self.\n".
502 "You can also run without any graphic card, using a serial device as terminal.",
503 },
504 watchdog => {
505 optional => 1,
506 type => 'string', format => 'pve-qm-watchdog',
507 description => "Create a virtual hardware watchdog device.",
508 verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
509 " (by a guest action), the watchdog must be periodically polled " .
510 "by an agent inside the guest or else the watchdog will reset " .
511 "the guest (or execute the respective action specified)",
512 },
513 startdate => {
514 optional => 1,
515 type => 'string',
516 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
517 description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
519 default => 'now',
520 },
521 startup => get_standard_option('pve-startup-order'),
522 template => {
523 optional => 1,
524 type => 'boolean',
525 description => "Enable/disable Template.",
526 default => 0,
527 },
528 args => {
529 optional => 1,
530 type => 'string',
531 description => "Arbitrary arguments passed to kvm.",
532 verbose_description => <<EODESCR,
533Arbitrary arguments passed to kvm, for example:
534
535args: -no-reboot -no-hpet
536
537NOTE: this option is for experts only.
538EODESCR
539 },
540 tablet => {
541 optional => 1,
542 type => 'boolean',
543 default => 1,
544 description => "Enable/disable the USB tablet device.",
545 verbose_description => "Enable/disable the USB tablet device. This device is " .
546 "usually needed to allow absolute mouse positioning with VNC. " .
547 "Else the mouse runs out of sync with normal VNC clients. " .
548 "If you're running lots of console-only guests on one host, " .
549 "you may consider disabling this to save some context switches. " .
550 "This is turned off by default if you use spice (-vga=qxl).",
551 },
552 migrate_speed => {
553 optional => 1,
554 type => 'integer',
555 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
556 minimum => 0,
557 default => 0,
558 },
559 migrate_downtime => {
560 optional => 1,
561 type => 'number',
562 description => "Set maximum tolerated downtime (in seconds) for migrations.",
563 minimum => 0,
564 default => 0.1,
565 },
566 cdrom => {
567 optional => 1,
568 type => 'string', format => 'pve-qm-ide',
569 typetext => '<volume>',
570 description => "This is an alias for option -ide2",
571 },
572 cpu => {
573 optional => 1,
574 description => "Emulated CPU type.",
575 type => 'string',
576 format => $cpu_fmt,
577 },
578 parent => get_standard_option('pve-snapshot-name', {
579 optional => 1,
580 description => "Parent snapshot name. This is used internally, and should not be modified.",
581 }),
582 snaptime => {
583 optional => 1,
584 description => "Timestamp for snapshots.",
585 type => 'integer',
586 minimum => 0,
587 },
588 vmstate => {
589 optional => 1,
590 type => 'string', format => 'pve-volume-id',
591 description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
592 },
593 vmstatestorage => get_standard_option('pve-storage-id', {
594 description => "Default storage for VM state volumes/files.",
595 optional => 1,
596 }),
597 runningmachine => get_standard_option('pve-qemu-machine', {
598 description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
599 }),
600 machine => get_standard_option('pve-qemu-machine'),
601 arch => {
602 description => "Virtual processor architecture. Defaults to the host.",
603 optional => 1,
604 type => 'string',
605 enum => [qw(x86_64 aarch64)],
606 },
607 smbios1 => {
608 description => "Specify SMBIOS type 1 fields.",
609 type => 'string', format => 'pve-qm-smbios1',
610 maxLength => 512,
611 optional => 1,
612 },
613 protection => {
614 optional => 1,
615 type => 'boolean',
616 description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
617 default => 0,
618 },
619 bios => {
620 optional => 1,
621 type => 'string',
622 enum => [ qw(seabios ovmf) ],
623 description => "Select BIOS implementation.",
624 default => 'seabios',
625 },
626 vmgenid => {
627 type => 'string',
628 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
629 format_description => 'UUID',
630 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
631 verbose_description => "The VM generation ID (vmgenid) device exposes a".
632 " 128-bit integer value identifier to the guest OS. This allows to".
633 " notify the guest operating system when the virtual machine is".
634 " executed with a different configuration (e.g. snapshot execution".
635 " or creation from a template). The guest operating system notices".
636 " the change, and is then able to react as appropriate by marking".
637 " its copies of distributed databases as dirty, re-initializing its".
638 " random number generator, etc.\n".
639 "Note that auto-creation only works when done throug API/CLI create".
640 " or update methods, but not when manually editing the config file.",
641 default => "1 (autogenerated)",
642 optional => 1,
643 },
644 hookscript => {
645 type => 'string',
646 format => 'pve-volume-id',
647 optional => 1,
648 description => "Script that will be executed during various steps in the vms lifetime.",
649 },
650 ivshmem => {
651 type => 'string',
652 format => $ivshmem_fmt,
653 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
654 optional => 1,
655 },
656 audio0 => {
657 type => 'string',
658 format => $audio_fmt,
659 description => "Configure a audio device, useful in combination with QXL/Spice.",
660 optional => 1
661 },
662};
663
664my $cicustom_fmt = {
665 meta => {
666 type => 'string',
667 optional => 1,
668 description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
669 format => 'pve-volume-id',
670 format_description => 'volume',
671 },
672 network => {
673 type => 'string',
674 optional => 1,
675 description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
676 format => 'pve-volume-id',
677 format_description => 'volume',
678 },
679 user => {
680 type => 'string',
681 optional => 1,
682 description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
683 format => 'pve-volume-id',
684 format_description => 'volume',
685 },
686};
687PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
688
689my $confdesc_cloudinit = {
690 citype => {
691 optional => 1,
692 type => 'string',
693 description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
694 enum => ['configdrive2', 'nocloud'],
695 },
696 ciuser => {
697 optional => 1,
698 type => 'string',
699 description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
700 },
701 cipassword => {
702 optional => 1,
703 type => 'string',
704 description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
705 },
706 cicustom => {
707 optional => 1,
708 type => 'string',
709 description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
710 format => 'pve-qm-cicustom',
711 },
712 searchdomain => {
713 optional => 1,
714 type => 'string',
715 description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
716 },
717 nameserver => {
718 optional => 1,
719 type => 'string', format => 'address-list',
720 description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
721 },
722 sshkeys => {
723 optional => 1,
724 type => 'string',
725 format => 'urlencoded',
726 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
727 },
728};
729
730# what about other qemu settings ?
731#cpu => 'string',
732#machine => 'string',
733#fda => 'file',
734#fdb => 'file',
735#mtdblock => 'file',
736#sd => 'file',
737#pflash => 'file',
738#snapshot => 'bool',
739#bootp => 'file',
740##tftp => 'dir',
741##smb => 'dir',
742#kernel => 'file',
743#append => 'string',
744#initrd => 'file',
745##soundhw => 'string',
746
747while (my ($k, $v) = each %$confdesc) {
748 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
749}
750
751my $MAX_IDE_DISKS = 4;
752my $MAX_SCSI_DISKS = 14;
753my $MAX_VIRTIO_DISKS = 16;
754my $MAX_SATA_DISKS = 6;
755my $MAX_USB_DEVICES = 5;
756my $MAX_NETS = 32;
757my $MAX_UNUSED_DISKS = 256;
758my $MAX_HOSTPCI_DEVICES = 4;
759my $MAX_SERIAL_PORTS = 4;
760my $MAX_PARALLEL_PORTS = 3;
761my $MAX_NUMA = 8;
762
763my $numa_fmt = {
764 cpus => {
765 type => "string",
766 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
767 description => "CPUs accessing this NUMA node.",
768 format_description => "id[-id];...",
769 },
770 memory => {
771 type => "number",
772 description => "Amount of memory this NUMA node provides.",
773 optional => 1,
774 },
775 hostnodes => {
776 type => "string",
777 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
778 description => "Host NUMA nodes to use.",
779 format_description => "id[-id];...",
780 optional => 1,
781 },
782 policy => {
783 type => 'string',
784 enum => [qw(preferred bind interleave)],
785 description => "NUMA allocation policy.",
786 optional => 1,
787 },
788};
789PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
790my $numadesc = {
791 optional => 1,
792 type => 'string', format => $numa_fmt,
793 description => "NUMA topology.",
794};
795PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
796
797for (my $i = 0; $i < $MAX_NUMA; $i++) {
798 $confdesc->{"numa$i"} = $numadesc;
799}
800
801my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
802 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
803 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
804my $nic_model_list_txt = join(' ', sort @$nic_model_list);
805
806my $net_fmt_bridge_descr = <<__EOD__;
807Bridge to attach the network device to. The Proxmox VE standard bridge
808is called 'vmbr0'.
809
810If you do not specify a bridge, we create a kvm user (NATed) network
811device, which provides DHCP and DNS services. The following addresses
812are used:
813
814 10.0.2.2 Gateway
815 10.0.2.3 DNS Server
816 10.0.2.4 SMB Server
817
818The DHCP server assign addresses to the guest starting from 10.0.2.15.
819__EOD__
820
821my $net_fmt = {
822 macaddr => get_standard_option('mac-addr', {
823 description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
824 }),
825 model => {
826 type => 'string',
827 description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
828 enum => $nic_model_list,
829 default_key => 1,
830 },
831 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
832 bridge => {
833 type => 'string',
834 description => $net_fmt_bridge_descr,
835 format_description => 'bridge',
836 optional => 1,
837 },
838 queues => {
839 type => 'integer',
840 minimum => 0, maximum => 16,
841 description => 'Number of packet queues to be used on the device.',
842 optional => 1,
843 },
844 rate => {
845 type => 'number',
846 minimum => 0,
847 description => "Rate limit in mbps (megabytes per second) as floating point number.",
848 optional => 1,
849 },
850 tag => {
851 type => 'integer',
852 minimum => 1, maximum => 4094,
853 description => 'VLAN tag to apply to packets on this interface.',
854 optional => 1,
855 },
856 trunks => {
857 type => 'string',
858 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
859 description => 'VLAN trunks to pass through this interface.',
860 format_description => 'vlanid[;vlanid...]',
861 optional => 1,
862 },
863 firewall => {
864 type => 'boolean',
865 description => 'Whether this interface should be protected by the firewall.',
866 optional => 1,
867 },
868 link_down => {
869 type => 'boolean',
870 description => 'Whether this interface should be disconnected (like pulling the plug).',
871 optional => 1,
872 },
873};
874
875my $netdesc = {
876 optional => 1,
877 type => 'string', format => $net_fmt,
878 description => "Specify network devices.",
879};
880
881PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
882
883my $ipconfig_fmt = {
884 ip => {
885 type => 'string',
886 format => 'pve-ipv4-config',
887 format_description => 'IPv4Format/CIDR',
888 description => 'IPv4 address in CIDR format.',
889 optional => 1,
890 default => 'dhcp',
891 },
892 gw => {
893 type => 'string',
894 format => 'ipv4',
895 format_description => 'GatewayIPv4',
896 description => 'Default gateway for IPv4 traffic.',
897 optional => 1,
898 requires => 'ip',
899 },
900 ip6 => {
901 type => 'string',
902 format => 'pve-ipv6-config',
903 format_description => 'IPv6Format/CIDR',
904 description => 'IPv6 address in CIDR format.',
905 optional => 1,
906 default => 'dhcp',
907 },
908 gw6 => {
909 type => 'string',
910 format => 'ipv6',
911 format_description => 'GatewayIPv6',
912 description => 'Default gateway for IPv6 traffic.',
913 optional => 1,
914 requires => 'ip6',
915 },
916};
917PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
918my $ipconfigdesc = {
919 optional => 1,
920 type => 'string', format => 'pve-qm-ipconfig',
921 description => <<'EODESCR',
922cloud-init: Specify IP addresses and gateways for the corresponding interface.
923
924IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
925
926The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
927For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
928
929If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
930EODESCR
931};
932PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
933
934for (my $i = 0; $i < $MAX_NETS; $i++) {
935 $confdesc->{"net$i"} = $netdesc;
936 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
937}
938
939foreach my $key (keys %$confdesc_cloudinit) {
940 $confdesc->{$key} = $confdesc_cloudinit->{$key};
941}
942
943PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
944sub verify_volume_id_or_qm_path {
945 my ($volid, $noerr) = @_;
946
947 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
948 return $volid;
949 }
950
951 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
952 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
953 if ($@) {
954 return undef if $noerr;
955 die $@;
956 }
957 return $volid;
958}
959
960my $drivename_hash;
961
962my %drivedesc_base = (
963 volume => { alias => 'file' },
964 file => {
965 type => 'string',
966 format => 'pve-volume-id-or-qm-path',
967 default_key => 1,
968 format_description => 'volume',
969 description => "The drive's backing volume.",
970 },
971 media => {
972 type => 'string',
973 enum => [qw(cdrom disk)],
974 description => "The drive's media type.",
975 default => 'disk',
976 optional => 1
977 },
978 cyls => {
979 type => 'integer',
980 description => "Force the drive's physical geometry to have a specific cylinder count.",
981 optional => 1
982 },
983 heads => {
984 type => 'integer',
985 description => "Force the drive's physical geometry to have a specific head count.",
986 optional => 1
987 },
988 secs => {
989 type => 'integer',
990 description => "Force the drive's physical geometry to have a specific sector count.",
991 optional => 1
992 },
993 trans => {
994 type => 'string',
995 enum => [qw(none lba auto)],
996 description => "Force disk geometry bios translation mode.",
997 optional => 1,
998 },
999 snapshot => {
1000 type => 'boolean',
1001 description => "Controls qemu's snapshot mode feature."
1002 . " If activated, changes made to the disk are temporary and will"
1003 . " be discarded when the VM is shutdown.",
1004 optional => 1,
1005 },
1006 cache => {
1007 type => 'string',
1008 enum => [qw(none writethrough writeback unsafe directsync)],
1009 description => "The drive's cache mode",
1010 optional => 1,
1011 },
1012 format => get_standard_option('pve-qm-image-format'),
1013 size => {
1014 type => 'string',
1015 format => 'disk-size',
1016 format_description => 'DiskSize',
1017 description => "Disk size. This is purely informational and has no effect.",
1018 optional => 1,
1019 },
1020 backup => {
1021 type => 'boolean',
1022 description => "Whether the drive should be included when making backups.",
1023 optional => 1,
1024 },
1025 replicate => {
1026 type => 'boolean',
1027 description => 'Whether the drive should considered for replication jobs.',
1028 optional => 1,
1029 default => 1,
1030 },
1031 rerror => {
1032 type => 'string',
1033 enum => [qw(ignore report stop)],
1034 description => 'Read error action.',
1035 optional => 1,
1036 },
1037 werror => {
1038 type => 'string',
1039 enum => [qw(enospc ignore report stop)],
1040 description => 'Write error action.',
1041 optional => 1,
1042 },
1043 aio => {
1044 type => 'string',
1045 enum => [qw(native threads)],
1046 description => 'AIO type to use.',
1047 optional => 1,
1048 },
1049 discard => {
1050 type => 'string',
1051 enum => [qw(ignore on)],
1052 description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1053 optional => 1,
1054 },
1055 detect_zeroes => {
1056 type => 'boolean',
1057 description => 'Controls whether to detect and try to optimize writes of zeroes.',
1058 optional => 1,
1059 },
1060 serial => {
1061 type => 'string',
1062 format => 'urlencoded',
1063 format_description => 'serial',
1064 maxLength => 20*3, # *3 since it's %xx url enoded
1065 description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1066 optional => 1,
1067 },
1068 shared => {
1069 type => 'boolean',
1070 description => 'Mark this locally-managed volume as available on all nodes',
1071 verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1072 optional => 1,
1073 default => 0,
1074 }
1075);
1076
1077my %iothread_fmt = ( iothread => {
1078 type => 'boolean',
1079 description => "Whether to use iothreads for this drive",
1080 optional => 1,
1081});
1082
1083my %model_fmt = (
1084 model => {
1085 type => 'string',
1086 format => 'urlencoded',
1087 format_description => 'model',
1088 maxLength => 40*3, # *3 since it's %xx url enoded
1089 description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1090 optional => 1,
1091 },
1092);
1093
1094my %queues_fmt = (
1095 queues => {
1096 type => 'integer',
1097 description => "Number of queues.",
1098 minimum => 2,
1099 optional => 1
1100 }
1101);
1102
1103my %scsiblock_fmt = (
1104 scsiblock => {
1105 type => 'boolean',
1106 description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1107 optional => 1,
1108 default => 0,
1109 },
1110);
1111
1112my %ssd_fmt = (
1113 ssd => {
1114 type => 'boolean',
1115 description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1116 optional => 1,
1117 },
1118);
1119
1120my %wwn_fmt = (
1121 wwn => {
1122 type => 'string',
1123 pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1124 format_description => 'wwn',
1125 description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1126 optional => 1,
1127 },
1128);
1129
1130my $add_throttle_desc = sub {
1131 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1132 my $d = {
1133 type => $type,
1134 format_description => $unit,
1135 description => "Maximum $what in $longunit.",
1136 optional => 1,
1137 };
1138 $d->{minimum} = $minimum if defined($minimum);
1139 $drivedesc_base{$key} = $d;
1140};
1141# throughput: (leaky bucket)
1142$add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1143$add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1144$add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1145$add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1146$add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1147$add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1148$add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1149$add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1150$add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1151
1152# pools: (pool of IO before throttling starts taking effect)
1153$add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1154$add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1155$add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1156$add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1157$add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1158$add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1159
1160# burst lengths
1161$add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1162$add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1163$add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1164$add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1165$add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1166$add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1167
1168# legacy support
1169$drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1170$drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1171$drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1172$drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1173
1174my $ide_fmt = {
1175 %drivedesc_base,
1176 %model_fmt,
1177 %ssd_fmt,
1178 %wwn_fmt,
1179};
1180PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1181
1182my $idedesc = {
1183 optional => 1,
1184 type => 'string', format => $ide_fmt,
1185 description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1186};
1187PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1188
1189my $scsi_fmt = {
1190 %drivedesc_base,
1191 %iothread_fmt,
1192 %queues_fmt,
1193 %scsiblock_fmt,
1194 %ssd_fmt,
1195 %wwn_fmt,
1196};
1197my $scsidesc = {
1198 optional => 1,
1199 type => 'string', format => $scsi_fmt,
1200 description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1201};
1202PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1203
1204my $sata_fmt = {
1205 %drivedesc_base,
1206 %ssd_fmt,
1207 %wwn_fmt,
1208};
1209my $satadesc = {
1210 optional => 1,
1211 type => 'string', format => $sata_fmt,
1212 description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1213};
1214PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1215
1216my $virtio_fmt = {
1217 %drivedesc_base,
1218 %iothread_fmt,
1219};
1220my $virtiodesc = {
1221 optional => 1,
1222 type => 'string', format => $virtio_fmt,
1223 description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1224};
1225PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1226
1227my $alldrive_fmt = {
1228 %drivedesc_base,
1229 %iothread_fmt,
1230 %model_fmt,
1231 %queues_fmt,
1232 %scsiblock_fmt,
1233 %ssd_fmt,
1234 %wwn_fmt,
1235};
1236
1237my $efidisk_fmt = {
1238 volume => { alias => 'file' },
1239 file => {
1240 type => 'string',
1241 format => 'pve-volume-id-or-qm-path',
1242 default_key => 1,
1243 format_description => 'volume',
1244 description => "The drive's backing volume.",
1245 },
1246 format => get_standard_option('pve-qm-image-format'),
1247 size => {
1248 type => 'string',
1249 format => 'disk-size',
1250 format_description => 'DiskSize',
1251 description => "Disk size. This is purely informational and has no effect.",
1252 optional => 1,
1253 },
1254};
1255
1256my $efidisk_desc = {
1257 optional => 1,
1258 type => 'string', format => $efidisk_fmt,
1259 description => "Configure a Disk for storing EFI vars",
1260};
1261
1262PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1263
1264my $usb_fmt = {
1265 host => {
1266 default_key => 1,
1267 type => 'string', format => 'pve-qm-usb-device',
1268 format_description => 'HOSTUSBDEVICE|spice',
1269 description => <<EODESCR,
1270The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1271
1272 'bus-port(.port)*' (decimal numbers) or
1273 'vendor_id:product_id' (hexadeciaml numbers) or
1274 'spice'
1275
1276You can use the 'lsusb -t' command to list existing usb devices.
1277
1278NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1279
1280The value 'spice' can be used to add a usb redirection devices for spice.
1281EODESCR
1282 },
1283 usb3 => {
1284 optional => 1,
1285 type => 'boolean',
1286 description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1287 default => 0,
1288 },
1289};
1290
1291my $usbdesc = {
1292 optional => 1,
1293 type => 'string', format => $usb_fmt,
1294 description => "Configure an USB device (n is 0 to 4).",
1295};
1296PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1297
1298my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1299my $hostpci_fmt = {
1300 host => {
1301 default_key => 1,
1302 type => 'string',
1303 pattern => qr/$PCIRE(;$PCIRE)*/,
1304 format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1305 description => <<EODESCR,
1306Host PCI device pass through. The PCI ID of a host's PCI device or a list
1307of PCI virtual functions of the host. HOSTPCIID syntax is:
1308
1309'bus:dev.func' (hexadecimal numbers)
1310
1311You can us the 'lspci' command to list existing PCI devices.
1312EODESCR
1313 },
1314 rombar => {
1315 type => 'boolean',
1316 description => "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1317 optional => 1,
1318 default => 1,
1319 },
1320 romfile => {
1321 type => 'string',
1322 pattern => '[^,;]+',
1323 format_description => 'string',
1324 description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1325 optional => 1,
1326 },
1327 pcie => {
1328 type => 'boolean',
1329 description => "Choose the PCI-express bus (needs the 'q35' machine model).",
1330 optional => 1,
1331 default => 0,
1332 },
1333 'x-vga' => {
1334 type => 'boolean',
1335 description => "Enable vfio-vga device support.",
1336 optional => 1,
1337 default => 0,
1338 },
1339 'mdev' => {
1340 type => 'string',
1341 format_description => 'string',
1342 pattern => '[^/\.:]+',
1343 optional => 1,
1344 description => <<EODESCR
1345The type of mediated device to use.
1346An instance of this type will be created on startup of the VM and
1347will be cleaned up when the VM stops.
1348EODESCR
1349 }
1350};
1351PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1352
1353my $hostpcidesc = {
1354 optional => 1,
1355 type => 'string', format => 'pve-qm-hostpci',
1356 description => "Map host PCI devices into guest.",
1357 verbose_description => <<EODESCR,
1358Map host PCI devices into guest.
1359
1360NOTE: This option allows direct access to host hardware. So it is no longer
1361possible to migrate such machines - use with special care.
1362
1363CAUTION: Experimental! User reported problems with this option.
1364EODESCR
1365};
1366PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1367
1368my $serialdesc = {
1369 optional => 1,
1370 type => 'string',
1371 pattern => '(/dev/.+|socket)',
1372 description => "Create a serial device inside the VM (n is 0 to 3)",
1373 verbose_description => <<EODESCR,
1374Create a serial device inside the VM (n is 0 to 3), and pass through a
1375host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1376host side (use 'qm terminal' to open a terminal connection).
1377
1378NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1379
1380CAUTION: Experimental! User reported problems with this option.
1381EODESCR
1382};
1383
1384my $paralleldesc= {
1385 optional => 1,
1386 type => 'string',
1387 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1388 description => "Map host parallel devices (n is 0 to 2).",
1389 verbose_description => <<EODESCR,
1390Map host parallel devices (n is 0 to 2).
1391
1392NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1393
1394CAUTION: Experimental! User reported problems with this option.
1395EODESCR
1396};
1397
1398for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1399 $confdesc->{"parallel$i"} = $paralleldesc;
1400}
1401
1402for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1403 $confdesc->{"serial$i"} = $serialdesc;
1404}
1405
1406for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1407 $confdesc->{"hostpci$i"} = $hostpcidesc;
1408}
1409
1410for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1411 $drivename_hash->{"ide$i"} = 1;
1412 $confdesc->{"ide$i"} = $idedesc;
1413}
1414
1415for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1416 $drivename_hash->{"sata$i"} = 1;
1417 $confdesc->{"sata$i"} = $satadesc;
1418}
1419
1420for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1421 $drivename_hash->{"scsi$i"} = 1;
1422 $confdesc->{"scsi$i"} = $scsidesc ;
1423}
1424
1425for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1426 $drivename_hash->{"virtio$i"} = 1;
1427 $confdesc->{"virtio$i"} = $virtiodesc;
1428}
1429
1430$drivename_hash->{efidisk0} = 1;
1431$confdesc->{efidisk0} = $efidisk_desc;
1432
1433for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1434 $confdesc->{"usb$i"} = $usbdesc;
1435}
1436
1437my $unuseddesc = {
1438 optional => 1,
1439 type => 'string', format => 'pve-volume-id',
1440 description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1441};
1442
1443for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1444 $confdesc->{"unused$i"} = $unuseddesc;
1445}
1446
1447my $kvm_api_version = 0;
1448
1449sub kvm_version {
1450 return $kvm_api_version if $kvm_api_version;
1451
1452 open my $fh, '<', '/dev/kvm'
1453 or return undef;
1454
1455 # 0xae00 => KVM_GET_API_VERSION
1456 $kvm_api_version = ioctl($fh, 0xae00, 0);
1457
1458 return $kvm_api_version;
1459}
1460
1461my $kvm_user_version;
1462
1463sub kvm_user_version {
1464
1465 return $kvm_user_version if $kvm_user_version;
1466
1467 $kvm_user_version = 'unknown';
1468
1469 my $code = sub {
1470 my $line = shift;
1471 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1472 $kvm_user_version = $2;
1473 }
1474 };
1475
1476 eval { run_command("kvm -version", outfunc => $code); };
1477 warn $@ if $@;
1478
1479 return $kvm_user_version;
1480
1481}
1482
1483sub kernel_has_vhost_net {
1484 return -c '/dev/vhost-net';
1485}
1486
1487sub valid_drive_names {
1488 # order is important - used to autoselect boot disk
1489 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1490 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1491 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1492 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1493 'efidisk0');
1494}
1495
1496sub is_valid_drivename {
1497 my $dev = shift;
1498
1499 return defined($drivename_hash->{$dev});
1500}
1501
1502sub option_exists {
1503 my $key = shift;
1504 return defined($confdesc->{$key});
1505}
1506
1507sub nic_models {
1508 return $nic_model_list;
1509}
1510
1511sub os_list_description {
1512
1513 return {
1514 other => 'Other',
1515 wxp => 'Windows XP',
1516 w2k => 'Windows 2000',
1517 w2k3 =>, 'Windows 2003',
1518 w2k8 => 'Windows 2008',
1519 wvista => 'Windows Vista',
1520 win7 => 'Windows 7',
1521 win8 => 'Windows 8/2012',
1522 win10 => 'Windows 10/2016',
1523 l24 => 'Linux 2.4',
1524 l26 => 'Linux 2.6',
1525 };
1526}
1527
1528my $cdrom_path;
1529
1530sub get_cdrom_path {
1531
1532 return $cdrom_path if $cdrom_path;
1533
1534 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1535 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1536 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1537}
1538
1539sub get_iso_path {
1540 my ($storecfg, $vmid, $cdrom) = @_;
1541
1542 if ($cdrom eq 'cdrom') {
1543 return get_cdrom_path();
1544 } elsif ($cdrom eq 'none') {
1545 return '';
1546 } elsif ($cdrom =~ m|^/|) {
1547 return $cdrom;
1548 } else {
1549 return PVE::Storage::path($storecfg, $cdrom);
1550 }
1551}
1552
1553# try to convert old style file names to volume IDs
1554sub filename_to_volume_id {
1555 my ($vmid, $file, $media) = @_;
1556
1557 if (!($file eq 'none' || $file eq 'cdrom' ||
1558 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1559
1560 return undef if $file =~ m|/|;
1561
1562 if ($media && $media eq 'cdrom') {
1563 $file = "local:iso/$file";
1564 } else {
1565 $file = "local:$vmid/$file";
1566 }
1567 }
1568
1569 return $file;
1570}
1571
1572sub verify_media_type {
1573 my ($opt, $vtype, $media) = @_;
1574
1575 return if !$media;
1576
1577 my $etype;
1578 if ($media eq 'disk') {
1579 $etype = 'images';
1580 } elsif ($media eq 'cdrom') {
1581 $etype = 'iso';
1582 } else {
1583 die "internal error";
1584 }
1585
1586 return if ($vtype eq $etype);
1587
1588 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1589}
1590
1591sub cleanup_drive_path {
1592 my ($opt, $storecfg, $drive) = @_;
1593
1594 # try to convert filesystem paths to volume IDs
1595
1596 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1597 ($drive->{file} !~ m|^/dev/.+|) &&
1598 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1599 ($drive->{file} !~ m/^\d+$/)) {
1600 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1601 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1602 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1603 verify_media_type($opt, $vtype, $drive->{media});
1604 $drive->{file} = $volid;
1605 }
1606
1607 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1608}
1609
1610sub parse_hotplug_features {
1611 my ($data) = @_;
1612
1613 my $res = {};
1614
1615 return $res if $data eq '0';
1616
1617 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1618
1619 foreach my $feature (PVE::Tools::split_list($data)) {
1620 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1621 $res->{$1} = 1;
1622 } else {
1623 die "invalid hotplug feature '$feature'\n";
1624 }
1625 }
1626 return $res;
1627}
1628
1629PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1630sub pve_verify_hotplug_features {
1631 my ($value, $noerr) = @_;
1632
1633 return $value if parse_hotplug_features($value);
1634
1635 return undef if $noerr;
1636
1637 die "unable to parse hotplug option\n";
1638}
1639
1640# ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1641# [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1642# [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1643# [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1644# [,iothread=on][,serial=serial][,model=model]
1645
1646sub parse_drive {
1647 my ($key, $data) = @_;
1648
1649 my ($interface, $index);
1650
1651 if ($key =~ m/^([^\d]+)(\d+)$/) {
1652 $interface = $1;
1653 $index = $2;
1654 } else {
1655 return undef;
1656 }
1657
1658 my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1659 : $confdesc->{$key}->{format};
1660 if (!$desc) {
1661 warn "invalid drive key: $key\n";
1662 return undef;
1663 }
1664 my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1665 return undef if !$res;
1666 $res->{interface} = $interface;
1667 $res->{index} = $index;
1668
1669 my $error = 0;
1670 foreach my $opt (qw(bps bps_rd bps_wr)) {
1671 if (my $bps = defined(delete $res->{$opt})) {
1672 if (defined($res->{"m$opt"})) {
1673 warn "both $opt and m$opt specified\n";
1674 ++$error;
1675 next;
1676 }
1677 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1678 }
1679 }
1680
1681 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1682 for my $requirement (
1683 [mbps_max => 'mbps'],
1684 [mbps_rd_max => 'mbps_rd'],
1685 [mbps_wr_max => 'mbps_wr'],
1686 [miops_max => 'miops'],
1687 [miops_rd_max => 'miops_rd'],
1688 [miops_wr_max => 'miops_wr'],
1689 [bps_max_length => 'mbps_max'],
1690 [bps_rd_max_length => 'mbps_rd_max'],
1691 [bps_wr_max_length => 'mbps_wr_max'],
1692 [iops_max_length => 'iops_max'],
1693 [iops_rd_max_length => 'iops_rd_max'],
1694 [iops_wr_max_length => 'iops_wr_max']) {
1695 my ($option, $requires) = @$requirement;
1696 if ($res->{$option} && !$res->{$requires}) {
1697 warn "$option requires $requires\n";
1698 ++$error;
1699 }
1700 }
1701
1702 return undef if $error;
1703
1704 return undef if $res->{mbps_rd} && $res->{mbps};
1705 return undef if $res->{mbps_wr} && $res->{mbps};
1706 return undef if $res->{iops_rd} && $res->{iops};
1707 return undef if $res->{iops_wr} && $res->{iops};
1708
1709 if ($res->{media} && ($res->{media} eq 'cdrom')) {
1710 return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1711 return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1712 return undef if $res->{interface} eq 'virtio';
1713 }
1714
1715 if (my $size = $res->{size}) {
1716 return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1717 }
1718
1719 return $res;
1720}
1721
1722sub print_drive {
1723 my ($vmid, $drive) = @_;
1724 my $data = { %$drive };
1725 delete $data->{$_} for qw(index interface);
1726 return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1727}
1728
1729sub scsi_inquiry {
1730 my($fh, $noerr) = @_;
1731
1732 my $SG_IO = 0x2285;
1733 my $SG_GET_VERSION_NUM = 0x2282;
1734
1735 my $versionbuf = "\x00" x 8;
1736 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1737 if (!$ret) {
1738 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1739 return undef;
1740 }
1741 my $version = unpack("I", $versionbuf);
1742 if ($version < 30000) {
1743 die "scsi generic interface too old\n" if !$noerr;
1744 return undef;
1745 }
1746
1747 my $buf = "\x00" x 36;
1748 my $sensebuf = "\x00" x 8;
1749 my $cmd = pack("C x3 C x1", 0x12, 36);
1750
1751 # see /usr/include/scsi/sg.h
1752 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1753
1754 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1755 length($sensebuf), 0, length($buf), $buf,
1756 $cmd, $sensebuf, 6000);
1757
1758 $ret = ioctl($fh, $SG_IO, $packet);
1759 if (!$ret) {
1760 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1761 return undef;
1762 }
1763
1764 my @res = unpack($sg_io_hdr_t, $packet);
1765 if ($res[17] || $res[18]) {
1766 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1767 return undef;
1768 }
1769
1770 my $res = {};
1771 (my $byte0, my $byte1, $res->{vendor},
1772 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1773
1774 $res->{removable} = $byte1 & 128 ? 1 : 0;
1775 $res->{type} = $byte0 & 31;
1776
1777 return $res;
1778}
1779
1780sub path_is_scsi {
1781 my ($path) = @_;
1782
1783 my $fh = IO::File->new("+<$path") || return undef;
1784 my $res = scsi_inquiry($fh, 1);
1785 close($fh);
1786
1787 return $res;
1788}
1789
1790sub machine_type_is_q35 {
1791 my ($conf) = @_;
1792
1793 return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1794}
1795
1796sub print_tabletdevice_full {
1797 my ($conf, $arch) = @_;
1798
1799 my $q35 = machine_type_is_q35($conf);
1800
1801 # we use uhci for old VMs because tablet driver was buggy in older qemu
1802 my $usbbus;
1803 if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1804 $usbbus = 'ehci';
1805 } else {
1806 $usbbus = 'uhci';
1807 }
1808
1809 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1810}
1811
1812sub print_keyboarddevice_full {
1813 my ($conf, $arch, $machine) = @_;
1814
1815 return undef if $arch ne 'aarch64';
1816
1817 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1818}
1819
1820sub print_drivedevice_full {
1821 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1822
1823 my $device = '';
1824 my $maxdev = 0;
1825
1826 if ($drive->{interface} eq 'virtio') {
1827 my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1828 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1829 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1830 } elsif ($drive->{interface} eq 'scsi') {
1831
1832 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1833 my $unit = $drive->{index} % $maxdev;
1834 my $devicetype = 'hd';
1835 my $path = '';
1836 if (drive_is_cdrom($drive)) {
1837 $devicetype = 'cd';
1838 } else {
1839 if ($drive->{file} =~ m|^/|) {
1840 $path = $drive->{file};
1841 if (my $info = path_is_scsi($path)) {
1842 if ($info->{type} == 0 && $drive->{scsiblock}) {
1843 $devicetype = 'block';
1844 } elsif ($info->{type} == 1) { # tape
1845 $devicetype = 'generic';
1846 }
1847 }
1848 } else {
1849 $path = PVE::Storage::path($storecfg, $drive->{file});
1850 }
1851
1852 if($path =~ m/^iscsi\:\/\//){
1853 $devicetype = 'generic';
1854 }
1855 }
1856
1857 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1858 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1859 } else {
1860 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1861 }
1862
1863 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1864 $device .= ",rotation_rate=1";
1865 }
1866 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1867
1868 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1869 my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1870 my $controller = int($drive->{index} / $maxdev);
1871 my $unit = $drive->{index} % $maxdev;
1872 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1873
1874 $device = "ide-$devicetype";
1875 if ($drive->{interface} eq 'ide') {
1876 $device .= ",bus=ide.$controller,unit=$unit";
1877 } else {
1878 $device .= ",bus=ahci$controller.$unit";
1879 }
1880 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1881
1882 if ($devicetype eq 'hd') {
1883 if (my $model = $drive->{model}) {
1884 $model = URI::Escape::uri_unescape($model);
1885 $device .= ",model=$model";
1886 }
1887 if ($drive->{ssd}) {
1888 $device .= ",rotation_rate=1";
1889 }
1890 }
1891 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1892 } elsif ($drive->{interface} eq 'usb') {
1893 die "implement me";
1894 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1895 } else {
1896 die "unsupported interface type";
1897 }
1898
1899 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1900
1901 if (my $serial = $drive->{serial}) {
1902 $serial = URI::Escape::uri_unescape($serial);
1903 $device .= ",serial=$serial";
1904 }
1905
1906
1907 return $device;
1908}
1909
1910sub get_initiator_name {
1911 my $initiator;
1912
1913 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1914 while (defined(my $line = <$fh>)) {
1915 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1916 $initiator = $1;
1917 last;
1918 }
1919 $fh->close();
1920
1921 return $initiator;
1922}
1923
1924sub print_drive_full {
1925 my ($storecfg, $vmid, $drive) = @_;
1926
1927 my $path;
1928 my $volid = $drive->{file};
1929 my $format;
1930
1931 if (drive_is_cdrom($drive)) {
1932 $path = get_iso_path($storecfg, $vmid, $volid);
1933 } else {
1934 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1935 if ($storeid) {
1936 $path = PVE::Storage::path($storecfg, $volid);
1937 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1938 $format = qemu_img_format($scfg, $volname);
1939 } else {
1940 $path = $volid;
1941 $format = "raw";
1942 }
1943 }
1944
1945 my $opts = '';
1946 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1947 foreach my $o (@qemu_drive_options) {
1948 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1949 }
1950
1951 # snapshot only accepts on|off
1952 if (defined($drive->{snapshot})) {
1953 my $v = $drive->{snapshot} ? 'on' : 'off';
1954 $opts .= ",snapshot=$v";
1955 }
1956
1957 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1958 my ($dir, $qmpname) = @$type;
1959 if (my $v = $drive->{"mbps$dir"}) {
1960 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1961 }
1962 if (my $v = $drive->{"mbps${dir}_max"}) {
1963 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1964 }
1965 if (my $v = $drive->{"bps${dir}_max_length"}) {
1966 $opts .= ",throttling.bps$qmpname-max-length=$v";
1967 }
1968 if (my $v = $drive->{"iops${dir}"}) {
1969 $opts .= ",throttling.iops$qmpname=$v";
1970 }
1971 if (my $v = $drive->{"iops${dir}_max"}) {
1972 $opts .= ",throttling.iops$qmpname-max=$v";
1973 }
1974 if (my $v = $drive->{"iops${dir}_max_length"}) {
1975 $opts .= ",throttling.iops$qmpname-max-length=$v";
1976 }
1977 }
1978
1979 $opts .= ",format=$format" if $format && !$drive->{format};
1980
1981 my $cache_direct = 0;
1982
1983 if (my $cache = $drive->{cache}) {
1984 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1985 } elsif (!drive_is_cdrom($drive)) {
1986 $opts .= ",cache=none";
1987 $cache_direct = 1;
1988 }
1989
1990 # aio native works only with O_DIRECT
1991 if (!$drive->{aio}) {
1992 if($cache_direct) {
1993 $opts .= ",aio=native";
1994 } else {
1995 $opts .= ",aio=threads";
1996 }
1997 }
1998
1999 if (!drive_is_cdrom($drive)) {
2000 my $detectzeroes;
2001 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2002 $detectzeroes = 'off';
2003 } elsif ($drive->{discard}) {
2004 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2005 } else {
2006 # This used to be our default with discard not being specified:
2007 $detectzeroes = 'on';
2008 }
2009 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2010 }
2011
2012 my $pathinfo = $path ? "file=$path," : '';
2013
2014 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2015}
2016
2017sub print_netdevice_full {
2018 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2019
2020 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2021
2022 my $device = $net->{model};
2023 if ($net->{model} eq 'virtio') {
2024 $device = 'virtio-net-pci';
2025 };
2026
2027 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2028 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2029 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2030 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2031 my $vectors = $net->{queues} * 2 + 2;
2032 $tmpstr .= ",vectors=$vectors,mq=on";
2033 }
2034 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2035
2036 if ($use_old_bios_files) {
2037 my $romfile;
2038 if ($device eq 'virtio-net-pci') {
2039 $romfile = 'pxe-virtio.rom';
2040 } elsif ($device eq 'e1000') {
2041 $romfile = 'pxe-e1000.rom';
2042 } elsif ($device eq 'ne2k') {
2043 $romfile = 'pxe-ne2k_pci.rom';
2044 } elsif ($device eq 'pcnet') {
2045 $romfile = 'pxe-pcnet.rom';
2046 } elsif ($device eq 'rtl8139') {
2047 $romfile = 'pxe-rtl8139.rom';
2048 }
2049 $tmpstr .= ",romfile=$romfile" if $romfile;
2050 }
2051
2052 return $tmpstr;
2053}
2054
2055sub print_netdev_full {
2056 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2057
2058 my $i = '';
2059 if ($netid =~ m/^net(\d+)$/) {
2060 $i = int($1);
2061 }
2062
2063 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2064
2065 my $ifname = "tap${vmid}i$i";
2066
2067 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2068 die "interface name '$ifname' is too long (max 15 character)\n"
2069 if length($ifname) >= 16;
2070
2071 my $vhostparam = '';
2072 if (is_native($arch)) {
2073 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2074 }
2075
2076 my $vmname = $conf->{name} || "vm$vmid";
2077
2078 my $netdev = "";
2079 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2080
2081 if ($net->{bridge}) {
2082 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2083 } else {
2084 $netdev = "type=user,id=$netid,hostname=$vmname";
2085 }
2086
2087 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2088
2089 return $netdev;
2090}
2091
2092
2093sub print_cpu_device {
2094 my ($conf, $id) = @_;
2095
2096 my $kvm = $conf->{kvm} // 1;
2097 my $cpu = $kvm ? "kvm64" : "qemu64";
2098 if (my $cputype = $conf->{cpu}) {
2099 my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2100 or die "Cannot parse cpu description: $cputype\n";
2101 $cpu = $cpuconf->{cputype};
2102 }
2103
2104 my $cores = $conf->{cores} || 1;
2105
2106 my $current_core = ($id - 1) % $cores;
2107 my $current_socket = int(($id - 1 - $current_core)/$cores);
2108
2109 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2110}
2111
2112my $vga_map = {
2113 'cirrus' => 'cirrus-vga',
2114 'std' => 'VGA',
2115 'vmware' => 'vmware-svga',
2116 'virtio' => 'virtio-vga',
2117};
2118
2119sub print_vga_device {
2120 my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2121
2122 my $type = $vga_map->{$vga->{type}};
2123 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2124 $type = 'virtio-gpu';
2125 }
2126 my $vgamem_mb = $vga->{memory};
2127 if ($qxlnum) {
2128 $type = $id ? 'qxl' : 'qxl-vga';
2129 }
2130 die "no devicetype for $vga->{type}\n" if !$type;
2131
2132 my $memory = "";
2133 if ($vgamem_mb) {
2134 if ($vga->{type} eq 'virtio') {
2135 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2136 $memory = ",max_hostmem=$bytes";
2137 } elsif ($qxlnum) {
2138 # from https://www.spice-space.org/multiple-monitors.html
2139 $memory = ",vgamem_mb=$vga->{memory}";
2140 my $ram = $vgamem_mb * 4;
2141 my $vram = $vgamem_mb * 2;
2142 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2143 } else {
2144 $memory = ",vgamem_mb=$vga->{memory}";
2145 }
2146 } elsif ($qxlnum && $id) {
2147 $memory = ",ram_size=67108864,vram_size=33554432";
2148 }
2149
2150 my $q35 = machine_type_is_q35($conf);
2151 my $vgaid = "vga" . ($id // '');
2152 my $pciaddr;
2153
2154 if ($q35 && $vgaid eq 'vga') {
2155 # the first display uses pcie.0 bus on q35 machines
2156 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2157 } else {
2158 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2159 }
2160
2161 return "$type,id=${vgaid}${memory}${pciaddr}";
2162}
2163
2164sub drive_is_cloudinit {
2165 my ($drive) = @_;
2166 return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2167}
2168
2169sub drive_is_cdrom {
2170 my ($drive, $exclude_cloudinit) = @_;
2171
2172 return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2173
2174 return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2175
2176}
2177
2178sub parse_number_sets {
2179 my ($set) = @_;
2180 my $res = [];
2181 foreach my $part (split(/;/, $set)) {
2182 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2183 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2184 push @$res, [ $1, $2 ];
2185 } else {
2186 die "invalid range: $part\n";
2187 }
2188 }
2189 return $res;
2190}
2191
2192sub parse_numa {
2193 my ($data) = @_;
2194
2195 my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2196 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2197 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2198 return $res;
2199}
2200
2201sub parse_hostpci {
2202 my ($value) = @_;
2203
2204 return undef if !$value;
2205
2206 my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2207
2208 my @idlist = split(/;/, $res->{host});
2209 delete $res->{host};
2210 foreach my $id (@idlist) {
2211 if ($id =~ m/\./) { # full id 00:00.1
2212 push @{$res->{pciid}}, {
2213 id => $id,
2214 };
2215 } else { # partial id 00:00
2216 $res->{pciid} = PVE::SysFSTools::lspci($id);
2217 }
2218 }
2219 return $res;
2220}
2221
2222# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2223sub parse_net {
2224 my ($data) = @_;
2225
2226 my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2227 if ($@) {
2228 warn $@;
2229 return undef;
2230 }
2231 if (!defined($res->{macaddr})) {
2232 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2233 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2234 }
2235 return $res;
2236}
2237
2238# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2239sub parse_ipconfig {
2240 my ($data) = @_;
2241
2242 my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2243 if ($@) {
2244 warn $@;
2245 return undef;
2246 }
2247
2248 if ($res->{gw} && !$res->{ip}) {
2249 warn 'gateway specified without specifying an IP address';
2250 return undef;
2251 }
2252 if ($res->{gw6} && !$res->{ip6}) {
2253 warn 'IPv6 gateway specified without specifying an IPv6 address';
2254 return undef;
2255 }
2256 if ($res->{gw} && $res->{ip} eq 'dhcp') {
2257 warn 'gateway specified together with DHCP';
2258 return undef;
2259 }
2260 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2261 # gw6 + auto/dhcp
2262 warn "IPv6 gateway specified together with $res->{ip6} address";
2263 return undef;
2264 }
2265
2266 if (!$res->{ip} && !$res->{ip6}) {
2267 return { ip => 'dhcp', ip6 => 'dhcp' };
2268 }
2269
2270 return $res;
2271}
2272
2273sub print_net {
2274 my $net = shift;
2275
2276 return PVE::JSONSchema::print_property_string($net, $net_fmt);
2277}
2278
2279sub add_random_macs {
2280 my ($settings) = @_;
2281
2282 foreach my $opt (keys %$settings) {
2283 next if $opt !~ m/^net(\d+)$/;
2284 my $net = parse_net($settings->{$opt});
2285 next if !$net;
2286 $settings->{$opt} = print_net($net);
2287 }
2288}
2289
2290sub vm_is_volid_owner {
2291 my ($storecfg, $vmid, $volid) = @_;
2292
2293 if ($volid !~ m|^/|) {
2294 my ($path, $owner);
2295 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2296 if ($owner && ($owner == $vmid)) {
2297 return 1;
2298 }
2299 }
2300
2301 return undef;
2302}
2303
2304sub split_flagged_list {
2305 my $text = shift || '';
2306 $text =~ s/[,;]/ /g;
2307 $text =~ s/^\s+//;
2308 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2309}
2310
2311sub join_flagged_list {
2312 my ($how, $lst) = @_;
2313 join $how, map { $lst->{$_} . $_ } keys %$lst;
2314}
2315
2316sub vmconfig_delete_pending_option {
2317 my ($conf, $key, $force) = @_;
2318
2319 delete $conf->{pending}->{$key};
2320 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2321 $pending_delete_hash->{$key} = $force ? '!' : '';
2322 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2323}
2324
2325sub vmconfig_undelete_pending_option {
2326 my ($conf, $key) = @_;
2327
2328 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2329 delete $pending_delete_hash->{$key};
2330
2331 if (%$pending_delete_hash) {
2332 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2333 } else {
2334 delete $conf->{pending}->{delete};
2335 }
2336}
2337
2338sub vmconfig_register_unused_drive {
2339 my ($storecfg, $vmid, $conf, $drive) = @_;
2340
2341 if (drive_is_cloudinit($drive)) {
2342 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2343 warn $@ if $@;
2344 } elsif (!drive_is_cdrom($drive)) {
2345 my $volid = $drive->{file};
2346 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2347 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2348 }
2349 }
2350}
2351
2352sub vmconfig_cleanup_pending {
2353 my ($conf) = @_;
2354
2355 # remove pending changes when nothing changed
2356 my $changes;
2357 foreach my $opt (keys %{$conf->{pending}}) {
2358 if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq $conf->{$opt})) {
2359 $changes = 1;
2360 delete $conf->{pending}->{$opt};
2361 }
2362 }
2363
2364 my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2365 my $pending_delete_hash = {};
2366 while (my ($opt, $force) = each %$current_delete_hash) {
2367 if (defined($conf->{$opt})) {
2368 $pending_delete_hash->{$opt} = $force;
2369 } else {
2370 $changes = 1;
2371 }
2372 }
2373
2374 if (%$pending_delete_hash) {
2375 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2376 } else {
2377 delete $conf->{pending}->{delete};
2378 }
2379
2380 return $changes;
2381}
2382
2383# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2384my $smbios1_fmt = {
2385 uuid => {
2386 type => 'string',
2387 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2388 format_description => 'UUID',
2389 description => "Set SMBIOS1 UUID.",
2390 optional => 1,
2391 },
2392 version => {
2393 type => 'string',
2394 pattern => '[A-Za-z0-9+\/]+={0,2}',
2395 format_description => 'Base64 encoded string',
2396 description => "Set SMBIOS1 version.",
2397 optional => 1,
2398 },
2399 serial => {
2400 type => 'string',
2401 pattern => '[A-Za-z0-9+\/]+={0,2}',
2402 format_description => 'Base64 encoded string',
2403 description => "Set SMBIOS1 serial number.",
2404 optional => 1,
2405 },
2406 manufacturer => {
2407 type => 'string',
2408 pattern => '[A-Za-z0-9+\/]+={0,2}',
2409 format_description => 'Base64 encoded string',
2410 description => "Set SMBIOS1 manufacturer.",
2411 optional => 1,
2412 },
2413 product => {
2414 type => 'string',
2415 pattern => '[A-Za-z0-9+\/]+={0,2}',
2416 format_description => 'Base64 encoded string',
2417 description => "Set SMBIOS1 product ID.",
2418 optional => 1,
2419 },
2420 sku => {
2421 type => 'string',
2422 pattern => '[A-Za-z0-9+\/]+={0,2}',
2423 format_description => 'Base64 encoded string',
2424 description => "Set SMBIOS1 SKU string.",
2425 optional => 1,
2426 },
2427 family => {
2428 type => 'string',
2429 pattern => '[A-Za-z0-9+\/]+={0,2}',
2430 format_description => 'Base64 encoded string',
2431 description => "Set SMBIOS1 family string.",
2432 optional => 1,
2433 },
2434 base64 => {
2435 type => 'boolean',
2436 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2437 optional => 1,
2438 },
2439};
2440
2441sub parse_smbios1 {
2442 my ($data) = @_;
2443
2444 my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2445 warn $@ if $@;
2446 return $res;
2447}
2448
2449sub print_smbios1 {
2450 my ($smbios1) = @_;
2451 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2452}
2453
2454PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2455
2456PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2457sub verify_bootdisk {
2458 my ($value, $noerr) = @_;
2459
2460 return $value if is_valid_drivename($value);
2461
2462 return undef if $noerr;
2463
2464 die "invalid boot disk '$value'\n";
2465}
2466
2467sub parse_watchdog {
2468 my ($value) = @_;
2469
2470 return undef if !$value;
2471
2472 my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2473 warn $@ if $@;
2474 return $res;
2475}
2476
2477sub parse_guest_agent {
2478 my ($value) = @_;
2479
2480 return {} if !defined($value->{agent});
2481
2482 my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2483 warn $@ if $@;
2484
2485 # if the agent is disabled ignore the other potentially set properties
2486 return {} if !$res->{enabled};
2487 return $res;
2488}
2489
2490sub parse_vga {
2491 my ($value) = @_;
2492
2493 return {} if !$value;
2494 my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2495 warn $@ if $@;
2496 return $res;
2497}
2498
2499PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2500sub verify_usb_device {
2501 my ($value, $noerr) = @_;
2502
2503 return $value if parse_usb_device($value);
2504
2505 return undef if $noerr;
2506
2507 die "unable to parse usb device\n";
2508}
2509
2510# add JSON properties for create and set function
2511sub json_config_properties {
2512 my $prop = shift;
2513
2514 foreach my $opt (keys %$confdesc) {
2515 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2516 $prop->{$opt} = $confdesc->{$opt};
2517 }
2518
2519 return $prop;
2520}
2521
2522# return copy of $confdesc_cloudinit to generate documentation
2523sub cloudinit_config_properties {
2524
2525 return dclone($confdesc_cloudinit);
2526}
2527
2528sub check_type {
2529 my ($key, $value) = @_;
2530
2531 die "unknown setting '$key'\n" if !$confdesc->{$key};
2532
2533 my $type = $confdesc->{$key}->{type};
2534
2535 if (!defined($value)) {
2536 die "got undefined value\n";
2537 }
2538
2539 if ($value =~ m/[\n\r]/) {
2540 die "property contains a line feed\n";
2541 }
2542
2543 if ($type eq 'boolean') {
2544 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2545 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2546 die "type check ('boolean') failed - got '$value'\n";
2547 } elsif ($type eq 'integer') {
2548 return int($1) if $value =~ m/^(\d+)$/;
2549 die "type check ('integer') failed - got '$value'\n";
2550 } elsif ($type eq 'number') {
2551 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2552 die "type check ('number') failed - got '$value'\n";
2553 } elsif ($type eq 'string') {
2554 if (my $fmt = $confdesc->{$key}->{format}) {
2555 PVE::JSONSchema::check_format($fmt, $value);
2556 return $value;
2557 }
2558 $value =~ s/^\"(.*)\"$/$1/;
2559 return $value;
2560 } else {
2561 die "internal error"
2562 }
2563}
2564
2565sub touch_config {
2566 my ($vmid) = @_;
2567
2568 my $conf = PVE::QemuConfig->config_file($vmid);
2569 utime undef, undef, $conf;
2570}
2571
2572sub destroy_vm {
2573 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2574
2575 my $conffile = PVE::QemuConfig->config_file($vmid);
2576
2577 my $conf = PVE::QemuConfig->load_config($vmid);
2578
2579 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2580
2581 if ($conf->{template}) {
2582 # check if any base image is still used by a linked clone
2583 foreach_drive($conf, sub {
2584 my ($ds, $drive) = @_;
2585
2586 return if drive_is_cdrom($drive);
2587
2588 my $volid = $drive->{file};
2589
2590 return if !$volid || $volid =~ m|^/|;
2591
2592 die "base volume '$volid' is still in use by linked cloned\n"
2593 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2594
2595 });
2596 }
2597
2598 # only remove disks owned by this VM
2599 foreach_drive($conf, sub {
2600 my ($ds, $drive) = @_;
2601
2602 return if drive_is_cdrom($drive, 1);
2603
2604 my $volid = $drive->{file};
2605
2606 return if !$volid || $volid =~ m|^/|;
2607
2608 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2609 return if !$path || !$owner || ($owner != $vmid);
2610
2611 eval {
2612 PVE::Storage::vdisk_free($storecfg, $volid);
2613 };
2614 warn "Could not remove disk '$volid', check manually: $@" if $@;
2615
2616 });
2617
2618 if ($keep_empty_config) {
2619 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2620 } else {
2621 unlink $conffile;
2622 }
2623
2624 # also remove unused disk
2625 eval {
2626 my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2627
2628 eval {
2629 PVE::Storage::foreach_volid($dl, sub {
2630 my ($volid, $sid, $volname, $d) = @_;
2631 PVE::Storage::vdisk_free($storecfg, $volid);
2632 });
2633 };
2634 warn $@ if $@;
2635
2636 };
2637 warn $@ if $@;
2638}
2639
2640sub parse_vm_config {
2641 my ($filename, $raw) = @_;
2642
2643 return undef if !defined($raw);
2644
2645 my $res = {
2646 digest => Digest::SHA::sha1_hex($raw),
2647 snapshots => {},
2648 pending => {},
2649 };
2650
2651 $filename =~ m|/qemu-server/(\d+)\.conf$|
2652 || die "got strange filename '$filename'";
2653
2654 my $vmid = $1;
2655
2656 my $conf = $res;
2657 my $descr;
2658 my $section = '';
2659
2660 my @lines = split(/\n/, $raw);
2661 foreach my $line (@lines) {
2662 next if $line =~ m/^\s*$/;
2663
2664 if ($line =~ m/^\[PENDING\]\s*$/i) {
2665 $section = 'pending';
2666 if (defined($descr)) {
2667 $descr =~ s/\s+$//;
2668 $conf->{description} = $descr;
2669 }
2670 $descr = undef;
2671 $conf = $res->{$section} = {};
2672 next;
2673
2674 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2675 $section = $1;
2676 if (defined($descr)) {
2677 $descr =~ s/\s+$//;
2678 $conf->{description} = $descr;
2679 }
2680 $descr = undef;
2681 $conf = $res->{snapshots}->{$section} = {};
2682 next;
2683 }
2684
2685 if ($line =~ m/^\#(.*)\s*$/) {
2686 $descr = '' if !defined($descr);
2687 $descr .= PVE::Tools::decode_text($1) . "\n";
2688 next;
2689 }
2690
2691 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2692 $descr = '' if !defined($descr);
2693 $descr .= PVE::Tools::decode_text($2);
2694 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2695 $conf->{snapstate} = $1;
2696 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2697 my $key = $1;
2698 my $value = $2;
2699 $conf->{$key} = $value;
2700 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2701 my $value = $1;
2702 if ($section eq 'pending') {
2703 $conf->{delete} = $value; # we parse this later
2704 } else {
2705 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2706 }
2707 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2708 my $key = $1;
2709 my $value = $2;
2710 eval { $value = check_type($key, $value); };
2711 if ($@) {
2712 warn "vm $vmid - unable to parse value of '$key' - $@";
2713 } else {
2714 $key = 'ide2' if $key eq 'cdrom';
2715 my $fmt = $confdesc->{$key}->{format};
2716 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2717 my $v = parse_drive($key, $value);
2718 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2719 $v->{file} = $volid;
2720 $value = print_drive($vmid, $v);
2721 } else {
2722 warn "vm $vmid - unable to parse value of '$key'\n";
2723 next;
2724 }
2725 }
2726
2727 $conf->{$key} = $value;
2728 }
2729 }
2730 }
2731
2732 if (defined($descr)) {
2733 $descr =~ s/\s+$//;
2734 $conf->{description} = $descr;
2735 }
2736 delete $res->{snapstate}; # just to be sure
2737
2738 return $res;
2739}
2740
2741sub write_vm_config {
2742 my ($filename, $conf) = @_;
2743
2744 delete $conf->{snapstate}; # just to be sure
2745
2746 if ($conf->{cdrom}) {
2747 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2748 $conf->{ide2} = $conf->{cdrom};
2749 delete $conf->{cdrom};
2750 }
2751
2752 # we do not use 'smp' any longer
2753 if ($conf->{sockets}) {
2754 delete $conf->{smp};
2755 } elsif ($conf->{smp}) {
2756 $conf->{sockets} = $conf->{smp};
2757 delete $conf->{cores};
2758 delete $conf->{smp};
2759 }
2760
2761 my $used_volids = {};
2762
2763 my $cleanup_config = sub {
2764 my ($cref, $pending, $snapname) = @_;
2765
2766 foreach my $key (keys %$cref) {
2767 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2768 $key eq 'snapstate' || $key eq 'pending';
2769 my $value = $cref->{$key};
2770 if ($key eq 'delete') {
2771 die "propertry 'delete' is only allowed in [PENDING]\n"
2772 if !$pending;
2773 # fixme: check syntax?
2774 next;
2775 }
2776 eval { $value = check_type($key, $value); };
2777 die "unable to parse value of '$key' - $@" if $@;
2778
2779 $cref->{$key} = $value;
2780
2781 if (!$snapname && is_valid_drivename($key)) {
2782 my $drive = parse_drive($key, $value);
2783 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2784 }
2785 }
2786 };
2787
2788 &$cleanup_config($conf);
2789
2790 &$cleanup_config($conf->{pending}, 1);
2791
2792 foreach my $snapname (keys %{$conf->{snapshots}}) {
2793 die "internal error" if $snapname eq 'pending';
2794 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2795 }
2796
2797 # remove 'unusedX' settings if we re-add a volume
2798 foreach my $key (keys %$conf) {
2799 my $value = $conf->{$key};
2800 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2801 delete $conf->{$key};
2802 }
2803 }
2804
2805 my $generate_raw_config = sub {
2806 my ($conf, $pending) = @_;
2807
2808 my $raw = '';
2809
2810 # add description as comment to top of file
2811 if (defined(my $descr = $conf->{description})) {
2812 if ($descr) {
2813 foreach my $cl (split(/\n/, $descr)) {
2814 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2815 }
2816 } else {
2817 $raw .= "#\n" if $pending;
2818 }
2819 }
2820
2821 foreach my $key (sort keys %$conf) {
2822 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2823 $raw .= "$key: $conf->{$key}\n";
2824 }
2825 return $raw;
2826 };
2827
2828 my $raw = &$generate_raw_config($conf);
2829
2830 if (scalar(keys %{$conf->{pending}})){
2831 $raw .= "\n[PENDING]\n";
2832 $raw .= &$generate_raw_config($conf->{pending}, 1);
2833 }
2834
2835 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2836 $raw .= "\n[$snapname]\n";
2837 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2838 }
2839
2840 return $raw;
2841}
2842
2843sub load_defaults {
2844
2845 my $res = {};
2846
2847 # we use static defaults from our JSON schema configuration
2848 foreach my $key (keys %$confdesc) {
2849 if (defined(my $default = $confdesc->{$key}->{default})) {
2850 $res->{$key} = $default;
2851 }
2852 }
2853
2854 return $res;
2855}
2856
2857sub config_list {
2858 my $vmlist = PVE::Cluster::get_vmlist();
2859 my $res = {};
2860 return $res if !$vmlist || !$vmlist->{ids};
2861 my $ids = $vmlist->{ids};
2862
2863 foreach my $vmid (keys %$ids) {
2864 my $d = $ids->{$vmid};
2865 next if !$d->{node} || $d->{node} ne $nodename;
2866 next if !$d->{type} || $d->{type} ne 'qemu';
2867 $res->{$vmid}->{exists} = 1;
2868 }
2869 return $res;
2870}
2871
2872# test if VM uses local resources (to prevent migration)
2873sub check_local_resources {
2874 my ($conf, $noerr) = @_;
2875
2876 my @loc_res = ();
2877
2878 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2879 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2880
2881 push @loc_res, "ivshmem" if $conf->{ivshmem};
2882
2883 foreach my $k (keys %$conf) {
2884 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2885 # sockets are safe: they will recreated be on the target side post-migrate
2886 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2887 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2888 }
2889
2890 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2891
2892 return \@loc_res;
2893}
2894
2895# check if used storages are available on all nodes (use by migrate)
2896sub check_storage_availability {
2897 my ($storecfg, $conf, $node) = @_;
2898
2899 foreach_drive($conf, sub {
2900 my ($ds, $drive) = @_;
2901
2902 my $volid = $drive->{file};
2903 return if !$volid;
2904
2905 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2906 return if !$sid;
2907
2908 # check if storage is available on both nodes
2909 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2910 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2911 });
2912}
2913
2914# list nodes where all VM images are available (used by has_feature API)
2915sub shared_nodes {
2916 my ($conf, $storecfg) = @_;
2917
2918 my $nodelist = PVE::Cluster::get_nodelist();
2919 my $nodehash = { map { $_ => 1 } @$nodelist };
2920 my $nodename = PVE::INotify::nodename();
2921
2922 foreach_drive($conf, sub {
2923 my ($ds, $drive) = @_;
2924
2925 my $volid = $drive->{file};
2926 return if !$volid;
2927
2928 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2929 if ($storeid) {
2930 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2931 if ($scfg->{disable}) {
2932 $nodehash = {};
2933 } elsif (my $avail = $scfg->{nodes}) {
2934 foreach my $node (keys %$nodehash) {
2935 delete $nodehash->{$node} if !$avail->{$node};
2936 }
2937 } elsif (!$scfg->{shared}) {
2938 foreach my $node (keys %$nodehash) {
2939 delete $nodehash->{$node} if $node ne $nodename
2940 }
2941 }
2942 }
2943 });
2944
2945 return $nodehash
2946}
2947
2948sub check_local_storage_availability {
2949 my ($conf, $storecfg) = @_;
2950
2951 my $nodelist = PVE::Cluster::get_nodelist();
2952 my $nodehash = { map { $_ => {} } @$nodelist };
2953
2954 foreach_drive($conf, sub {
2955 my ($ds, $drive) = @_;
2956
2957 my $volid = $drive->{file};
2958 return if !$volid;
2959
2960 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2961 if ($storeid) {
2962 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2963
2964 if ($scfg->{disable}) {
2965 foreach my $node (keys %$nodehash) {
2966 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2967 }
2968 } elsif (my $avail = $scfg->{nodes}) {
2969 foreach my $node (keys %$nodehash) {
2970 if (!$avail->{$node}) {
2971 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2972 }
2973 }
2974 }
2975 }
2976 });
2977
2978 foreach my $node (values %$nodehash) {
2979 if (my $unavail = $node->{unavailable_storages}) {
2980 $node->{unavailable_storages} = [ sort keys %$unavail ];
2981 }
2982 }
2983
2984 return $nodehash
2985}
2986
2987sub check_cmdline {
2988 my ($pidfile, $pid) = @_;
2989
2990 my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2991 if (defined($fh)) {
2992 my $line = <$fh>;
2993 $fh->close;
2994 return undef if !$line;
2995 my @param = split(/\0/, $line);
2996
2997 my $cmd = $param[0];
2998 return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2999
3000 for (my $i = 0; $i < scalar (@param); $i++) {
3001 my $p = $param[$i];
3002 next if !$p;
3003 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3004 my $p = $param[$i+1];
3005 return 1 if $p && ($p eq $pidfile);
3006 return undef;
3007 }
3008 }
3009 }
3010 return undef;
3011}
3012
3013sub check_running {
3014 my ($vmid, $nocheck, $node) = @_;
3015
3016 my $filename = PVE::QemuConfig->config_file($vmid, $node);
3017
3018 die "unable to find configuration file for VM $vmid - no such machine\n"
3019 if !$nocheck && ! -f $filename;
3020
3021 my $pidfile = pidfile_name($vmid);
3022
3023 if (my $fd = IO::File->new("<$pidfile")) {
3024 my $st = stat($fd);
3025 my $line = <$fd>;
3026 close($fd);
3027
3028 my $mtime = $st->mtime;
3029 if ($mtime > time()) {
3030 warn "file '$filename' modified in future\n";
3031 }
3032
3033 if ($line =~ m/^(\d+)$/) {
3034 my $pid = $1;
3035 if (check_cmdline($pidfile, $pid)) {
3036 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3037 return $pid;
3038 }
3039 }
3040 }
3041 }
3042
3043 return undef;
3044}
3045
3046sub vzlist {
3047
3048 my $vzlist = config_list();
3049
3050 my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3051
3052 while (defined(my $de = $fd->read)) {
3053 next if $de !~ m/^(\d+)\.pid$/;
3054 my $vmid = $1;
3055 next if !defined($vzlist->{$vmid});
3056 if (my $pid = check_running($vmid)) {
3057 $vzlist->{$vmid}->{pid} = $pid;
3058 }
3059 }
3060
3061 return $vzlist;
3062}
3063
3064sub disksize {
3065 my ($storecfg, $conf) = @_;
3066
3067 my $bootdisk = $conf->{bootdisk};
3068 return undef if !$bootdisk;
3069 return undef if !is_valid_drivename($bootdisk);
3070
3071 return undef if !$conf->{$bootdisk};
3072
3073 my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3074 return undef if !defined($drive);
3075
3076 return undef if drive_is_cdrom($drive);
3077
3078 my $volid = $drive->{file};
3079 return undef if !$volid;
3080
3081 return $drive->{size};
3082}
3083
3084our $vmstatus_return_properties = {
3085 vmid => get_standard_option('pve-vmid'),
3086 status => {
3087 description => "Qemu process status.",
3088 type => 'string',
3089 enum => ['stopped', 'running'],
3090 },
3091 maxmem => {
3092 description => "Maximum memory in bytes.",
3093 type => 'integer',
3094 optional => 1,
3095 renderer => 'bytes',
3096 },
3097 maxdisk => {
3098 description => "Root disk size in bytes.",
3099 type => 'integer',
3100 optional => 1,
3101 renderer => 'bytes',
3102 },
3103 name => {
3104 description => "VM name.",
3105 type => 'string',
3106 optional => 1,
3107 },
3108 qmpstatus => {
3109 description => "Qemu QMP agent status.",
3110 type => 'string',
3111 optional => 1,
3112 },
3113 pid => {
3114 description => "PID of running qemu process.",
3115 type => 'integer',
3116 optional => 1,
3117 },
3118 uptime => {
3119 description => "Uptime.",
3120 type => 'integer',
3121 optional => 1,
3122 renderer => 'duration',
3123 },
3124 cpus => {
3125 description => "Maximum usable CPUs.",
3126 type => 'number',
3127 optional => 1,
3128 },
3129 lock => {
3130 description => "The current config lock, if any.",
3131 type => 'string',
3132 optional => 1,
3133 }
3134};
3135
3136my $last_proc_pid_stat;
3137
3138# get VM status information
3139# This must be fast and should not block ($full == false)
3140# We only query KVM using QMP if $full == true (this can be slow)
3141sub vmstatus {
3142 my ($opt_vmid, $full) = @_;
3143
3144 my $res = {};
3145
3146 my $storecfg = PVE::Storage::config();
3147
3148 my $list = vzlist();
3149 my $defaults = load_defaults();
3150
3151 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3152
3153 my $cpucount = $cpuinfo->{cpus} || 1;
3154
3155 foreach my $vmid (keys %$list) {
3156 next if $opt_vmid && ($vmid ne $opt_vmid);
3157
3158 my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3159 my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3160
3161 my $d = { vmid => $vmid };
3162 $d->{pid} = $list->{$vmid}->{pid};
3163
3164 # fixme: better status?
3165 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3166
3167 my $size = disksize($storecfg, $conf);
3168 if (defined($size)) {
3169 $d->{disk} = 0; # no info available
3170 $d->{maxdisk} = $size;
3171 } else {
3172 $d->{disk} = 0;
3173 $d->{maxdisk} = 0;
3174 }
3175
3176 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3177 * ($conf->{cores} || $defaults->{cores});
3178 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3179 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3180
3181 $d->{name} = $conf->{name} || "VM $vmid";
3182 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3183 : $defaults->{memory}*(1024*1024);
3184
3185 if ($conf->{balloon}) {
3186 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3187 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3188 : $defaults->{shares};
3189 }
3190
3191 $d->{uptime} = 0;
3192 $d->{cpu} = 0;
3193 $d->{mem} = 0;
3194
3195 $d->{netout} = 0;
3196 $d->{netin} = 0;
3197
3198 $d->{diskread} = 0;
3199 $d->{diskwrite} = 0;
3200
3201 $d->{template} = PVE::QemuConfig->is_template($conf);
3202
3203 $d->{serial} = 1 if conf_has_serial($conf);
3204 $d->{lock} = $conf->{lock} if $conf->{lock};
3205
3206 $res->{$vmid} = $d;
3207 }
3208
3209 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3210 foreach my $dev (keys %$netdev) {
3211 next if $dev !~ m/^tap([1-9]\d*)i/;
3212 my $vmid = $1;
3213 my $d = $res->{$vmid};
3214 next if !$d;
3215
3216 $d->{netout} += $netdev->{$dev}->{receive};
3217 $d->{netin} += $netdev->{$dev}->{transmit};
3218
3219 if ($full) {
3220 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3221 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3222 }
3223
3224 }
3225
3226 my $ctime = gettimeofday;
3227
3228 foreach my $vmid (keys %$list) {
3229
3230 my $d = $res->{$vmid};
3231 my $pid = $d->{pid};
3232 next if !$pid;
3233
3234 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3235 next if !$pstat; # not running
3236
3237 my $used = $pstat->{utime} + $pstat->{stime};
3238
3239 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3240
3241 if ($pstat->{vsize}) {
3242 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3243 }
3244
3245 my $old = $last_proc_pid_stat->{$pid};
3246 if (!$old) {
3247 $last_proc_pid_stat->{$pid} = {
3248 time => $ctime,
3249 used => $used,
3250 cpu => 0,
3251 };
3252 next;
3253 }
3254
3255 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3256
3257 if ($dtime > 1000) {
3258 my $dutime = $used - $old->{used};
3259
3260 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3261 $last_proc_pid_stat->{$pid} = {
3262 time => $ctime,
3263 used => $used,
3264 cpu => $d->{cpu},
3265 };
3266 } else {
3267 $d->{cpu} = $old->{cpu};
3268 }
3269 }
3270
3271 return $res if !$full;
3272
3273 my $qmpclient = PVE::QMPClient->new();
3274
3275 my $ballooncb = sub {
3276 my ($vmid, $resp) = @_;
3277
3278 my $info = $resp->{'return'};
3279 return if !$info->{max_mem};
3280
3281 my $d = $res->{$vmid};
3282
3283 # use memory assigned to VM
3284 $d->{maxmem} = $info->{max_mem};
3285 $d->{balloon} = $info->{actual};
3286
3287 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3288 $d->{mem} = $info->{total_mem} - $info->{free_mem};
3289 $d->{freemem} = $info->{free_mem};
3290 }
3291
3292 $d->{ballooninfo} = $info;
3293 };
3294
3295 my $blockstatscb = sub {
3296 my ($vmid, $resp) = @_;
3297 my $data = $resp->{'return'} || [];
3298 my $totalrdbytes = 0;
3299 my $totalwrbytes = 0;
3300
3301 for my $blockstat (@$data) {
3302 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3303 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3304
3305 $blockstat->{device} =~ s/drive-//;
3306 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3307 }
3308 $res->{$vmid}->{diskread} = $totalrdbytes;
3309 $res->{$vmid}->{diskwrite} = $totalwrbytes;
3310 };
3311
3312 my $statuscb = sub {
3313 my ($vmid, $resp) = @_;
3314
3315 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3316 # this fails if ballon driver is not loaded, so this must be
3317 # the last commnand (following command are aborted if this fails).
3318 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3319
3320 my $status = 'unknown';
3321 if (!defined($status = $resp->{'return'}->{status})) {
3322 warn "unable to get VM status\n";
3323 return;
3324 }
3325
3326 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3327 };
3328
3329 foreach my $vmid (keys %$list) {
3330 next if $opt_vmid && ($vmid ne $opt_vmid);
3331 next if !$res->{$vmid}->{pid}; # not running
3332 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3333 }
3334
3335 $qmpclient->queue_execute(undef, 2);
3336
3337 foreach my $vmid (keys %$list) {
3338 next if $opt_vmid && ($vmid ne $opt_vmid);
3339 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3340 }
3341
3342 return $res;
3343}
3344
3345sub foreach_drive {
3346 my ($conf, $func, @param) = @_;
3347
3348 foreach my $ds (valid_drive_names()) {
3349 next if !defined($conf->{$ds});
3350
3351 my $drive = parse_drive($ds, $conf->{$ds});
3352 next if !$drive;
3353
3354 &$func($ds, $drive, @param);
3355 }
3356}
3357
3358sub foreach_volid {
3359 my ($conf, $func, @param) = @_;
3360
3361 my $volhash = {};
3362
3363 my $test_volid = sub {
3364 my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3365
3366 return if !$volid;
3367
3368 $volhash->{$volid}->{cdrom} //= 1;
3369 $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3370
3371 $volhash->{$volid}->{replicate} //= 0;
3372 $volhash->{$volid}->{replicate} = 1 if $replicate;
3373
3374 $volhash->{$volid}->{shared} //= 0;
3375 $volhash->{$volid}->{shared} = 1 if $shared;
3376
3377 $volhash->{$volid}->{referenced_in_config} //= 0;
3378 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3379
3380 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3381 if defined($snapname);
3382 $volhash->{$volid}->{size} = $size if $size;
3383 };
3384
3385 foreach_drive($conf, sub {
3386 my ($ds, $drive) = @_;
3387 $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3388 });
3389
3390 foreach my $snapname (keys %{$conf->{snapshots}}) {
3391 my $snap = $conf->{snapshots}->{$snapname};
3392 $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3393 foreach_drive($snap, sub {
3394 my ($ds, $drive) = @_;
3395 $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3396 });
3397 }
3398
3399 foreach my $volid (keys %$volhash) {
3400 &$func($volid, $volhash->{$volid}, @param);
3401 }
3402}
3403
3404sub conf_has_serial {
3405 my ($conf) = @_;
3406
3407 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3408 if ($conf->{"serial$i"}) {
3409 return 1;
3410 }
3411 }
3412
3413 return 0;
3414}
3415
3416sub conf_has_audio {
3417 my ($conf, $id) = @_;
3418
3419 $id //= 0;
3420 my $audio = $conf->{"audio$id"};
3421 return undef if !defined($audio);
3422
3423 my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3424 my $audiodriver = $audioproperties->{driver} // 'spice';
3425
3426 return {
3427 dev => $audioproperties->{device},
3428 dev_id => "audio$id",
3429 backend => $audiodriver,
3430 backend_id => "$audiodriver-backend${id}",
3431 };
3432}
3433
3434sub vga_conf_has_spice {
3435 my ($vga) = @_;
3436
3437 my $vgaconf = parse_vga($vga);
3438 my $vgatype = $vgaconf->{type};
3439 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3440
3441 return $1 || 1;
3442}
3443
3444my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3445sub get_host_arch() {
3446 $host_arch = (POSIX::uname())[4] if !$host_arch;
3447 return $host_arch;
3448}
3449
3450sub is_native($) {
3451 my ($arch) = @_;
3452 return get_host_arch() eq $arch;
3453}
3454
3455my $default_machines = {
3456 x86_64 => 'pc',
3457 aarch64 => 'virt',
3458};
3459
3460sub get_basic_machine_info {
3461 my ($conf, $forcemachine) = @_;
3462
3463 my $arch = $conf->{arch} // get_host_arch();
3464 my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3465 return ($arch, $machine);
3466}
3467
3468sub get_ovmf_files($) {
3469 my ($arch) = @_;
3470
3471 my $ovmf = $OVMF->{$arch}
3472 or die "no OVMF images known for architecture '$arch'\n";
3473
3474 return @$ovmf;
3475}
3476
3477my $Arch2Qemu = {
3478 aarch64 => '/usr/bin/qemu-system-aarch64',
3479 x86_64 => '/usr/bin/qemu-system-x86_64',
3480};
3481sub get_command_for_arch($) {
3482 my ($arch) = @_;
3483 return '/usr/bin/kvm' if is_native($arch);
3484
3485 my $cmd = $Arch2Qemu->{$arch}
3486 or die "don't know how to emulate architecture '$arch'\n";
3487 return $cmd;
3488}
3489
3490sub get_cpu_options {
3491 my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3492
3493 my $cpuFlags = [];
3494 my $ostype = $conf->{ostype};
3495
3496 my $cpu = $kvm ? "kvm64" : "qemu64";
3497 if ($arch eq 'aarch64') {
3498 $cpu = 'cortex-a57';
3499 }
3500 my $hv_vendor_id;
3501 if (my $cputype = $conf->{cpu}) {
3502 my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3503 or die "Cannot parse cpu description: $cputype\n";
3504 $cpu = $cpuconf->{cputype};
3505 $kvm_off = 1 if $cpuconf->{hidden};
3506 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3507
3508 if (defined(my $flags = $cpuconf->{flags})) {
3509 push @$cpuFlags, split(";", $flags);
3510 }
3511 }
3512
3513 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3514
3515 push @$cpuFlags , '-x2apic'
3516 if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3517
3518 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3519
3520 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3521
3522 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3523
3524 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3525 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3526 }
3527
3528 add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3529
3530 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3531
3532 push @$cpuFlags, 'kvm=off' if $kvm_off;
3533
3534 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3535 push @$cpuFlags, "vendor=${cpu_vendor}"
3536 if $cpu_vendor ne 'default';
3537 } elsif ($arch ne 'aarch64') {
3538 die "internal error"; # should not happen
3539 }
3540
3541 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3542
3543 return ('-cpu', $cpu);
3544}
3545
3546sub config_to_command {
3547 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3548
3549 my $cmd = [];
3550 my $globalFlags = [];
3551 my $machineFlags = [];
3552 my $rtcFlags = [];
3553 my $devices = [];
3554 my $pciaddr = '';
3555 my $bridges = {};
3556 my $kvmver = kvm_user_version();
3557 my $vernum = 0; # unknown
3558 my $ostype = $conf->{ostype};
3559 my $winversion = windows_version($ostype);
3560 my $kvm = $conf->{kvm};
3561
3562 my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3563 $kvm //= 1 if is_native($arch);
3564
3565 if ($kvm) {
3566 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3567 if !defined kvm_version();
3568 }
3569
3570 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3571 $vernum = $1*1000000+$2*1000;
3572 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3573 $vernum = $1*1000000+$2*1000+$3;
3574 }
3575
3576 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3577
3578 my $have_ovz = -f '/proc/vz/vestat';
3579
3580 my $q35 = machine_type_is_q35($conf);
3581 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3582 my $use_old_bios_files = undef;
3583 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3584
3585 my $cpuunits = defined($conf->{cpuunits}) ?
3586 $conf->{cpuunits} : $defaults->{cpuunits};
3587
3588 push @$cmd, get_command_for_arch($arch);
3589
3590 push @$cmd, '-id', $vmid;
3591
3592 my $vmname = $conf->{name} || "vm$vmid";
3593
3594 push @$cmd, '-name', $vmname;
3595
3596 my $use_virtio = 0;
3597
3598 my $qmpsocket = qmp_socket($vmid);
3599 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3600 push @$cmd, '-mon', "chardev=qmp,mode=control";
3601
3602 if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3603 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3604 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3605 }
3606
3607 push @$cmd, '-pidfile' , pidfile_name($vmid);
3608
3609 push @$cmd, '-daemonize';
3610
3611 if ($conf->{smbios1}) {
3612 my $smbios_conf = parse_smbios1($conf->{smbios1});
3613 if ($smbios_conf->{base64}) {
3614 # Do not pass base64 flag to qemu
3615 delete $smbios_conf->{base64};
3616 my $smbios_string = "";
3617 foreach my $key (keys %$smbios_conf) {
3618 my $value;
3619 if ($key eq "uuid") {
3620 $value = $smbios_conf->{uuid}
3621 } else {
3622 $value = decode_base64($smbios_conf->{$key});
3623 }
3624 # qemu accepts any binary data, only commas need escaping by double comma
3625 $value =~ s/,/,,/g;
3626 $smbios_string .= "," . $key . "=" . $value if $value;
3627 }
3628 push @$cmd, '-smbios', "type=1" . $smbios_string;
3629 } else {
3630 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3631 }
3632 }
3633
3634 if ($conf->{vmgenid}) {
3635 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3636 }
3637
3638 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3639 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3640 die "uefi base image not found\n" if ! -f $ovmf_code;
3641
3642 my $path;
3643 my $format;
3644 if (my $efidisk = $conf->{efidisk0}) {
3645 my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3646 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3647 $format = $d->{format};
3648 if ($storeid) {
3649 $path = PVE::Storage::path($storecfg, $d->{file});
3650 if (!defined($format)) {
3651 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3652 $format = qemu_img_format($scfg, $volname);
3653 }
3654 } else {
3655 $path = $d->{file};
3656 die "efidisk format must be specified\n"
3657 if !defined($format);
3658 }
3659 } else {
3660 warn "no efidisk configured! Using temporary efivars disk.\n";
3661 $path = "/tmp/$vmid-ovmf.fd";
3662 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3663 $format = 'raw';
3664 }
3665
3666 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3667 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3668 }
3669
3670 # load q35 config
3671 if ($q35) {
3672 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3673 if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3674 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3675 } else {
3676 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3677 }
3678 }
3679
3680 # add usb controllers
3681 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3682 push @$devices, @usbcontrollers if @usbcontrollers;
3683 my $vga = parse_vga($conf->{vga});
3684
3685 my $qxlnum = vga_conf_has_spice($conf->{vga});
3686 $vga->{type} = 'qxl' if $qxlnum;
3687
3688 if (!$vga->{type}) {
3689 if ($arch eq 'aarch64') {
3690 $vga->{type} = 'virtio';
3691 } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3692 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3693 } else {
3694 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3695 }
3696 }
3697
3698 # enable absolute mouse coordinates (needed by vnc)
3699 my $tablet;
3700 if (defined($conf->{tablet})) {
3701 $tablet = $conf->{tablet};
3702 } else {
3703 $tablet = $defaults->{tablet};
3704 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3705 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3706 }
3707
3708 if ($tablet) {
3709 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3710 my $kbd = print_keyboarddevice_full($conf, $arch);
3711 push @$devices, '-device', $kbd if defined($kbd);
3712 }
3713
3714 my $kvm_off = 0;
3715 my $gpu_passthrough;
3716
3717 # host pci devices
3718 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3719 my $d = parse_hostpci($conf->{"hostpci$i"});
3720 next if !$d;
3721
3722 my $pcie = $d->{pcie};
3723 if ($pcie) {
3724 die "q35 machine model is not enabled" if !$q35;
3725 # win7 wants to have the pcie devices directly on the pcie bus
3726 # instead of in the root port
3727 if ($winversion == 7) {
3728 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3729 } else {
3730 $pciaddr = print_pcie_addr("hostpci$i");
3731 }
3732 } else {
3733 $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3734 }
3735
3736 my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3737 my $romfile = $d->{romfile};
3738
3739 my $xvga = '';
3740 if ($d->{'x-vga'}) {
3741 $xvga = ',x-vga=on';
3742 $kvm_off = 1;
3743 $vga->{type} = 'none' if !defined($conf->{vga});
3744 $gpu_passthrough = 1;
3745
3746 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3747 $xvga = "";
3748 }
3749 }
3750 my $pcidevices = $d->{pciid};
3751 my $multifunction = 1 if @$pcidevices > 1;
3752 my $sysfspath;
3753 if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3754 my $id = $pcidevices->[0]->{id};
3755 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3756 $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3757 } elsif ($d->{mdev}) {
3758 warn "ignoring mediated device with multifunction device\n";
3759 }
3760
3761 my $j=0;
3762 foreach my $pcidevice (@$pcidevices) {
3763
3764 my $id = "hostpci$i";
3765 $id .= ".$j" if $multifunction;
3766 my $addr = $pciaddr;
3767 $addr .= ".$j" if $multifunction;
3768 my $devicestr = "vfio-pci";
3769 if ($sysfspath) {
3770 $devicestr .= ",sysfsdev=$sysfspath";
3771 } else {
3772 $devicestr .= ",host=$pcidevice->{id}";
3773 }
3774 $devicestr .= ",id=$id$addr";
3775
3776 if($j == 0){
3777 $devicestr .= "$rombar$xvga";
3778 $devicestr .= ",multifunction=on" if $multifunction;
3779 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3780 }
3781
3782 push @$devices, '-device', $devicestr;
3783 $j++;
3784 }
3785 }
3786
3787 # usb devices
3788 my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3789 push @$devices, @usbdevices if @usbdevices;
3790 # serial devices
3791 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3792 if (my $path = $conf->{"serial$i"}) {
3793 if ($path eq 'socket') {
3794 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3795 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3796 # On aarch64, serial0 is the UART device. Qemu only allows
3797 # connecting UART devices via the '-serial' command line, as
3798 # the device has a fixed slot on the hardware...
3799 if ($arch eq 'aarch64' && $i == 0) {
3800 push @$devices, '-serial', "chardev:serial$i";
3801 } else {
3802 push @$devices, '-device', "isa-serial,chardev=serial$i";
3803 }
3804 } else {
3805 die "no such serial device\n" if ! -c $path;
3806 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3807 push @$devices, '-device', "isa-serial,chardev=serial$i";
3808 }
3809 }
3810 }
3811
3812 # parallel devices
3813 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3814 if (my $path = $conf->{"parallel$i"}) {
3815 die "no such parallel device\n" if ! -c $path;
3816 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3817 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3818 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3819 }
3820 }
3821
3822 if (my $audio = conf_has_audio($conf)) {
3823
3824 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3825
3826 my $id = $audio->{dev_id};
3827 if ($audio->{dev} eq 'AC97') {
3828 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3829 } elsif ($audio->{dev} =~ /intel\-hda$/) {
3830 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3831 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3832 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3833 } else {
3834 die "unkown audio device '$audio->{dev}', implement me!";
3835 }
3836
3837 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3838 }
3839
3840 my $sockets = 1;
3841 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3842 $sockets = $conf->{sockets} if $conf->{sockets};
3843
3844 my $cores = $conf->{cores} || 1;
3845
3846 my $maxcpus = $sockets * $cores;
3847
3848 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3849
3850 my $allowed_vcpus = $cpuinfo->{cpus};
3851
3852 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3853 if ($allowed_vcpus < $maxcpus);
3854
3855 if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3856
3857 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3858 for (my $i = 2; $i <= $vcpus; $i++) {
3859 my $cpustr = print_cpu_device($conf,$i);
3860 push @$cmd, '-device', $cpustr;
3861 }
3862
3863 } else {
3864
3865 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3866 }
3867 push @$cmd, '-nodefaults';
3868
3869 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3870
3871 my $bootindex_hash = {};
3872 my $i = 1;
3873 foreach my $o (split(//, $bootorder)) {
3874 $bootindex_hash->{$o} = $i*100;
3875 $i++;
3876 }
3877
3878 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3879
3880 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3881
3882 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3883
3884 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3885 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3886 my $socket = vnc_socket($vmid);
3887 push @$cmd, '-vnc', "unix:$socket,password";
3888 } else {
3889 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3890 push @$cmd, '-nographic';
3891 }
3892
3893 # time drift fix
3894 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3895
3896 my $useLocaltime = $conf->{localtime};
3897
3898 if ($winversion >= 5) { # windows
3899 $useLocaltime = 1 if !defined($conf->{localtime});
3900
3901 # use time drift fix when acpi is enabled
3902 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3903 $tdf = 1 if !defined($conf->{tdf});
3904 }
3905 }
3906
3907 if ($winversion >= 6) {
3908 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3909 push @$cmd, '-no-hpet';
3910 }
3911
3912 push @$rtcFlags, 'driftfix=slew' if $tdf;
3913
3914 if (!$kvm) {
3915 push @$machineFlags, 'accel=tcg';
3916 }
3917
3918 if ($machine_type) {
3919 push @$machineFlags, "type=${machine_type}";
3920 }
3921
3922 if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3923 push @$rtcFlags, "base=$conf->{startdate}";
3924 } elsif ($useLocaltime) {
3925 push @$rtcFlags, 'base=localtime';
3926 }
3927
3928 push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3929
3930 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3931
3932 push @$cmd, '-S' if $conf->{freeze};
3933
3934 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3935
3936 if (parse_guest_agent($conf)->{enabled}) {
3937 my $qgasocket = qmp_socket($vmid, 1);
3938 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3939 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3940 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3941 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3942 }
3943
3944 my $spice_port;
3945
3946 if ($qxlnum) {
3947 if ($qxlnum > 1) {
3948 if ($winversion){
3949 for(my $i = 1; $i < $qxlnum; $i++){
3950 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3951 }
3952 } else {
3953 # assume other OS works like Linux
3954 my ($ram, $vram) = ("134217728", "67108864");
3955 if ($vga->{memory}) {
3956 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3957 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3958 }
3959 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3960 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3961 }
3962 }
3963
3964 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3965
3966 my $nodename = PVE::INotify::nodename();
3967 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3968 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3969 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3970 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3971 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3972
3973 push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3974
3975 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3976 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3977 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3978 }
3979
3980 # enable balloon by default, unless explicitly disabled
3981 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3982 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3983 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3984 }
3985
3986 if ($conf->{watchdog}) {
3987 my $wdopts = parse_watchdog($conf->{watchdog});
3988 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3989 my $watchdog = $wdopts->{model} || 'i6300esb';
3990 push @$devices, '-device', "$watchdog$pciaddr";
3991 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3992 }
3993
3994 my $vollist = [];
3995 my $scsicontroller = {};
3996 my $ahcicontroller = {};
3997 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3998
3999 # Add iscsi initiator name if available
4000 if (my $initiator = get_initiator_name()) {
4001 push @$devices, '-iscsi', "initiator-name=$initiator";
4002 }
4003
4004 foreach_drive($conf, sub {
4005 my ($ds, $drive) = @_;
4006
4007 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4008 push @$vollist, $drive->{file};
4009 }
4010
4011 # ignore efidisk here, already added in bios/fw handling code above
4012 return if $drive->{interface} eq 'efidisk';
4013
4014 $use_virtio = 1 if $ds =~ m/^virtio/;
4015
4016 if (drive_is_cdrom ($drive)) {
4017 if ($bootindex_hash->{d}) {
4018 $drive->{bootindex} = $bootindex_hash->{d};
4019 $bootindex_hash->{d} += 1;
4020 }
4021 } else {
4022 if ($bootindex_hash->{c}) {
4023 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4024 $bootindex_hash->{c} += 1;
4025 }
4026 }
4027
4028 if($drive->{interface} eq 'virtio'){
4029 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4030 }
4031
4032 if ($drive->{interface} eq 'scsi') {
4033
4034 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4035
4036 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4037 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4038
4039 my $iothread = '';
4040 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4041 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4042 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4043 } elsif ($drive->{iothread}) {
4044 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4045 }
4046
4047 my $queues = '';
4048 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4049 $queues = ",num_queues=$drive->{queues}";
4050 }
4051
4052 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4053 $scsicontroller->{$controller}=1;
4054 }
4055
4056 if ($drive->{interface} eq 'sata') {
4057 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4058 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4059 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4060 $ahcicontroller->{$controller}=1;
4061 }
4062
4063 my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4064 push @$devices, '-drive',$drive_cmd;
4065 push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4066 });
4067
4068 for (my $i = 0; $i < $MAX_NETS; $i++) {
4069 next if !$conf->{"net$i"};
4070 my $d = parse_net($conf->{"net$i"});
4071 next if !$d;
4072
4073 $use_virtio = 1 if $d->{model} eq 'virtio';
4074
4075 if ($bootindex_hash->{n}) {
4076 $d->{bootindex} = $bootindex_hash->{n};
4077 $bootindex_hash->{n} += 1;
4078 }
4079
4080 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4081 push @$devices, '-netdev', $netdevfull;
4082
4083 my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4084 push @$devices, '-device', $netdevicefull;
4085 }
4086
4087 if ($conf->{ivshmem}) {
4088 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4089
4090 my $bus;
4091 if ($q35) {
4092 $bus = print_pcie_addr("ivshmem");
4093 } else {
4094 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4095 }
4096
4097 my $ivshmem_name = $ivshmem->{name} // $vmid;
4098 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4099
4100 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4101 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4102 }
4103
4104 if (!$q35) {
4105 # add pci bridges
4106 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4107 $bridges->{1} = 1;
4108 $bridges->{2} = 1;
4109 }
4110
4111 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4112
4113 while (my ($k, $v) = each %$bridges) {
4114 $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4115 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4116 }
4117 }
4118
4119 push @$cmd, @$devices;
4120 push @$cmd, '-rtc', join(',', @$rtcFlags)
4121 if scalar(@$rtcFlags);
4122 push @$cmd, '-machine', join(',', @$machineFlags)
4123 if scalar(@$machineFlags);
4124 push @$cmd, '-global', join(',', @$globalFlags)
4125 if scalar(@$globalFlags);
4126
4127 if (my $vmstate = $conf->{vmstate}) {
4128 my $statepath = PVE::Storage::path($storecfg, $vmstate);
4129 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4130 push @$cmd, '-loadstate', $statepath;
4131 }
4132
4133 # add custom args
4134 if ($conf->{args}) {
4135 my $aa = PVE::Tools::split_args($conf->{args});
4136 push @$cmd, @$aa;
4137 }
4138
4139 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4140}
4141
4142sub vnc_socket {
4143 my ($vmid) = @_;
4144 return "${var_run_tmpdir}/$vmid.vnc";
4145}
4146
4147sub spice_port {
4148 my ($vmid) = @_;
4149
4150 my $res = vm_mon_cmd($vmid, 'query-spice');
4151
4152 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4153}
4154
4155sub qmp_socket {
4156 my ($vmid, $qga, $name) = @_;
4157 my $sockettype = $qga ? 'qga' : 'qmp';
4158 my $ext = $name ? '-'.$name : '';
4159 return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4160}
4161
4162sub pidfile_name {
4163 my ($vmid) = @_;
4164 return "${var_run_tmpdir}/$vmid.pid";
4165}
4166
4167sub vm_devices_list {
4168 my ($vmid) = @_;
4169
4170 my $res = vm_mon_cmd($vmid, 'query-pci');
4171 my $devices_to_check = [];
4172 my $devices = {};
4173 foreach my $pcibus (@$res) {
4174 push @$devices_to_check, @{$pcibus->{devices}},
4175 }
4176
4177 while (@$devices_to_check) {
4178 my $to_check = [];
4179 for my $d (@$devices_to_check) {
4180 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4181 next if !$d->{'pci_bridge'};
4182
4183 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4184 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4185 }
4186 $devices_to_check = $to_check;
4187 }
4188
4189 my $resblock = vm_mon_cmd($vmid, 'query-block');
4190 foreach my $block (@$resblock) {
4191 if($block->{device} =~ m/^drive-(\S+)/){
4192 $devices->{$1} = 1;
4193 }
4194 }
4195
4196 my $resmice = vm_mon_cmd($vmid, 'query-mice');
4197 foreach my $mice (@$resmice) {
4198 if ($mice->{name} eq 'QEMU HID Tablet') {
4199 $devices->{tablet} = 1;
4200 last;
4201 }
4202 }
4203
4204 # for usb devices there is no query-usb
4205 # but we can iterate over the entries in
4206 # qom-list path=/machine/peripheral
4207 my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4208 foreach my $per (@$resperipheral) {
4209 if ($per->{name} =~ m/^usb\d+$/) {
4210 $devices->{$per->{name}} = 1;
4211 }
4212 }
4213
4214 return $devices;
4215}
4216
4217sub vm_deviceplug {
4218 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4219
4220 my $q35 = machine_type_is_q35($conf);
4221
4222 my $devices_list = vm_devices_list($vmid);
4223 return 1 if defined($devices_list->{$deviceid});
4224
4225 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4226
4227 if ($deviceid eq 'tablet') {
4228
4229 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4230
4231 } elsif ($deviceid eq 'keyboard') {
4232
4233 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4234
4235 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4236
4237 die "usb hotplug currently not reliable\n";
4238 # since we can't reliably hot unplug all added usb devices
4239 # and usb passthrough disables live migration
4240 # we disable usb hotplugging for now
4241 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4242
4243 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4244
4245 qemu_iothread_add($vmid, $deviceid, $device);
4246
4247 qemu_driveadd($storecfg, $vmid, $device);
4248 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4249
4250 qemu_deviceadd($vmid, $devicefull);
4251 eval { qemu_deviceaddverify($vmid, $deviceid); };
4252 if (my $err = $@) {
4253 eval { qemu_drivedel($vmid, $deviceid); };
4254 warn $@ if $@;
4255 die $err;
4256 }
4257
4258 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4259
4260
4261 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4262 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4263 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4264
4265 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4266
4267 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4268 qemu_iothread_add($vmid, $deviceid, $device);
4269 $devicefull .= ",iothread=iothread-$deviceid";
4270 }
4271
4272 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4273 $devicefull .= ",num_queues=$device->{queues}";
4274 }
4275
4276 qemu_deviceadd($vmid, $devicefull);
4277 qemu_deviceaddverify($vmid, $deviceid);
4278
4279 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4280
4281 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4282 qemu_driveadd($storecfg, $vmid, $device);
4283
4284 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4285 eval { qemu_deviceadd($vmid, $devicefull); };
4286 if (my $err = $@) {
4287 eval { qemu_drivedel($vmid, $deviceid); };
4288 warn $@ if $@;
4289 die $err;
4290 }
4291
4292 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4293
4294 return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4295
4296 my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4297 my $use_old_bios_files = undef;
4298 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4299
4300 my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4301 qemu_deviceadd($vmid, $netdevicefull);
4302 eval {
4303 qemu_deviceaddverify($vmid, $deviceid);
4304 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4305 };
4306 if (my $err = $@) {
4307 eval { qemu_netdevdel($vmid, $deviceid); };
4308 warn $@ if $@;
4309 die $err;
4310 }
4311
4312 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4313
4314 my $bridgeid = $2;
4315 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4316 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4317
4318 qemu_deviceadd($vmid, $devicefull);
4319 qemu_deviceaddverify($vmid, $deviceid);
4320
4321 } else {
4322 die "can't hotplug device '$deviceid'\n";
4323 }
4324
4325 return 1;
4326}
4327
4328# fixme: this should raise exceptions on error!
4329sub vm_deviceunplug {
4330 my ($vmid, $conf, $deviceid) = @_;
4331
4332 my $devices_list = vm_devices_list($vmid);
4333 return 1 if !defined($devices_list->{$deviceid});
4334
4335 die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4336
4337 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4338
4339 qemu_devicedel($vmid, $deviceid);
4340
4341 } elsif ($deviceid =~ m/^usb\d+$/) {
4342
4343 die "usb hotplug currently not reliable\n";
4344 # when unplugging usb devices this way,
4345 # there may be remaining usb controllers/hubs
4346 # so we disable it for now
4347 qemu_devicedel($vmid, $deviceid);
4348 qemu_devicedelverify($vmid, $deviceid);
4349
4350 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4351
4352 qemu_devicedel($vmid, $deviceid);
4353 qemu_devicedelverify($vmid, $deviceid);
4354 qemu_drivedel($vmid, $deviceid);
4355 qemu_iothread_del($conf, $vmid, $deviceid);
4356
4357 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4358
4359 qemu_devicedel($vmid, $deviceid);
4360 qemu_devicedelverify($vmid, $deviceid);
4361 qemu_iothread_del($conf, $vmid, $deviceid);
4362
4363 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4364
4365 qemu_devicedel($vmid, $deviceid);
4366 qemu_drivedel($vmid, $deviceid);
4367 qemu_deletescsihw($conf, $vmid, $deviceid);
4368
4369 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4370
4371 qemu_devicedel($vmid, $deviceid);
4372 qemu_devicedelverify($vmid, $deviceid);
4373 qemu_netdevdel($vmid, $deviceid);
4374
4375 } else {
4376 die "can't unplug device '$deviceid'\n";
4377 }
4378
4379 return 1;
4380}
4381
4382sub qemu_deviceadd {
4383 my ($vmid, $devicefull) = @_;
4384
4385 $devicefull = "driver=".$devicefull;
4386 my %options = split(/[=,]/, $devicefull);
4387
4388 vm_mon_cmd($vmid, "device_add" , %options);
4389}
4390
4391sub qemu_devicedel {
4392 my ($vmid, $deviceid) = @_;
4393
4394 my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4395}
4396
4397sub qemu_iothread_add {
4398 my($vmid, $deviceid, $device) = @_;
4399
4400 if ($device->{iothread}) {
4401 my $iothreads = vm_iothreads_list($vmid);
4402 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4403 }
4404}
4405
4406sub qemu_iothread_del {
4407 my($conf, $vmid, $deviceid) = @_;
4408
4409 my $confid = $deviceid;
4410 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4411 $confid = 'scsi' . $1;
4412 }
4413 my $device = parse_drive($confid, $conf->{$confid});
4414 if ($device->{iothread}) {
4415 my $iothreads = vm_iothreads_list($vmid);
4416 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4417 }
4418}
4419
4420sub qemu_objectadd {
4421 my($vmid, $objectid, $qomtype) = @_;
4422
4423 vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4424
4425 return 1;
4426}
4427
4428sub qemu_objectdel {
4429 my($vmid, $objectid) = @_;
4430
4431 vm_mon_cmd($vmid, "object-del", id => $objectid);
4432
4433 return 1;
4434}
4435
4436sub qemu_driveadd {
4437 my ($storecfg, $vmid, $device) = @_;
4438
4439 my $drive = print_drive_full($storecfg, $vmid, $device);
4440 $drive =~ s/\\/\\\\/g;
4441 my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4442
4443 # If the command succeeds qemu prints: "OK"
4444 return 1 if $ret =~ m/OK/s;
4445
4446 die "adding drive failed: $ret\n";
4447}
4448
4449sub qemu_drivedel {
4450 my($vmid, $deviceid) = @_;
4451
4452 my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4453 $ret =~ s/^\s+//;
4454
4455 return 1 if $ret eq "";
4456
4457 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4458 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4459
4460 die "deleting drive $deviceid failed : $ret\n";
4461}
4462
4463sub qemu_deviceaddverify {
4464 my ($vmid, $deviceid) = @_;
4465
4466 for (my $i = 0; $i <= 5; $i++) {
4467 my $devices_list = vm_devices_list($vmid);
4468 return 1 if defined($devices_list->{$deviceid});
4469 sleep 1;
4470 }
4471
4472 die "error on hotplug device '$deviceid'\n";
4473}
4474
4475
4476sub qemu_devicedelverify {
4477 my ($vmid, $deviceid) = @_;
4478
4479 # need to verify that the device is correctly removed as device_del
4480 # is async and empty return is not reliable
4481
4482 for (my $i = 0; $i <= 5; $i++) {
4483 my $devices_list = vm_devices_list($vmid);
4484 return 1 if !defined($devices_list->{$deviceid});
4485 sleep 1;
4486 }
4487
4488 die "error on hot-unplugging device '$deviceid'\n";
4489}
4490
4491sub qemu_findorcreatescsihw {
4492 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4493
4494 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4495
4496 my $scsihwid="$controller_prefix$controller";
4497 my $devices_list = vm_devices_list($vmid);
4498
4499 if(!defined($devices_list->{$scsihwid})) {
4500 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4501 }
4502
4503 return 1;
4504}
4505
4506sub qemu_deletescsihw {
4507 my ($conf, $vmid, $opt) = @_;
4508
4509 my $device = parse_drive($opt, $conf->{$opt});
4510
4511 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4512 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4513 return 1;
4514 }
4515
4516 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4517
4518 my $devices_list = vm_devices_list($vmid);
4519 foreach my $opt (keys %{$devices_list}) {
4520 if (PVE::QemuServer::is_valid_drivename($opt)) {
4521 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4522 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4523 return 1;
4524 }
4525 }
4526 }
4527
4528 my $scsihwid="scsihw$controller";
4529
4530 vm_deviceunplug($vmid, $conf, $scsihwid);
4531
4532 return 1;
4533}
4534
4535sub qemu_add_pci_bridge {
4536 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4537
4538 my $bridges = {};
4539
4540 my $bridgeid;
4541
4542 print_pci_addr($device, $bridges, $arch, $machine_type);
4543
4544 while (my ($k, $v) = each %$bridges) {
4545 $bridgeid = $k;
4546 }
4547 return 1 if !defined($bridgeid) || $bridgeid < 1;
4548
4549 my $bridge = "pci.$bridgeid";
4550 my $devices_list = vm_devices_list($vmid);
4551
4552 if (!defined($devices_list->{$bridge})) {
4553 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4554 }
4555
4556 return 1;
4557}
4558
4559sub qemu_set_link_status {
4560 my ($vmid, $device, $up) = @_;
4561
4562 vm_mon_cmd($vmid, "set_link", name => $device,
4563 up => $up ? JSON::true : JSON::false);
4564}
4565
4566sub qemu_netdevadd {
4567 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4568
4569 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4570 my %options = split(/[=,]/, $netdev);
4571
4572 vm_mon_cmd($vmid, "netdev_add", %options);
4573 return 1;
4574}
4575
4576sub qemu_netdevdel {
4577 my ($vmid, $deviceid) = @_;
4578
4579 vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4580}
4581
4582sub qemu_usb_hotplug {
4583 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4584
4585 return if !$device;
4586
4587 # remove the old one first
4588 vm_deviceunplug($vmid, $conf, $deviceid);
4589
4590 # check if xhci controller is necessary and available
4591 if ($device->{usb3}) {
4592
4593 my $devicelist = vm_devices_list($vmid);
4594
4595 if (!$devicelist->{xhci}) {
4596 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4597 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4598 }
4599 }
4600 my $d = parse_usb_device($device->{host});
4601 $d->{usb3} = $device->{usb3};
4602
4603 # add the new one
4604 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4605}
4606
4607sub qemu_cpu_hotplug {
4608 my ($vmid, $conf, $vcpus) = @_;
4609
4610 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4611
4612 my $sockets = 1;
4613 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4614 $sockets = $conf->{sockets} if $conf->{sockets};
4615 my $cores = $conf->{cores} || 1;
4616 my $maxcpus = $sockets * $cores;
4617
4618 $vcpus = $maxcpus if !$vcpus;
4619
4620 die "you can't add more vcpus than maxcpus\n"
4621 if $vcpus > $maxcpus;
4622
4623 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4624
4625 if ($vcpus < $currentvcpus) {
4626
4627 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4628
4629 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4630 qemu_devicedel($vmid, "cpu$i");
4631 my $retry = 0;
4632 my $currentrunningvcpus = undef;
4633 while (1) {
4634 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4635 last if scalar(@{$currentrunningvcpus}) == $i-1;
4636 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4637 $retry++;
4638 sleep 1;
4639 }
4640 #update conf after each succesfull cpu unplug
4641 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4642 PVE::QemuConfig->write_config($vmid, $conf);
4643 }
4644 } else {
4645 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4646 }
4647
4648 return;
4649 }
4650
4651 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4652 die "vcpus in running vm does not match its configuration\n"
4653 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4654
4655 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4656
4657 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4658 my $cpustr = print_cpu_device($conf, $i);
4659 qemu_deviceadd($vmid, $cpustr);
4660
4661 my $retry = 0;
4662 my $currentrunningvcpus = undef;
4663 while (1) {
4664 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4665 last if scalar(@{$currentrunningvcpus}) == $i;
4666 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4667 sleep 1;
4668 $retry++;
4669 }
4670 #update conf after each succesfull cpu hotplug
4671 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4672 PVE::QemuConfig->write_config($vmid, $conf);
4673 }
4674 } else {
4675
4676 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4677 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4678 }
4679 }
4680}
4681
4682sub qemu_block_set_io_throttle {
4683 my ($vmid, $deviceid,
4684 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4685 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4686 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4687 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4688
4689 return if !check_running($vmid) ;
4690
4691 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4692 bps => int($bps),
4693 bps_rd => int($bps_rd),
4694 bps_wr => int($bps_wr),
4695 iops => int($iops),
4696 iops_rd => int($iops_rd),
4697 iops_wr => int($iops_wr),
4698 bps_max => int($bps_max),
4699 bps_rd_max => int($bps_rd_max),
4700 bps_wr_max => int($bps_wr_max),
4701 iops_max => int($iops_max),
4702 iops_rd_max => int($iops_rd_max),
4703 iops_wr_max => int($iops_wr_max),
4704 bps_max_length => int($bps_max_length),
4705 bps_rd_max_length => int($bps_rd_max_length),
4706 bps_wr_max_length => int($bps_wr_max_length),
4707 iops_max_length => int($iops_max_length),
4708 iops_rd_max_length => int($iops_rd_max_length),
4709 iops_wr_max_length => int($iops_wr_max_length),
4710 );
4711
4712}
4713
4714# old code, only used to shutdown old VM after update
4715sub __read_avail {
4716 my ($fh, $timeout) = @_;
4717
4718 my $sel = new IO::Select;
4719 $sel->add($fh);
4720
4721 my $res = '';
4722 my $buf;
4723
4724 my @ready;
4725 while (scalar (@ready = $sel->can_read($timeout))) {
4726 my $count;
4727 if ($count = $fh->sysread($buf, 8192)) {
4728 if ($buf =~ /^(.*)\(qemu\) $/s) {
4729 $res .= $1;
4730 last;
4731 } else {
4732 $res .= $buf;
4733 }
4734 } else {
4735 if (!defined($count)) {
4736 die "$!\n";
4737 }
4738 last;
4739 }
4740 }
4741
4742 die "monitor read timeout\n" if !scalar(@ready);
4743
4744 return $res;
4745}
4746
4747sub qemu_block_resize {
4748 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4749
4750 my $running = check_running($vmid);
4751
4752 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4753
4754 return if !$running;
4755
4756 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4757
4758}
4759
4760sub qemu_volume_snapshot {
4761 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4762
4763 my $running = check_running($vmid);
4764
4765 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4766 vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4767 } else {
4768 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4769 }
4770}
4771
4772sub qemu_volume_snapshot_delete {
4773 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4774
4775 my $running = check_running($vmid);
4776
4777 if($running) {
4778
4779 $running = undef;
4780 my $conf = PVE::QemuConfig->load_config($vmid);
4781 foreach_drive($conf, sub {
4782 my ($ds, $drive) = @_;
4783 $running = 1 if $drive->{file} eq $volid;
4784 });
4785 }
4786
4787 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4788 vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4789 } else {
4790 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4791 }
4792}
4793
4794sub set_migration_caps {
4795 my ($vmid) = @_;
4796
4797 my $cap_ref = [];
4798
4799 my $enabled_cap = {
4800 "auto-converge" => 1,
4801 "xbzrle" => 1,
4802 "x-rdma-pin-all" => 0,
4803 "zero-blocks" => 0,
4804 "compress" => 0
4805 };
4806
4807 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4808
4809 for my $supported_capability (@$supported_capabilities) {
4810 push @$cap_ref, {
4811 capability => $supported_capability->{capability},
4812 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4813 };
4814 }
4815
4816 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4817}
4818
4819my $fast_plug_option = {
4820 'lock' => 1,
4821 'name' => 1,
4822 'onboot' => 1,
4823 'shares' => 1,
4824 'startup' => 1,
4825 'description' => 1,
4826 'protection' => 1,
4827 'vmstatestorage' => 1,
4828 'hookscript' => 1,
4829};
4830
4831# hotplug changes in [PENDING]
4832# $selection hash can be used to only apply specified options, for
4833# example: { cores => 1 } (only apply changed 'cores')
4834# $errors ref is used to return error messages
4835sub vmconfig_hotplug_pending {
4836 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4837
4838 my $defaults = load_defaults();
4839 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4840
4841 # commit values which do not have any impact on running VM first
4842 # Note: those option cannot raise errors, we we do not care about
4843 # $selection and always apply them.
4844
4845 my $add_error = sub {
4846 my ($opt, $msg) = @_;
4847 $errors->{$opt} = "hotplug problem - $msg";
4848 };
4849
4850 my $changes = 0;
4851 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4852 if ($fast_plug_option->{$opt}) {
4853 $conf->{$opt} = $conf->{pending}->{$opt};
4854 delete $conf->{pending}->{$opt};
4855 $changes = 1;
4856 }
4857 }
4858
4859 if ($changes) {
4860 PVE::QemuConfig->write_config($vmid, $conf);
4861 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4862 }
4863
4864 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4865
4866 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4867 while (my ($opt, $force) = each %$pending_delete_hash) {
4868 next if $selection && !$selection->{$opt};
4869 eval {
4870 if ($opt eq 'hotplug') {
4871 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4872 } elsif ($opt eq 'tablet') {
4873 die "skip\n" if !$hotplug_features->{usb};
4874 if ($defaults->{tablet}) {
4875 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4876 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4877 if $arch eq 'aarch64';
4878 } else {
4879 vm_deviceunplug($vmid, $conf, 'tablet');
4880 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4881 }
4882 } elsif ($opt =~ m/^usb\d+/) {
4883 die "skip\n";
4884 # since we cannot reliably hot unplug usb devices
4885 # we are disabling it
4886 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4887 vm_deviceunplug($vmid, $conf, $opt);
4888 } elsif ($opt eq 'vcpus') {
4889 die "skip\n" if !$hotplug_features->{cpu};
4890 qemu_cpu_hotplug($vmid, $conf, undef);
4891 } elsif ($opt eq 'balloon') {
4892 # enable balloon device is not hotpluggable
4893 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4894 # here we reset the ballooning value to memory
4895 my $balloon = $conf->{memory} || $defaults->{memory};
4896 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4897 } elsif ($fast_plug_option->{$opt}) {
4898 # do nothing
4899 } elsif ($opt =~ m/^net(\d+)$/) {
4900 die "skip\n" if !$hotplug_features->{network};
4901 vm_deviceunplug($vmid, $conf, $opt);
4902 } elsif (is_valid_drivename($opt)) {
4903 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4904 vm_deviceunplug($vmid, $conf, $opt);
4905 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4906 } elsif ($opt =~ m/^memory$/) {
4907 die "skip\n" if !$hotplug_features->{memory};
4908 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4909 } elsif ($opt eq 'cpuunits') {
4910 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4911 } elsif ($opt eq 'cpulimit') {
4912 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4913 } else {
4914 die "skip\n";
4915 }
4916 };
4917 if (my $err = $@) {
4918 &$add_error($opt, $err) if $err ne "skip\n";
4919 } else {
4920 # save new config if hotplug was successful
4921 delete $conf->{$opt};
4922 vmconfig_undelete_pending_option($conf, $opt);
4923 PVE::QemuConfig->write_config($vmid, $conf);
4924 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4925 }
4926 }
4927
4928 my $apply_pending_cloudinit;
4929 $apply_pending_cloudinit = sub {
4930 my ($key, $value) = @_;
4931 $apply_pending_cloudinit = sub {}; # once is enough
4932
4933 my @cloudinit_opts = keys %$confdesc_cloudinit;
4934 foreach my $opt (keys %{$conf->{pending}}) {
4935 next if !grep { $_ eq $opt } @cloudinit_opts;
4936 $conf->{$opt} = delete $conf->{pending}->{$opt};
4937 }
4938
4939 my $new_conf = { %$conf };
4940 $new_conf->{$key} = $value;
4941 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4942 };
4943
4944 foreach my $opt (keys %{$conf->{pending}}) {
4945 next if $selection && !$selection->{$opt};
4946 my $value = $conf->{pending}->{$opt};
4947 eval {
4948 if ($opt eq 'hotplug') {
4949 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4950 } elsif ($opt eq 'tablet') {
4951 die "skip\n" if !$hotplug_features->{usb};
4952 if ($value == 1) {
4953 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4954 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4955 if $arch eq 'aarch64';
4956 } elsif ($value == 0) {
4957 vm_deviceunplug($vmid, $conf, 'tablet');
4958 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4959 }
4960 } elsif ($opt =~ m/^usb\d+$/) {
4961 die "skip\n";
4962 # since we cannot reliably hot unplug usb devices
4963 # we are disabling it
4964 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4965 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4966 die "skip\n" if !$d;
4967 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4968 } elsif ($opt eq 'vcpus') {
4969 die "skip\n" if !$hotplug_features->{cpu};
4970 qemu_cpu_hotplug($vmid, $conf, $value);
4971 } elsif ($opt eq 'balloon') {
4972 # enable/disable balloning device is not hotpluggable
4973 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4974 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4975 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4976
4977 # allow manual ballooning if shares is set to zero
4978 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4979 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4980 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4981 }
4982 } elsif ($opt =~ m/^net(\d+)$/) {
4983 # some changes can be done without hotplug
4984 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4985 $vmid, $opt, $value, $arch, $machine_type);
4986 } elsif (is_valid_drivename($opt)) {
4987 # some changes can be done without hotplug
4988 my $drive = parse_drive($opt, $value);
4989 if (drive_is_cloudinit($drive)) {
4990 &$apply_pending_cloudinit($opt, $value);
4991 }
4992 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4993 $vmid, $opt, $value, 1, $arch, $machine_type);
4994 } elsif ($opt =~ m/^memory$/) { #dimms
4995 die "skip\n" if !$hotplug_features->{memory};
4996 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4997 } elsif ($opt eq 'cpuunits') {
4998 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4999 } elsif ($opt eq 'cpulimit') {
5000 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5001 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5002 } else {
5003 die "skip\n"; # skip non-hot-pluggable options
5004 }
5005 };
5006 if (my $err = $@) {
5007 &$add_error($opt, $err) if $err ne "skip\n";
5008 } else {
5009 # save new config if hotplug was successful
5010 $conf->{$opt} = $value;
5011 delete $conf->{pending}->{$opt};
5012 PVE::QemuConfig->write_config($vmid, $conf);
5013 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5014 }
5015 }
5016}
5017
5018sub try_deallocate_drive {
5019 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5020
5021 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5022 my $volid = $drive->{file};
5023 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5024 my $sid = PVE::Storage::parse_volume_id($volid);
5025 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5026
5027 # check if the disk is really unused
5028 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5029 if is_volume_in_use($storecfg, $conf, $key, $volid);
5030 PVE::Storage::vdisk_free($storecfg, $volid);
5031 return 1;
5032 } else {
5033 # If vm is not owner of this disk remove from config
5034 return 1;
5035 }
5036 }
5037
5038 return undef;
5039}
5040
5041sub vmconfig_delete_or_detach_drive {
5042 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5043
5044 my $drive = parse_drive($opt, $conf->{$opt});
5045
5046 my $rpcenv = PVE::RPCEnvironment::get();
5047 my $authuser = $rpcenv->get_user();
5048
5049 if ($force) {
5050 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5051 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5052 } else {
5053 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5054 }
5055}
5056
5057sub vmconfig_apply_pending {
5058 my ($vmid, $conf, $storecfg) = @_;
5059
5060 # cold plug
5061
5062 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5063 while (my ($opt, $force) = each %$pending_delete_hash) {
5064 die "internal error" if $opt =~ m/^unused/;
5065 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5066 if (!defined($conf->{$opt})) {
5067 vmconfig_undelete_pending_option($conf, $opt);
5068 PVE::QemuConfig->write_config($vmid, $conf);
5069 } elsif (is_valid_drivename($opt)) {
5070 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5071 vmconfig_undelete_pending_option($conf, $opt);
5072 delete $conf->{$opt};
5073 PVE::QemuConfig->write_config($vmid, $conf);
5074 } else {
5075 vmconfig_undelete_pending_option($conf, $opt);
5076 delete $conf->{$opt};
5077 PVE::QemuConfig->write_config($vmid, $conf);
5078 }
5079 }
5080
5081 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5082
5083 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5084 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5085
5086 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5087 # skip if nothing changed
5088 } elsif (is_valid_drivename($opt)) {
5089 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5090 if defined($conf->{$opt});
5091 $conf->{$opt} = $conf->{pending}->{$opt};
5092 } else {
5093 $conf->{$opt} = $conf->{pending}->{$opt};
5094 }
5095
5096 delete $conf->{pending}->{$opt};
5097 PVE::QemuConfig->write_config($vmid, $conf);
5098 }
5099}
5100
5101my $safe_num_ne = sub {
5102 my ($a, $b) = @_;
5103
5104 return 0 if !defined($a) && !defined($b);
5105 return 1 if !defined($a);
5106 return 1 if !defined($b);
5107
5108 return $a != $b;
5109};
5110
5111my $safe_string_ne = sub {
5112 my ($a, $b) = @_;
5113
5114 return 0 if !defined($a) && !defined($b);
5115 return 1 if !defined($a);
5116 return 1 if !defined($b);
5117
5118 return $a ne $b;
5119};
5120
5121sub vmconfig_update_net {
5122 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5123
5124 my $newnet = parse_net($value);
5125
5126 if ($conf->{$opt}) {
5127 my $oldnet = parse_net($conf->{$opt});
5128
5129 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5130 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5131 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5132 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5133
5134 # for non online change, we try to hot-unplug
5135 die "skip\n" if !$hotplug;
5136 vm_deviceunplug($vmid, $conf, $opt);
5137 } else {
5138
5139 die "internal error" if $opt !~ m/net(\d+)/;
5140 my $iface = "tap${vmid}i$1";
5141
5142 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5143 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5144 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5145 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5146 PVE::Network::tap_unplug($iface);
5147 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5148 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5149 # Rate can be applied on its own but any change above needs to
5150 # include the rate in tap_plug since OVS resets everything.
5151 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5152 }
5153
5154 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5155 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5156 }
5157
5158 return 1;
5159 }
5160 }
5161
5162 if ($hotplug) {
5163 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5164 } else {
5165 die "skip\n";
5166 }
5167}
5168
5169sub vmconfig_update_disk {
5170 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5171
5172 # fixme: do we need force?
5173
5174 my $drive = parse_drive($opt, $value);
5175
5176 if ($conf->{$opt}) {
5177
5178 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5179
5180 my $media = $drive->{media} || 'disk';
5181 my $oldmedia = $old_drive->{media} || 'disk';
5182 die "unable to change media type\n" if $media ne $oldmedia;
5183
5184 if (!drive_is_cdrom($old_drive)) {
5185
5186 if ($drive->{file} ne $old_drive->{file}) {
5187
5188 die "skip\n" if !$hotplug;
5189
5190 # unplug and register as unused
5191 vm_deviceunplug($vmid, $conf, $opt);
5192 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5193
5194 } else {
5195 # update existing disk
5196
5197 # skip non hotpluggable value
5198 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5199 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5200 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5201 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5202 die "skip\n";
5203 }
5204
5205 # apply throttle
5206 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5207 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5208 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5209 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5210 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5211 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5212 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5213 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5214 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5215 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5216 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5217 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5218 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5219 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5220 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5221 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5222 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5223 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5224
5225 qemu_block_set_io_throttle($vmid,"drive-$opt",
5226 ($drive->{mbps} || 0)*1024*1024,
5227 ($drive->{mbps_rd} || 0)*1024*1024,
5228 ($drive->{mbps_wr} || 0)*1024*1024,
5229 $drive->{iops} || 0,
5230 $drive->{iops_rd} || 0,
5231 $drive->{iops_wr} || 0,
5232 ($drive->{mbps_max} || 0)*1024*1024,
5233 ($drive->{mbps_rd_max} || 0)*1024*1024,
5234 ($drive->{mbps_wr_max} || 0)*1024*1024,
5235 $drive->{iops_max} || 0,
5236 $drive->{iops_rd_max} || 0,
5237 $drive->{iops_wr_max} || 0,
5238 $drive->{bps_max_length} || 1,
5239 $drive->{bps_rd_max_length} || 1,
5240 $drive->{bps_wr_max_length} || 1,
5241 $drive->{iops_max_length} || 1,
5242 $drive->{iops_rd_max_length} || 1,
5243 $drive->{iops_wr_max_length} || 1);
5244
5245 }
5246
5247 return 1;
5248 }
5249
5250 } else { # cdrom
5251
5252 if ($drive->{file} eq 'none') {
5253 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5254 if (drive_is_cloudinit($old_drive)) {
5255 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5256 }
5257 } else {
5258 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5259 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5260 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5261 }
5262
5263 return 1;
5264 }
5265 }
5266 }
5267
5268 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5269 # hotplug new disks
5270 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5271 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5272}
5273
5274sub vm_start {
5275 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5276 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5277
5278 PVE::QemuConfig->lock_config($vmid, sub {
5279 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5280
5281 die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5282
5283 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5284
5285 PVE::QemuConfig->check_lock($conf)
5286 if !($skiplock || $is_suspended);
5287
5288 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5289
5290 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5291 vmconfig_apply_pending($vmid, $conf, $storecfg);
5292 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5293 }
5294
5295 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5296
5297 my $defaults = load_defaults();
5298
5299 # set environment variable useful inside network script
5300 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5301
5302 my $local_volumes = {};
5303
5304 if ($targetstorage) {
5305 foreach_drive($conf, sub {
5306 my ($ds, $drive) = @_;
5307
5308 return if drive_is_cdrom($drive);
5309
5310 my $volid = $drive->{file};
5311
5312 return if !$volid;
5313
5314 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5315
5316 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5317 return if $scfg->{shared};
5318 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5319 });
5320
5321 my $format = undef;
5322
5323 foreach my $opt (sort keys %$local_volumes) {
5324
5325 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5326 my $drive = parse_drive($opt, $conf->{$opt});
5327
5328 #if remote storage is specified, use default format
5329 if ($targetstorage && $targetstorage ne "1") {
5330 $storeid = $targetstorage;
5331 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5332 $format = $defFormat;
5333 } else {
5334 #else we use same format than original
5335 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5336 $format = qemu_img_format($scfg, $volid);
5337 }
5338
5339 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5340 my $newdrive = $drive;
5341 $newdrive->{format} = $format;
5342 $newdrive->{file} = $newvolid;
5343 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5344 $local_volumes->{$opt} = $drivestr;
5345 #pass drive to conf for command line
5346 $conf->{$opt} = $drivestr;
5347 }
5348 }
5349
5350 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5351
5352 if ($is_suspended) {
5353 # enforce machine type on suspended vm to ensure HW compatibility
5354 $forcemachine = $conf->{runningmachine};
5355 print "Resuming suspended VM\n";
5356 }
5357
5358 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5359
5360 my $migrate_port = 0;
5361 my $migrate_uri;
5362 if ($statefile) {
5363 if ($statefile eq 'tcp') {
5364 my $localip = "localhost";
5365 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5366 my $nodename = PVE::INotify::nodename();
5367
5368 if (!defined($migration_type)) {
5369 if (defined($datacenterconf->{migration}->{type})) {
5370 $migration_type = $datacenterconf->{migration}->{type};
5371 } else {
5372 $migration_type = 'secure';
5373 }
5374 }
5375
5376 if ($migration_type eq 'insecure') {
5377 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5378 if ($migrate_network_addr) {
5379 $localip = $migrate_network_addr;
5380 } else {
5381 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5382 }
5383
5384 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5385 }
5386
5387 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5388 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5389 $migrate_uri = "tcp:${localip}:${migrate_port}";
5390 push @$cmd, '-incoming', $migrate_uri;
5391 push @$cmd, '-S';
5392
5393 } elsif ($statefile eq 'unix') {
5394 # should be default for secure migrations as a ssh TCP forward
5395 # tunnel is not deterministic reliable ready and fails regurarly
5396 # to set up in time, so use UNIX socket forwards
5397 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5398 unlink $socket_addr;
5399
5400 $migrate_uri = "unix:$socket_addr";
5401
5402 push @$cmd, '-incoming', $migrate_uri;
5403 push @$cmd, '-S';
5404
5405 } else {
5406 push @$cmd, '-loadstate', $statefile;
5407 }
5408 } elsif ($paused) {
5409 push @$cmd, '-S';
5410 }
5411
5412 # host pci devices
5413 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5414 my $d = parse_hostpci($conf->{"hostpci$i"});
5415 next if !$d;
5416 my $pcidevices = $d->{pciid};
5417 foreach my $pcidevice (@$pcidevices) {
5418 my $pciid = $pcidevice->{id};
5419
5420 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5421 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5422 die "no pci device info for device '$pciid'\n" if !$info;
5423
5424 if ($d->{mdev}) {
5425 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5426 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5427 } else {
5428 die "can't unbind/bind pci group to vfio '$pciid'\n"
5429 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5430 die "can't reset pci device '$pciid'\n"
5431 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5432 }
5433 }
5434 }
5435
5436 PVE::Storage::activate_volumes($storecfg, $vollist);
5437
5438 eval {
5439 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5440 outfunc => sub {}, errfunc => sub {});
5441 };
5442 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5443 # timeout should be more than enough here...
5444 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5445
5446 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5447 : $defaults->{cpuunits};
5448
5449 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5450 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5451
5452 my %properties = (
5453 Slice => 'qemu.slice',
5454 KillMode => 'none',
5455 CPUShares => $cpuunits
5456 );
5457
5458 if (my $cpulimit = $conf->{cpulimit}) {
5459 $properties{CPUQuota} = int($cpulimit * 100);
5460 }
5461 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5462
5463 my $run_qemu = sub {
5464 PVE::Tools::run_fork sub {
5465 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5466 run_command($cmd, %run_params);
5467 };
5468 };
5469
5470 if ($conf->{hugepages}) {
5471
5472 my $code = sub {
5473 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5474 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5475
5476 PVE::QemuServer::Memory::hugepages_mount();
5477 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5478
5479 eval { $run_qemu->() };
5480 if (my $err = $@) {
5481 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5482 die $err;
5483 }
5484
5485 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5486 };
5487 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5488
5489 } else {
5490 eval { $run_qemu->() };
5491 }
5492
5493 if (my $err = $@) {
5494 # deactivate volumes if start fails
5495 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5496 die "start failed: $err";
5497 }
5498
5499 print "migration listens on $migrate_uri\n" if $migrate_uri;
5500
5501 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5502 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5503 warn $@ if $@;
5504 }
5505
5506 #start nbd server for storage migration
5507 if ($targetstorage) {
5508 my $nodename = PVE::INotify::nodename();
5509 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5510 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5511 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5512 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5513
5514 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5515
5516 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5517
5518 foreach my $opt (sort keys %$local_volumes) {
5519 my $volid = $local_volumes->{$opt};
5520 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5521 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5522 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5523 }
5524 }
5525
5526 if ($migratedfrom) {
5527 eval {
5528 set_migration_caps($vmid);
5529 };
5530 warn $@ if $@;
5531
5532 if ($spice_port) {
5533 print "spice listens on port $spice_port\n";
5534 if ($spice_ticket) {
5535 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5536 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5537 }
5538 }
5539
5540 } else {
5541 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5542 if !$statefile && $conf->{balloon};
5543
5544 foreach my $opt (keys %$conf) {
5545 next if $opt !~ m/^net\d+$/;
5546 my $nicconf = parse_net($conf->{$opt});
5547 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5548 }
5549 }
5550
5551 vm_mon_cmd_nocheck($vmid, 'qom-set',
5552 path => "machine/peripheral/balloon0",
5553 property => "guest-stats-polling-interval",
5554 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5555
5556 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5557 print "Resumed VM, removing state\n";
5558 delete $conf->@{qw(lock vmstate runningmachine)};
5559 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5560 PVE::Storage::vdisk_free($storecfg, $vmstate);
5561 PVE::QemuConfig->write_config($vmid, $conf);
5562 }
5563
5564 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5565 });
5566}
5567
5568sub vm_mon_cmd {
5569 my ($vmid, $execute, %params) = @_;
5570
5571 my $cmd = { execute => $execute, arguments => \%params };
5572 vm_qmp_command($vmid, $cmd);
5573}
5574
5575sub vm_mon_cmd_nocheck {
5576 my ($vmid, $execute, %params) = @_;
5577
5578 my $cmd = { execute => $execute, arguments => \%params };
5579 vm_qmp_command($vmid, $cmd, 1);
5580}
5581
5582sub vm_qmp_command {
5583 my ($vmid, $cmd, $nocheck) = @_;
5584
5585 my $res;
5586
5587 my $timeout;
5588 if ($cmd->{arguments}) {
5589 $timeout = delete $cmd->{arguments}->{timeout};
5590 }
5591
5592 eval {
5593 die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5594 my $sname = qmp_socket($vmid);
5595 if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5596 my $qmpclient = PVE::QMPClient->new();
5597
5598 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5599 } else {
5600 die "unable to open monitor socket\n";
5601 }
5602 };
5603 if (my $err = $@) {
5604 syslog("err", "VM $vmid qmp command failed - $err");
5605 die $err;
5606 }
5607
5608 return $res;
5609}
5610
5611sub vm_human_monitor_command {
5612 my ($vmid, $cmdline) = @_;
5613
5614 my $cmd = {
5615 execute => 'human-monitor-command',
5616 arguments => { 'command-line' => $cmdline},
5617 };
5618
5619 return vm_qmp_command($vmid, $cmd);
5620}
5621
5622sub vm_commandline {
5623 my ($storecfg, $vmid, $snapname) = @_;
5624
5625 my $conf = PVE::QemuConfig->load_config($vmid);
5626
5627 if ($snapname) {
5628 my $snapshot = $conf->{snapshots}->{$snapname};
5629 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5630
5631 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5632
5633 $conf = $snapshot;
5634 }
5635
5636 my $defaults = load_defaults();
5637
5638 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5639
5640 return PVE::Tools::cmd2string($cmd);
5641}
5642
5643sub vm_reset {
5644 my ($vmid, $skiplock) = @_;
5645
5646 PVE::QemuConfig->lock_config($vmid, sub {
5647
5648 my $conf = PVE::QemuConfig->load_config($vmid);
5649
5650 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5651
5652 vm_mon_cmd($vmid, "system_reset");
5653 });
5654}
5655
5656sub get_vm_volumes {
5657 my ($conf) = @_;
5658
5659 my $vollist = [];
5660 foreach_volid($conf, sub {
5661 my ($volid, $attr) = @_;
5662
5663 return if $volid =~ m|^/|;
5664
5665 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5666 return if !$sid;
5667
5668 push @$vollist, $volid;
5669 });
5670
5671 return $vollist;
5672}
5673
5674sub vm_stop_cleanup {
5675 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5676
5677 eval {
5678
5679 if (!$keepActive) {
5680 my $vollist = get_vm_volumes($conf);
5681 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5682 }
5683
5684 foreach my $ext (qw(mon qmp pid vnc qga)) {
5685 unlink "/var/run/qemu-server/${vmid}.$ext";
5686 }
5687
5688 if ($conf->{ivshmem}) {
5689 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5690 # just delete it for now, VMs which have this already open do not
5691 # are affected, but new VMs will get a separated one. If this
5692 # becomes an issue we either add some sort of ref-counting or just
5693 # add a "don't delete on stop" flag to the ivshmem format.
5694 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5695 }
5696
5697 foreach my $key (keys %$conf) {
5698 next if $key !~ m/^hostpci(\d+)$/;
5699 my $hostpciindex = $1;
5700 my $d = parse_hostpci($conf->{$key});
5701 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5702
5703 foreach my $pci (@{$d->{pciid}}) {
5704 my $pciid = $pci->{id};
5705 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5706 }
5707 }
5708
5709 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5710 };
5711 warn $@ if $@; # avoid errors - just warn
5712}
5713
5714# Note: use $nockeck to skip tests if VM configuration file exists.
5715# We need that when migration VMs to other nodes (files already moved)
5716# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5717sub vm_stop {
5718 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5719
5720 $force = 1 if !defined($force) && !$shutdown;
5721
5722 if ($migratedfrom){
5723 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5724 kill 15, $pid if $pid;
5725 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5726 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5727 return;
5728 }
5729
5730 PVE::QemuConfig->lock_config($vmid, sub {
5731
5732 my $pid = check_running($vmid, $nocheck);
5733 return if !$pid;
5734
5735 my $conf;
5736 if (!$nocheck) {
5737 $conf = PVE::QemuConfig->load_config($vmid);
5738 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5739 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5740 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5741 $timeout = $opts->{down} if $opts->{down};
5742 }
5743 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5744 }
5745
5746 eval {
5747 if ($shutdown) {
5748 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5749 vm_qmp_command($vmid, {
5750 execute => "guest-shutdown",
5751 arguments => { timeout => $timeout }
5752 }, $nocheck);
5753 } else {
5754 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5755 }
5756 } else {
5757 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5758 }
5759 };
5760 my $err = $@;
5761
5762 if (!$err) {
5763 $timeout = 60 if !defined($timeout);
5764
5765 my $count = 0;
5766 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5767 $count++;
5768 sleep 1;
5769 }
5770
5771 if ($count >= $timeout) {
5772 if ($force) {
5773 warn "VM still running - terminating now with SIGTERM\n";
5774 kill 15, $pid;
5775 } else {
5776 die "VM quit/powerdown failed - got timeout\n";
5777 }
5778 } else {
5779 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5780 return;
5781 }
5782 } else {
5783 if ($force) {
5784 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5785 kill 15, $pid;
5786 } else {
5787 die "VM quit/powerdown failed\n";
5788 }
5789 }
5790
5791 # wait again
5792 $timeout = 10;
5793
5794 my $count = 0;
5795 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5796 $count++;
5797 sleep 1;
5798 }
5799
5800 if ($count >= $timeout) {
5801 warn "VM still running - terminating now with SIGKILL\n";
5802 kill 9, $pid;
5803 sleep 1;
5804 }
5805
5806 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5807 });
5808}
5809
5810sub vm_suspend {
5811 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5812
5813 my $conf;
5814 my $path;
5815 my $storecfg;
5816 my $vmstate;
5817
5818 PVE::QemuConfig->lock_config($vmid, sub {
5819
5820 $conf = PVE::QemuConfig->load_config($vmid);
5821
5822 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5823 PVE::QemuConfig->check_lock($conf)
5824 if !($skiplock || $is_backing_up);
5825
5826 die "cannot suspend to disk during backup\n"
5827 if $is_backing_up && $includestate;
5828
5829 if ($includestate) {
5830 $conf->{lock} = 'suspending';
5831 my $date = strftime("%Y-%m-%d", localtime(time()));
5832 $storecfg = PVE::Storage::config();
5833 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5834 $path = PVE::Storage::path($storecfg, $vmstate);
5835 PVE::QemuConfig->write_config($vmid, $conf);
5836 } else {
5837 vm_mon_cmd($vmid, "stop");
5838 }
5839 });
5840
5841 if ($includestate) {
5842 # save vm state
5843 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5844
5845 eval {
5846 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5847 for(;;) {
5848 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5849 if (!$state->{status}) {
5850 die "savevm not active\n";
5851 } elsif ($state->{status} eq 'active') {
5852 sleep(1);
5853 next;
5854 } elsif ($state->{status} eq 'completed') {
5855 print "State saved, quitting\n";
5856 last;
5857 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5858 die "query-savevm failed with error '$state->{error}'\n"
5859 } else {
5860 die "query-savevm returned status '$state->{status}'\n";
5861 }
5862 }
5863 };
5864 my $err = $@;
5865
5866 PVE::QemuConfig->lock_config($vmid, sub {
5867 $conf = PVE::QemuConfig->load_config($vmid);
5868 if ($err) {
5869 # cleanup, but leave suspending lock, to indicate something went wrong
5870 eval {
5871 vm_mon_cmd($vmid, "savevm-end");
5872 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5873 PVE::Storage::vdisk_free($storecfg, $vmstate);
5874 delete $conf->@{qw(vmstate runningmachine)};
5875 PVE::QemuConfig->write_config($vmid, $conf);
5876 };
5877 warn $@ if $@;
5878 die $err;
5879 }
5880
5881 die "lock changed unexpectedly\n"
5882 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5883
5884 vm_qmp_command($vmid, { execute => "quit" });
5885 $conf->{lock} = 'suspended';
5886 PVE::QemuConfig->write_config($vmid, $conf);
5887 });
5888 }
5889}
5890
5891sub vm_resume {
5892 my ($vmid, $skiplock, $nocheck) = @_;
5893
5894 PVE::QemuConfig->lock_config($vmid, sub {
5895 my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5896 my $res = $vm_mon_cmd->($vmid, 'query-status');
5897 my $resume_cmd = 'cont';
5898
5899 if ($res->{status} && $res->{status} eq 'suspended') {
5900 $resume_cmd = 'system_wakeup';
5901 }
5902
5903 if (!$nocheck) {
5904
5905 my $conf = PVE::QemuConfig->load_config($vmid);
5906
5907 PVE::QemuConfig->check_lock($conf)
5908 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5909 }
5910
5911 $vm_mon_cmd->($vmid, $resume_cmd);
5912 });
5913}
5914
5915sub vm_sendkey {
5916 my ($vmid, $skiplock, $key) = @_;
5917
5918 PVE::QemuConfig->lock_config($vmid, sub {
5919
5920 my $conf = PVE::QemuConfig->load_config($vmid);
5921
5922 # there is no qmp command, so we use the human monitor command
5923 my $res = vm_human_monitor_command($vmid, "sendkey $key");
5924 die $res if $res ne '';
5925 });
5926}
5927
5928sub vm_destroy {
5929 my ($storecfg, $vmid, $skiplock) = @_;
5930
5931 PVE::QemuConfig->lock_config($vmid, sub {
5932
5933 my $conf = PVE::QemuConfig->load_config($vmid);
5934
5935 if (!check_running($vmid)) {
5936 destroy_vm($storecfg, $vmid, undef, $skiplock);
5937 } else {
5938 die "VM $vmid is running - destroy failed\n";
5939 }
5940 });
5941}
5942
5943# vzdump restore implementaion
5944
5945sub tar_archive_read_firstfile {
5946 my $archive = shift;
5947
5948 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5949
5950 # try to detect archive type first
5951 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5952 die "unable to open file '$archive'\n";
5953 my $firstfile = <$fh>;
5954 kill 15, $pid;
5955 close $fh;
5956
5957 die "ERROR: archive contaions no data\n" if !$firstfile;
5958 chomp $firstfile;
5959
5960 return $firstfile;
5961}
5962
5963sub tar_restore_cleanup {
5964 my ($storecfg, $statfile) = @_;
5965
5966 print STDERR "starting cleanup\n";
5967
5968 if (my $fd = IO::File->new($statfile, "r")) {
5969 while (defined(my $line = <$fd>)) {
5970 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5971 my $volid = $2;
5972 eval {
5973 if ($volid =~ m|^/|) {
5974 unlink $volid || die 'unlink failed\n';
5975 } else {
5976 PVE::Storage::vdisk_free($storecfg, $volid);
5977 }
5978 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5979 };
5980 print STDERR "unable to cleanup '$volid' - $@" if $@;
5981 } else {
5982 print STDERR "unable to parse line in statfile - $line";
5983 }
5984 }
5985 $fd->close();
5986 }
5987}
5988
5989sub restore_archive {
5990 my ($archive, $vmid, $user, $opts) = @_;
5991
5992 my $format = $opts->{format};
5993 my $comp;
5994
5995 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5996 $format = 'tar' if !$format;
5997 $comp = 'gzip';
5998 } elsif ($archive =~ m/\.tar$/) {
5999 $format = 'tar' if !$format;
6000 } elsif ($archive =~ m/.tar.lzo$/) {
6001 $format = 'tar' if !$format;
6002 $comp = 'lzop';
6003 } elsif ($archive =~ m/\.vma$/) {
6004 $format = 'vma' if !$format;
6005 } elsif ($archive =~ m/\.vma\.gz$/) {
6006 $format = 'vma' if !$format;
6007 $comp = 'gzip';
6008 } elsif ($archive =~ m/\.vma\.lzo$/) {
6009 $format = 'vma' if !$format;
6010 $comp = 'lzop';
6011 } else {
6012 $format = 'vma' if !$format; # default
6013 }
6014
6015 # try to detect archive format
6016 if ($format eq 'tar') {
6017 return restore_tar_archive($archive, $vmid, $user, $opts);
6018 } else {
6019 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6020 }
6021}
6022
6023sub restore_update_config_line {
6024 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6025
6026 return if $line =~ m/^\#qmdump\#/;
6027 return if $line =~ m/^\#vzdump\#/;
6028 return if $line =~ m/^lock:/;
6029 return if $line =~ m/^unused\d+:/;
6030 return if $line =~ m/^parent:/;
6031
6032 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6033 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6034 # try to convert old 1.X settings
6035 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6036 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6037 my ($model, $macaddr) = split(/\=/, $devconfig);
6038 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6039 my $net = {
6040 model => $model,
6041 bridge => "vmbr$ind",
6042 macaddr => $macaddr,
6043 };
6044 my $netstr = print_net($net);
6045
6046 print $outfd "net$cookie->{netcount}: $netstr\n";
6047 $cookie->{netcount}++;
6048 }
6049 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6050 my ($id, $netstr) = ($1, $2);
6051 my $net = parse_net($netstr);
6052 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6053 $netstr = print_net($net);
6054 print $outfd "$id: $netstr\n";
6055 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6056 my $virtdev = $1;
6057 my $value = $3;
6058 my $di = parse_drive($virtdev, $value);
6059 if (defined($di->{backup}) && !$di->{backup}) {
6060 print $outfd "#$line";
6061 } elsif ($map->{$virtdev}) {
6062 delete $di->{format}; # format can change on restore
6063 $di->{file} = $map->{$virtdev};
6064 $value = print_drive($vmid, $di);
6065 print $outfd "$virtdev: $value\n";
6066 } else {
6067 print $outfd $line;
6068 }
6069 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6070 my $vmgenid = $1;
6071 if ($vmgenid ne '0') {
6072 # always generate a new vmgenid if there was a valid one setup
6073 $vmgenid = generate_uuid();
6074 }
6075 print $outfd "vmgenid: $vmgenid\n";
6076 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6077 my ($uuid, $uuid_str);
6078 UUID::generate($uuid);
6079 UUID::unparse($uuid, $uuid_str);
6080 my $smbios1 = parse_smbios1($2);
6081 $smbios1->{uuid} = $uuid_str;
6082 print $outfd $1.print_smbios1($smbios1)."\n";
6083 } else {
6084 print $outfd $line;
6085 }
6086}
6087
6088sub scan_volids {
6089 my ($cfg, $vmid) = @_;
6090
6091 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6092
6093 my $volid_hash = {};
6094 foreach my $storeid (keys %$info) {
6095 foreach my $item (@{$info->{$storeid}}) {
6096 next if !($item->{volid} && $item->{size});
6097 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6098 $volid_hash->{$item->{volid}} = $item;
6099 }
6100 }
6101
6102 return $volid_hash;
6103}
6104
6105sub is_volume_in_use {
6106 my ($storecfg, $conf, $skip_drive, $volid) = @_;
6107
6108 my $path = PVE::Storage::path($storecfg, $volid);
6109
6110 my $scan_config = sub {
6111 my ($cref, $snapname) = @_;
6112
6113 foreach my $key (keys %$cref) {
6114 my $value = $cref->{$key};
6115 if (is_valid_drivename($key)) {
6116 next if $skip_drive && $key eq $skip_drive;
6117 my $drive = parse_drive($key, $value);
6118 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6119 return 1 if $volid eq $drive->{file};
6120 if ($drive->{file} =~ m!^/!) {
6121 return 1 if $drive->{file} eq $path;
6122 } else {
6123 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6124 next if !$storeid;
6125 my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6126 next if !$scfg;
6127 return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6128 }
6129 }
6130 }
6131
6132 return 0;
6133 };
6134
6135 return 1 if &$scan_config($conf);
6136
6137 undef $skip_drive;
6138
6139 foreach my $snapname (keys %{$conf->{snapshots}}) {
6140 return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6141 }
6142
6143 return 0;
6144}
6145
6146sub update_disksize {
6147 my ($vmid, $conf, $volid_hash) = @_;
6148
6149 my $changes;
6150 my $prefix = "VM $vmid:";
6151
6152 # used and unused disks
6153 my $referenced = {};
6154
6155 # Note: it is allowed to define multiple storages with same path (alias), so
6156 # we need to check both 'volid' and real 'path' (two different volid can point
6157 # to the same path).
6158
6159 my $referencedpath = {};
6160
6161 # update size info
6162 foreach my $opt (keys %$conf) {
6163 if (is_valid_drivename($opt)) {
6164 my $drive = parse_drive($opt, $conf->{$opt});
6165 my $volid = $drive->{file};
6166 next if !$volid;
6167
6168 $referenced->{$volid} = 1;
6169 if ($volid_hash->{$volid} &&
6170 (my $path = $volid_hash->{$volid}->{path})) {
6171 $referencedpath->{$path} = 1;
6172 }
6173
6174 next if drive_is_cdrom($drive);
6175 next if !$volid_hash->{$volid};
6176
6177 $drive->{size} = $volid_hash->{$volid}->{size};
6178 my $new = print_drive($vmid, $drive);
6179 if ($new ne $conf->{$opt}) {
6180 $changes = 1;
6181 $conf->{$opt} = $new;
6182 print "$prefix update disk '$opt' information.\n";
6183 }
6184 }
6185 }
6186
6187 # remove 'unusedX' entry if volume is used
6188 foreach my $opt (keys %$conf) {
6189 next if $opt !~ m/^unused\d+$/;
6190 my $volid = $conf->{$opt};
6191 my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6192 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6193 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6194 $changes = 1;
6195 delete $conf->{$opt};
6196 }
6197
6198 $referenced->{$volid} = 1;
6199 $referencedpath->{$path} = 1 if $path;
6200 }
6201
6202 foreach my $volid (sort keys %$volid_hash) {
6203 next if $volid =~ m/vm-$vmid-state-/;
6204 next if $referenced->{$volid};
6205 my $path = $volid_hash->{$volid}->{path};
6206 next if !$path; # just to be sure
6207 next if $referencedpath->{$path};
6208 $changes = 1;
6209 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6210 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6211 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6212 }
6213
6214 return $changes;
6215}
6216
6217sub rescan {
6218 my ($vmid, $nolock, $dryrun) = @_;
6219
6220 my $cfg = PVE::Storage::config();
6221
6222 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6223 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6224 foreach my $stor (keys %{$cfg->{ids}}) {
6225 delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6226 }
6227
6228 print "rescan volumes...\n";
6229 my $volid_hash = scan_volids($cfg, $vmid);
6230
6231 my $updatefn = sub {
6232 my ($vmid) = @_;
6233
6234 my $conf = PVE::QemuConfig->load_config($vmid);
6235
6236 PVE::QemuConfig->check_lock($conf);
6237
6238 my $vm_volids = {};
6239 foreach my $volid (keys %$volid_hash) {
6240 my $info = $volid_hash->{$volid};
6241 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6242 }
6243
6244 my $changes = update_disksize($vmid, $conf, $vm_volids);
6245
6246 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6247 };
6248
6249 if (defined($vmid)) {
6250 if ($nolock) {
6251 &$updatefn($vmid);
6252 } else {
6253 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6254 }
6255 } else {
6256 my $vmlist = config_list();
6257 foreach my $vmid (keys %$vmlist) {
6258 if ($nolock) {
6259 &$updatefn($vmid);
6260 } else {
6261 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6262 }
6263 }
6264 }
6265}
6266
6267sub restore_vma_archive {
6268 my ($archive, $vmid, $user, $opts, $comp) = @_;
6269
6270 my $readfrom = $archive;
6271
6272 my $cfg = PVE::Storage::config();
6273 my $commands = [];
6274 my $bwlimit = $opts->{bwlimit};
6275
6276 my $dbg_cmdstring = '';
6277 my $add_pipe = sub {
6278 my ($cmd) = @_;
6279 push @$commands, $cmd;
6280 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6281 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6282 $readfrom = '-';
6283 };
6284
6285 my $input = undef;
6286 if ($archive eq '-') {
6287 $input = '<&STDIN';
6288 } else {
6289 # If we use a backup from a PVE defined storage we also consider that
6290 # storage's rate limit:
6291 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6292 if (defined($volid)) {
6293 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6294 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6295 if ($readlimit) {
6296 print STDERR "applying read rate limit: $readlimit\n";
6297 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6298 $add_pipe->($cstream);
6299 }
6300 }
6301 }
6302
6303 if ($comp) {
6304 my $cmd;
6305 if ($comp eq 'gzip') {
6306 $cmd = ['zcat', $readfrom];
6307 } elsif ($comp eq 'lzop') {
6308 $cmd = ['lzop', '-d', '-c', $readfrom];
6309 } else {
6310 die "unknown compression method '$comp'\n";
6311 }
6312 $add_pipe->($cmd);
6313 }
6314
6315 my $tmpdir = "/var/tmp/vzdumptmp$$";
6316 rmtree $tmpdir;
6317
6318 # disable interrupts (always do cleanups)
6319 local $SIG{INT} =
6320 local $SIG{TERM} =
6321 local $SIG{QUIT} =
6322 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6323
6324 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6325 POSIX::mkfifo($mapfifo, 0600);
6326 my $fifofh;
6327
6328 my $openfifo = sub {
6329 open($fifofh, '>', $mapfifo) || die $!;
6330 };
6331
6332 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6333
6334 my $oldtimeout;
6335 my $timeout = 5;
6336
6337 my $devinfo = {};
6338
6339 my $rpcenv = PVE::RPCEnvironment::get();
6340
6341 my $conffile = PVE::QemuConfig->config_file($vmid);
6342 my $tmpfn = "$conffile.$$.tmp";
6343
6344 # Note: $oldconf is undef if VM does not exists
6345 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6346 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6347
6348 my %storage_limits;
6349
6350 my $print_devmap = sub {
6351 my $virtdev_hash = {};
6352
6353 my $cfgfn = "$tmpdir/qemu-server.conf";
6354
6355 # we can read the config - that is already extracted
6356 my $fh = IO::File->new($cfgfn, "r") ||
6357 "unable to read qemu-server.conf - $!\n";
6358
6359 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6360 if (-f $fwcfgfn) {
6361 my $pve_firewall_dir = '/etc/pve/firewall';
6362 mkdir $pve_firewall_dir; # make sure the dir exists
6363 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6364 }
6365
6366 while (defined(my $line = <$fh>)) {
6367 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6368 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6369 die "archive does not contain data for drive '$virtdev'\n"
6370 if !$devinfo->{$devname};
6371 if (defined($opts->{storage})) {
6372 $storeid = $opts->{storage} || 'local';
6373 } elsif (!$storeid) {
6374 $storeid = 'local';
6375 }
6376 $format = 'raw' if !$format;
6377 $devinfo->{$devname}->{devname} = $devname;
6378 $devinfo->{$devname}->{virtdev} = $virtdev;
6379 $devinfo->{$devname}->{format} = $format;
6380 $devinfo->{$devname}->{storeid} = $storeid;
6381
6382 # check permission on storage
6383 my $pool = $opts->{pool}; # todo: do we need that?
6384 if ($user ne 'root@pam') {
6385 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6386 }
6387
6388 $storage_limits{$storeid} = $bwlimit;
6389
6390 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6391 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6392 my $virtdev = $1;
6393 my $drive = parse_drive($virtdev, $2);
6394 if (drive_is_cloudinit($drive)) {
6395 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6396 my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6397 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6398
6399 my $d = {
6400 format => $format,
6401 storeid => $opts->{storage} // $storeid,
6402 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6403 file => $drive->{file}, # to make drive_is_cloudinit check possible
6404 name => "vm-$vmid-cloudinit",
6405 is_cloudinit => 1,
6406 };
6407 $virtdev_hash->{$virtdev} = $d;
6408 }
6409 }
6410 }
6411
6412 foreach my $key (keys %storage_limits) {
6413 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6414 next if !$limit;
6415 print STDERR "rate limit for storage $key: $limit KiB/s\n";
6416 $storage_limits{$key} = $limit * 1024;
6417 }
6418
6419 foreach my $devname (keys %$devinfo) {
6420 die "found no device mapping information for device '$devname'\n"
6421 if !$devinfo->{$devname}->{virtdev};
6422 }
6423
6424 # create empty/temp config
6425 if ($oldconf) {
6426 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6427 foreach_drive($oldconf, sub {
6428 my ($ds, $drive) = @_;
6429
6430 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6431
6432 my $volid = $drive->{file};
6433 return if !$volid || $volid =~ m|^/|;
6434
6435 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6436 return if !$path || !$owner || ($owner != $vmid);
6437
6438 # Note: only delete disk we want to restore
6439 # other volumes will become unused
6440 if ($virtdev_hash->{$ds}) {
6441 eval { PVE::Storage::vdisk_free($cfg, $volid); };
6442 if (my $err = $@) {
6443 warn $err;
6444 }
6445 }
6446 });
6447
6448 # delete vmstate files, after the restore we have no snapshots anymore
6449 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6450 my $snap = $oldconf->{snapshots}->{$snapname};
6451 if ($snap->{vmstate}) {
6452 eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6453 if (my $err = $@) {
6454 warn $err;
6455 }
6456 }
6457 }
6458 }
6459
6460 my $map = {};
6461 foreach my $virtdev (sort keys %$virtdev_hash) {
6462 my $d = $virtdev_hash->{$virtdev};
6463 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6464 my $storeid = $d->{storeid};
6465 my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6466
6467 my $map_opts = '';
6468 if (my $limit = $storage_limits{$storeid}) {
6469 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6470 }
6471
6472 # test if requested format is supported
6473 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6474 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6475 $d->{format} = $defFormat if !$supported;
6476
6477 my $name;
6478 if ($d->{is_cloudinit}) {
6479 $name = $d->{name};
6480 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6481 }
6482
6483 my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6484 print STDERR "new volume ID is '$volid'\n";
6485 $d->{volid} = $volid;
6486
6487 PVE::Storage::activate_volumes($cfg, [$volid]);
6488
6489 my $write_zeros = 1;
6490 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6491 $write_zeros = 0;
6492 }
6493
6494 if (!$d->{is_cloudinit}) {
6495 my $path = PVE::Storage::path($cfg, $volid);
6496
6497 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6498
6499 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6500 }
6501 $map->{$virtdev} = $volid;
6502 }
6503
6504 $fh->seek(0, 0) || die "seek failed - $!\n";
6505
6506 my $outfd = new IO::File ($tmpfn, "w") ||
6507 die "unable to write config for VM $vmid\n";
6508
6509 my $cookie = { netcount => 0 };
6510 while (defined(my $line = <$fh>)) {
6511 restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6512 }
6513
6514 $fh->close();
6515 $outfd->close();
6516 };
6517
6518 eval {
6519 # enable interrupts
6520 local $SIG{INT} =
6521 local $SIG{TERM} =
6522 local $SIG{QUIT} =
6523 local $SIG{HUP} =
6524 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6525 local $SIG{ALRM} = sub { die "got timeout\n"; };
6526
6527 $oldtimeout = alarm($timeout);
6528
6529 my $parser = sub {
6530 my $line = shift;
6531
6532 print "$line\n";
6533
6534 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6535 my ($dev_id, $size, $devname) = ($1, $2, $3);
6536 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6537 } elsif ($line =~ m/^CTIME: /) {
6538 # we correctly received the vma config, so we can disable
6539 # the timeout now for disk allocation (set to 10 minutes, so
6540 # that we always timeout if something goes wrong)
6541 alarm(600);
6542 &$print_devmap();
6543 print $fifofh "done\n";
6544 my $tmp = $oldtimeout || 0;
6545 $oldtimeout = undef;
6546 alarm($tmp);
6547 close($fifofh);
6548 }
6549 };
6550
6551 print "restore vma archive: $dbg_cmdstring\n";
6552 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6553 };
6554 my $err = $@;
6555
6556 alarm($oldtimeout) if $oldtimeout;
6557
6558 my $vollist = [];
6559 foreach my $devname (keys %$devinfo) {
6560 my $volid = $devinfo->{$devname}->{volid};
6561 push @$vollist, $volid if $volid;
6562 }
6563
6564 PVE::Storage::deactivate_volumes($cfg, $vollist);
6565
6566 unlink $mapfifo;
6567
6568 if ($err) {
6569 rmtree $tmpdir;
6570 unlink $tmpfn;
6571
6572 foreach my $devname (keys %$devinfo) {
6573 my $volid = $devinfo->{$devname}->{volid};
6574 next if !$volid;
6575 eval {
6576 if ($volid =~ m|^/|) {
6577 unlink $volid || die 'unlink failed\n';
6578 } else {
6579 PVE::Storage::vdisk_free($cfg, $volid);
6580 }
6581 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6582 };
6583 print STDERR "unable to cleanup '$volid' - $@" if $@;
6584 }
6585 die $err;
6586 }
6587
6588 rmtree $tmpdir;
6589
6590 rename($tmpfn, $conffile) ||
6591 die "unable to commit configuration file '$conffile'\n";
6592
6593 PVE::Cluster::cfs_update(); # make sure we read new file
6594
6595 eval { rescan($vmid, 1); };
6596 warn $@ if $@;
6597}
6598
6599sub restore_tar_archive {
6600 my ($archive, $vmid, $user, $opts) = @_;
6601
6602 if ($archive ne '-') {
6603 my $firstfile = tar_archive_read_firstfile($archive);
6604 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6605 if $firstfile ne 'qemu-server.conf';
6606 }
6607
6608 my $storecfg = PVE::Storage::config();
6609
6610 # destroy existing data - keep empty config
6611 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6612 destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6613
6614 my $tocmd = "/usr/lib/qemu-server/qmextract";
6615
6616 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6617 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6618 $tocmd .= ' --prealloc' if $opts->{prealloc};
6619 $tocmd .= ' --info' if $opts->{info};
6620
6621 # tar option "xf" does not autodetect compression when read from STDIN,
6622 # so we pipe to zcat
6623 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6624 PVE::Tools::shellquote("--to-command=$tocmd");
6625
6626 my $tmpdir = "/var/tmp/vzdumptmp$$";
6627 mkpath $tmpdir;
6628
6629 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6630 local $ENV{VZDUMP_VMID} = $vmid;
6631 local $ENV{VZDUMP_USER} = $user;
6632
6633 my $conffile = PVE::QemuConfig->config_file($vmid);
6634 my $tmpfn = "$conffile.$$.tmp";
6635
6636 # disable interrupts (always do cleanups)
6637 local $SIG{INT} =
6638 local $SIG{TERM} =
6639 local $SIG{QUIT} =
6640 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6641
6642 eval {
6643 # enable interrupts
6644 local $SIG{INT} =
6645 local $SIG{TERM} =
6646 local $SIG{QUIT} =
6647 local $SIG{HUP} =
6648 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6649
6650 if ($archive eq '-') {
6651 print "extracting archive from STDIN\n";
6652 run_command($cmd, input => "<&STDIN");
6653 } else {
6654 print "extracting archive '$archive'\n";
6655 run_command($cmd);
6656 }
6657
6658 return if $opts->{info};
6659
6660 # read new mapping
6661 my $map = {};
6662 my $statfile = "$tmpdir/qmrestore.stat";
6663 if (my $fd = IO::File->new($statfile, "r")) {
6664 while (defined (my $line = <$fd>)) {
6665 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6666 $map->{$1} = $2 if $1;
6667 } else {
6668 print STDERR "unable to parse line in statfile - $line\n";
6669 }
6670 }
6671 $fd->close();
6672 }
6673
6674 my $confsrc = "$tmpdir/qemu-server.conf";
6675
6676 my $srcfd = new IO::File($confsrc, "r") ||
6677 die "unable to open file '$confsrc'\n";
6678
6679 my $outfd = new IO::File ($tmpfn, "w") ||
6680 die "unable to write config for VM $vmid\n";
6681
6682 my $cookie = { netcount => 0 };
6683 while (defined (my $line = <$srcfd>)) {
6684 restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6685 }
6686
6687 $srcfd->close();
6688 $outfd->close();
6689 };
6690 my $err = $@;
6691
6692 if ($err) {
6693
6694 unlink $tmpfn;
6695
6696 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6697
6698 die $err;
6699 }
6700
6701 rmtree $tmpdir;
6702
6703 rename $tmpfn, $conffile ||
6704 die "unable to commit configuration file '$conffile'\n";
6705
6706 PVE::Cluster::cfs_update(); # make sure we read new file
6707
6708 eval { rescan($vmid, 1); };
6709 warn $@ if $@;
6710};
6711
6712sub foreach_storage_used_by_vm {
6713 my ($conf, $func) = @_;
6714
6715 my $sidhash = {};
6716
6717 foreach_drive($conf, sub {
6718 my ($ds, $drive) = @_;
6719 return if drive_is_cdrom($drive);
6720
6721 my $volid = $drive->{file};
6722
6723 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6724 $sidhash->{$sid} = $sid if $sid;
6725 });
6726
6727 foreach my $sid (sort keys %$sidhash) {
6728 &$func($sid);
6729 }
6730}
6731
6732sub do_snapshots_with_qemu {
6733 my ($storecfg, $volid) = @_;
6734
6735 my $storage_name = PVE::Storage::parse_volume_id($volid);
6736 my $scfg = $storecfg->{ids}->{$storage_name};
6737
6738 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6739 return 1;
6740 }
6741
6742 if ($volid =~ m/\.(qcow2|qed)$/){
6743 return 1;
6744 }
6745
6746 return undef;
6747}
6748
6749sub qga_check_running {
6750 my ($vmid, $nowarn) = @_;
6751
6752 eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6753 if ($@) {
6754 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6755 return 0;
6756 }
6757 return 1;
6758}
6759
6760sub template_create {
6761 my ($vmid, $conf, $disk) = @_;
6762
6763 my $storecfg = PVE::Storage::config();
6764
6765 foreach_drive($conf, sub {
6766 my ($ds, $drive) = @_;
6767
6768 return if drive_is_cdrom($drive);
6769 return if $disk && $ds ne $disk;
6770
6771 my $volid = $drive->{file};
6772 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6773
6774 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6775 $drive->{file} = $voliddst;
6776 $conf->{$ds} = print_drive($vmid, $drive);
6777 PVE::QemuConfig->write_config($vmid, $conf);
6778 });
6779}
6780
6781sub convert_iscsi_path {
6782 my ($path) = @_;
6783
6784 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6785 my $portal = $1;
6786 my $target = $2;
6787 my $lun = $3;
6788
6789 my $initiator_name = get_initiator_name();
6790
6791 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6792 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6793 }
6794
6795 die "cannot convert iscsi path '$path', unkown format\n";
6796}
6797
6798sub qemu_img_convert {
6799 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6800
6801 my $storecfg = PVE::Storage::config();
6802 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6803 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6804
6805 if ($src_storeid && $dst_storeid) {
6806
6807 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6808
6809 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6810 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6811
6812 my $src_format = qemu_img_format($src_scfg, $src_volname);
6813 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6814
6815 my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6816 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6817
6818 my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6819 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6820
6821 my $cmd = [];
6822 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6823 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6824 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6825 push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6826
6827 if ($src_is_iscsi) {
6828 push @$cmd, '--image-opts';
6829 $src_path = convert_iscsi_path($src_path);
6830 } else {
6831 push @$cmd, '-f', $src_format;
6832 }
6833
6834 if ($dst_is_iscsi) {
6835 push @$cmd, '--target-image-opts';
6836 $dst_path = convert_iscsi_path($dst_path);
6837 } else {
6838 push @$cmd, '-O', $dst_format;
6839 }
6840
6841 push @$cmd, $src_path;
6842
6843 if (!$dst_is_iscsi && $is_zero_initialized) {
6844 push @$cmd, "zeroinit:$dst_path";
6845 } else {
6846 push @$cmd, $dst_path;
6847 }
6848
6849 my $parser = sub {
6850 my $line = shift;
6851 if($line =~ m/\((\S+)\/100\%\)/){
6852 my $percent = $1;
6853 my $transferred = int($size * $percent / 100);
6854 my $remaining = $size - $transferred;
6855
6856 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6857 }
6858
6859 };
6860
6861 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6862 my $err = $@;
6863 die "copy failed: $err" if $err;
6864 }
6865}
6866
6867sub qemu_img_format {
6868 my ($scfg, $volname) = @_;
6869
6870 if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6871 return $1;
6872 } else {
6873 return "raw";
6874 }
6875}
6876
6877sub qemu_drive_mirror {
6878 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6879
6880 $jobs = {} if !$jobs;
6881
6882 my $qemu_target;
6883 my $format;
6884 $jobs->{"drive-$drive"} = {};
6885
6886 if ($dst_volid =~ /^nbd:/) {
6887 $qemu_target = $dst_volid;
6888 $format = "nbd";
6889 } else {
6890 my $storecfg = PVE::Storage::config();
6891 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6892
6893 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6894
6895 $format = qemu_img_format($dst_scfg, $dst_volname);
6896
6897 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6898
6899 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6900 }
6901
6902 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6903 $opts->{format} = $format if $format;
6904
6905 if (defined($bwlimit)) {
6906 $opts->{speed} = $bwlimit * 1024;
6907 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6908 } else {
6909 print "drive mirror is starting for drive-$drive\n";
6910 }
6911
6912 # if a job already runs for this device we get an error, catch it for cleanup
6913 eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6914 if (my $err = $@) {
6915 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6916 warn "$@\n" if $@;
6917 die "mirroring error: $err\n";
6918 }
6919
6920 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6921}
6922
6923sub qemu_drive_mirror_monitor {
6924 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6925
6926 eval {
6927 my $err_complete = 0;
6928
6929 while (1) {
6930 die "storage migration timed out\n" if $err_complete > 300;
6931
6932 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6933
6934 my $running_mirror_jobs = {};
6935 foreach my $stat (@$stats) {
6936 next if $stat->{type} ne 'mirror';
6937 $running_mirror_jobs->{$stat->{device}} = $stat;
6938 }
6939
6940 my $readycounter = 0;
6941
6942 foreach my $job (keys %$jobs) {
6943
6944 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6945 print "$job : finished\n";
6946 delete $jobs->{$job};
6947 next;
6948 }
6949
6950 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6951
6952 my $busy = $running_mirror_jobs->{$job}->{busy};
6953 my $ready = $running_mirror_jobs->{$job}->{ready};
6954 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6955 my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6956 my $remaining = $total - $transferred;
6957 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6958
6959 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6960 }
6961
6962 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6963 }
6964
6965 last if scalar(keys %$jobs) == 0;
6966
6967 if ($readycounter == scalar(keys %$jobs)) {
6968 print "all mirroring jobs are ready \n";
6969 last if $skipcomplete; #do the complete later
6970
6971 if ($vmiddst && $vmiddst != $vmid) {
6972 my $agent_running = $qga && qga_check_running($vmid);
6973 if ($agent_running) {
6974 print "freeze filesystem\n";
6975 eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6976 } else {
6977 print "suspend vm\n";
6978 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6979 }
6980
6981 # if we clone a disk for a new target vm, we don't switch the disk
6982 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6983
6984 if ($agent_running) {
6985 print "unfreeze filesystem\n";
6986 eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6987 } else {
6988 print "resume vm\n";
6989 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
6990 }
6991
6992 last;
6993 } else {
6994
6995 foreach my $job (keys %$jobs) {
6996 # try to switch the disk if source and destination are on the same guest
6997 print "$job: Completing block job...\n";
6998
6999 eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7000 if ($@ =~ m/cannot be completed/) {
7001 print "$job: Block job cannot be completed, try again.\n";
7002 $err_complete++;
7003 }else {
7004 print "$job: Completed successfully.\n";
7005 $jobs->{$job}->{complete} = 1;
7006 }
7007 }
7008 }
7009 }
7010 sleep 1;
7011 }
7012 };
7013 my $err = $@;
7014
7015 if ($err) {
7016 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7017 die "mirroring error: $err";
7018 }
7019
7020}
7021
7022sub qemu_blockjobs_cancel {
7023 my ($vmid, $jobs) = @_;
7024
7025 foreach my $job (keys %$jobs) {
7026 print "$job: Cancelling block job\n";
7027 eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7028 $jobs->{$job}->{cancel} = 1;
7029 }
7030
7031 while (1) {
7032 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7033
7034 my $running_jobs = {};
7035 foreach my $stat (@$stats) {
7036 $running_jobs->{$stat->{device}} = $stat;
7037 }
7038
7039 foreach my $job (keys %$jobs) {
7040
7041 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7042 print "$job: Done.\n";
7043 delete $jobs->{$job};
7044 }
7045 }
7046
7047 last if scalar(keys %$jobs) == 0;
7048
7049 sleep 1;
7050 }
7051}
7052
7053sub clone_disk {
7054 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7055 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7056
7057 my $newvolid;
7058
7059 if (!$full) {
7060 print "create linked clone of drive $drivename ($drive->{file})\n";
7061 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
7062 push @$newvollist, $newvolid;
7063 } else {
7064
7065 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7066 $storeid = $storage if $storage;
7067
7068 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7069 my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7070
7071 print "create full clone of drive $drivename ($drive->{file})\n";
7072 my $name = undef;
7073 if (drive_is_cloudinit($drive)) {
7074 $name = "vm-$newvmid-cloudinit";
7075 $snapname = undef;
7076 # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7077 if ($dst_format ne 'raw') {
7078 $name .= ".$dst_format";
7079 }
7080 }
7081 $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7082 push @$newvollist, $newvolid;
7083
7084 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7085
7086 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7087 if (!$running || $snapname) {
7088 # TODO: handle bwlimits
7089 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7090 } else {
7091
7092 my $kvmver = get_running_qemu_version ($vmid);
7093 if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7094 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7095 if $drive->{iothread};
7096 }
7097
7098 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7099 }
7100 }
7101
7102 my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7103
7104 my $disk = $drive;
7105 $disk->{format} = undef;
7106 $disk->{file} = $newvolid;
7107 $disk->{size} = $size;
7108
7109 return $disk;
7110}
7111
7112# this only works if VM is running
7113sub get_current_qemu_machine {
7114 my ($vmid) = @_;
7115
7116 my $cmd = { execute => 'query-machines', arguments => {} };
7117 my $res = vm_qmp_command($vmid, $cmd);
7118
7119 my ($current, $default);
7120 foreach my $e (@$res) {
7121 $default = $e->{name} if $e->{'is-default'};
7122 $current = $e->{name} if $e->{'is-current'};
7123 }
7124
7125 # fallback to the default machine if current is not supported by qemu
7126 return $current || $default || 'pc';
7127}
7128
7129sub get_running_qemu_version {
7130 my ($vmid) = @_;
7131 my $cmd = { execute => 'query-version', arguments => {} };
7132 my $res = vm_qmp_command($vmid, $cmd);
7133 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7134}
7135
7136sub qemu_machine_feature_enabled {
7137 my ($machine, $kvmver, $version_major, $version_minor) = @_;
7138
7139 my $current_major;
7140 my $current_minor;
7141
7142 if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7143
7144 $current_major = $3;
7145 $current_minor = $4;
7146
7147 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7148
7149 $current_major = $1;
7150 $current_minor = $2;
7151 }
7152
7153 return 1 if $current_major > $version_major ||
7154 ($current_major == $version_major &&
7155 $current_minor >= $version_minor);
7156}
7157
7158sub qemu_machine_pxe {
7159 my ($vmid, $conf, $machine) = @_;
7160
7161 $machine = PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7162
7163 if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7164 $machine .= '.pxe';
7165 }
7166
7167 return $machine;
7168}
7169
7170sub qemu_use_old_bios_files {
7171 my ($machine_type) = @_;
7172
7173 return if !$machine_type;
7174
7175 my $use_old_bios_files = undef;
7176
7177 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7178 $machine_type = $1;
7179 $use_old_bios_files = 1;
7180 } else {
7181 my $kvmver = kvm_user_version();
7182 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7183 # load new efi bios files on migration. So this hack is required to allow
7184 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7185 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7186 $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7187 }
7188
7189 return ($use_old_bios_files, $machine_type);
7190}
7191
7192sub create_efidisk($$$$$) {
7193 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7194
7195 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7196 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7197
7198 my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7199 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7200 PVE::Storage::activate_volumes($storecfg, [$volid]);
7201
7202 my $path = PVE::Storage::path($storecfg, $volid);
7203 eval {
7204 run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7205 };
7206 die "Copying EFI vars image failed: $@" if $@;
7207
7208 return ($volid, $vars_size);
7209}
7210
7211sub vm_iothreads_list {
7212 my ($vmid) = @_;
7213
7214 my $res = vm_mon_cmd($vmid, 'query-iothreads');
7215
7216 my $iothreads = {};
7217 foreach my $iothread (@$res) {
7218 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7219 }
7220
7221 return $iothreads;
7222}
7223
7224sub scsihw_infos {
7225 my ($conf, $drive) = @_;
7226
7227 my $maxdev = 0;
7228
7229 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7230 $maxdev = 7;
7231 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7232 $maxdev = 1;
7233 } else {
7234 $maxdev = 256;
7235 }
7236
7237 my $controller = int($drive->{index} / $maxdev);
7238 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7239
7240 return ($maxdev, $controller, $controller_prefix);
7241}
7242
7243sub add_hyperv_enlightenments {
7244 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7245
7246 return if $winversion < 6;
7247 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7248
7249 if ($gpu_passthrough || defined($hv_vendor_id)) {
7250 $hv_vendor_id //= 'proxmox';
7251 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7252 }
7253
7254 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7255 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7256 push @$cpuFlags , 'hv_vapic';
7257 push @$cpuFlags , 'hv_time';
7258 } else {
7259 push @$cpuFlags , 'hv_spinlocks=0xffff';
7260 }
7261
7262 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7263 push @$cpuFlags , 'hv_reset';
7264 push @$cpuFlags , 'hv_vpindex';
7265 push @$cpuFlags , 'hv_runtime';
7266 }
7267
7268 if ($winversion >= 7) {
7269 push @$cpuFlags , 'hv_relaxed';
7270
7271 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7272 push @$cpuFlags , 'hv_synic';
7273 push @$cpuFlags , 'hv_stimer';
7274 }
7275
7276 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7277 push @$cpuFlags , 'hv_tlbflush';
7278 push @$cpuFlags , 'hv_ipi';
7279 # FIXME: AMD does not supports this currently, only add with special flag??
7280 #push @$cpuFlags , 'hv_evmcs';
7281 }
7282 }
7283}
7284
7285sub windows_version {
7286 my ($ostype) = @_;
7287
7288 return 0 if !$ostype;
7289
7290 my $winversion = 0;
7291
7292 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7293 $winversion = 5;
7294 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7295 $winversion = 6;
7296 } elsif ($ostype =~ m/^win(\d+)$/) {
7297 $winversion = $1;
7298 }
7299
7300 return $winversion;
7301}
7302
7303sub resolve_dst_disk_format {
7304 my ($storecfg, $storeid, $src_volname, $format) = @_;
7305 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7306
7307 if (!$format) {
7308 # if no target format is specified, use the source disk format as hint
7309 if ($src_volname) {
7310 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7311 $format = qemu_img_format($scfg, $src_volname);
7312 } else {
7313 return $defFormat;
7314 }
7315 }
7316
7317 # test if requested format is supported - else use default
7318 my $supported = grep { $_ eq $format } @$validFormats;
7319 $format = $defFormat if !$supported;
7320 return $format;
7321}
7322
7323sub resolve_first_disk {
7324 my $conf = shift;
7325 my @disks = PVE::QemuServer::valid_drive_names();
7326 my $firstdisk;
7327 foreach my $ds (reverse @disks) {
7328 next if !$conf->{$ds};
7329 my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7330 next if PVE::QemuServer::drive_is_cdrom($disk);
7331 $firstdisk = $ds;
7332 }
7333 return $firstdisk;
7334}
7335
7336sub generate_uuid {
7337 my ($uuid, $uuid_str);
7338 UUID::generate($uuid);
7339 UUID::unparse($uuid, $uuid_str);
7340 return $uuid_str;
7341}
7342
7343sub generate_smbios1_uuid {
7344 return "uuid=".generate_uuid();
7345}
7346
7347sub nbd_stop {
7348 my ($vmid) = @_;
7349
7350 vm_mon_cmd($vmid, 'nbd-server-stop');
7351}
7352
7353# bash completion helper
7354
7355sub complete_backup_archives {
7356 my ($cmdname, $pname, $cvalue) = @_;
7357
7358 my $cfg = PVE::Storage::config();
7359
7360 my $storeid;
7361
7362 if ($cvalue =~ m/^([^:]+):/) {
7363 $storeid = $1;
7364 }
7365
7366 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7367
7368 my $res = [];
7369 foreach my $id (keys %$data) {
7370 foreach my $item (@{$data->{$id}}) {
7371 next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7372 push @$res, $item->{volid} if defined($item->{volid});
7373 }
7374 }
7375
7376 return $res;
7377}
7378
7379my $complete_vmid_full = sub {
7380 my ($running) = @_;
7381
7382 my $idlist = vmstatus();
7383
7384 my $res = [];
7385
7386 foreach my $id (keys %$idlist) {
7387 my $d = $idlist->{$id};
7388 if (defined($running)) {
7389 next if $d->{template};
7390 next if $running && $d->{status} ne 'running';
7391 next if !$running && $d->{status} eq 'running';
7392 }
7393 push @$res, $id;
7394
7395 }
7396 return $res;
7397};
7398
7399sub complete_vmid {
7400 return &$complete_vmid_full();
7401}
7402
7403sub complete_vmid_stopped {
7404 return &$complete_vmid_full(0);
7405}
7406
7407sub complete_vmid_running {
7408 return &$complete_vmid_full(1);
7409}
7410
7411sub complete_storage {
7412
7413 my $cfg = PVE::Storage::config();
7414 my $ids = $cfg->{ids};
7415
7416 my $res = [];
7417 foreach my $sid (keys %$ids) {
7418 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7419 next if !$ids->{$sid}->{content}->{images};
7420 push @$res, $sid;
7421 }
7422
7423 return $res;
7424}
7425
74261;
Virtual Machine Manager