]> git.proxmox.com Git - qemu-server.git/blame - PVE/QemuServer.pm
Fix #2412: Missing VMs in pools
[qemu-server.git] / PVE / QemuServer.pm
CommitLineData
1e3baf05
DM
1package PVE::QemuServer;
2
3use strict;
990fc5e2 4use warnings;
3ff84d6f 5
1e3baf05
DM
6use POSIX;
7use IO::Handle;
8use IO::Select;
9use IO::File;
10use IO::Dir;
11use IO::Socket::UNIX;
12use File::Basename;
13use File::Path;
14use File::stat;
15use Getopt::Long;
fc1ddcdc 16use Digest::SHA;
1e3baf05
DM
17use Fcntl ':flock';
18use Cwd 'abs_path';
19use IPC::Open3;
c971c4f2 20use JSON;
1e3baf05
DM
21use Fcntl;
22use PVE::SafeSyslog;
23use Storable qw(dclone);
1f30ac3a 24use MIME::Base64;
1e3baf05
DM
25use PVE::Exception qw(raise raise_param_exc);
26use PVE::Storage;
0c9a7596 27use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
b7ba6b79 28use PVE::JSONSchema qw(get_standard_option);
1e3baf05
DM
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30use PVE::INotify;
31use PVE::ProcFSTools;
ffda963f 32use PVE::QemuConfig;
26f11676 33use PVE::QMPClient;
91bd6c90 34use PVE::RPCEnvironment;
9e784b11 35use PVE::GuestHelpers;
c4e16381 36use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
3f669af2 37use PVE::QemuServer::Memory;
d40e5e18 38use PVE::QemuServer::USB qw(parse_usb_device);
0c9a7596 39use PVE::QemuServer::Cloudinit;
b71351a7 40use PVE::SysFSTools;
d04d6af1 41use PVE::Systemd;
6b64503e 42use Time::HiRes qw(gettimeofday);
a783c78e 43use File::Copy qw(copy);
46630a5f 44use URI::Escape;
1e3baf05 45
102cf9d8 46my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
96ed3574
WB
47my $OVMF = {
48 x86_64 => [
49 "$EDK2_FW_BASE/OVMF_CODE.fd",
50 "$EDK2_FW_BASE/OVMF_VARS.fd"
51 ],
52 aarch64 => [
53 "$EDK2_FW_BASE/AAVMF_CODE.fd",
54 "$EDK2_FW_BASE/AAVMF_VARS.fd"
55 ],
56};
2ddc0a5c 57
90041ba8 58my $qemu_snap_storage = { rbd => 1 };
e5eaa028 59
7f0b5beb 60my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
1e3baf05 61
9c52f5ed
WB
62my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
63
19672434 64# Note about locking: we use flock on the config file protect
1e3baf05
DM
65# against concurent actions.
66# Aditionaly, we have a 'lock' setting in the config file. This
22c377f0 67# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
1e3baf05
DM
68# allowed when such lock is set. But you can ignore this kind of
69# lock with the --skiplock flag.
70
97d62eb7 71cfs_register_file('/qemu-server/',
1858638f
DM
72 \&parse_vm_config,
73 \&write_vm_config);
1e3baf05 74
3ea94c60
DM
75PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
76 description => "Some command save/restore state from this location.",
77 type => 'string',
78 maxLength => 128,
79 optional => 1,
80});
81
c7d2b650
DM
82PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
83 type => 'string',
84 enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
85 description => "The drive's backing file's data format.",
86 optional => 1,
87});
88
c6737ef1
DC
89PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
90 description => "Specifies the Qemu machine type.",
91 type => 'string',
d731ecbe 92 pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
c6737ef1
DC
93 maxLength => 40,
94 optional => 1,
95});
96
1e3baf05
DM
97#no warnings 'redefine';
98
c8effec3
AD
99sub cgroups_write {
100 my ($controller, $vmid, $option, $value) = @_;
101
3a515a88
DM
102 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
103 PVE::ProcFSTools::write_proc_entry($path, $value);
c8effec3
AD
104
105}
106
1e3baf05
DM
107my $nodename = PVE::INotify::nodename();
108
109mkdir "/etc/pve/nodes/$nodename";
110my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
111mkdir $confdir;
112
113my $var_run_tmpdir = "/var/run/qemu-server";
114mkdir $var_run_tmpdir;
115
116my $lock_dir = "/var/lock/qemu-server";
117mkdir $lock_dir;
118
8930da74
DM
119my $cpu_vendor_list = {
120 # Intel CPUs
121 486 => 'GenuineIntel',
122 pentium => 'GenuineIntel',
123 pentium2 => 'GenuineIntel',
124 pentium3 => 'GenuineIntel',
125 coreduo => 'GenuineIntel',
126 core2duo => 'GenuineIntel',
127 Conroe => 'GenuineIntel',
9052caba 128 Penryn => 'GenuineIntel',
8930da74 129 Nehalem => 'GenuineIntel',
9052caba 130 'Nehalem-IBRS' => 'GenuineIntel',
8930da74 131 Westmere => 'GenuineIntel',
9052caba 132 'Westmere-IBRS' => 'GenuineIntel',
8930da74 133 SandyBridge => 'GenuineIntel',
9052caba 134 'SandyBridge-IBRS' => 'GenuineIntel',
8930da74 135 IvyBridge => 'GenuineIntel',
9052caba 136 'IvyBridge-IBRS' => 'GenuineIntel',
8930da74 137 Haswell => 'GenuineIntel',
9052caba 138 'Haswell-IBRS' => 'GenuineIntel',
8930da74 139 'Haswell-noTSX' => 'GenuineIntel',
9052caba 140 'Haswell-noTSX-IBRS' => 'GenuineIntel',
8930da74 141 Broadwell => 'GenuineIntel',
9052caba 142 'Broadwell-IBRS' => 'GenuineIntel',
8930da74 143 'Broadwell-noTSX' => 'GenuineIntel',
9052caba 144 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
3db920fc 145 'Skylake-Client' => 'GenuineIntel',
9052caba 146 'Skylake-Client-IBRS' => 'GenuineIntel',
a446dbf4
FG
147 'Skylake-Server' => 'GenuineIntel',
148 'Skylake-Server-IBRS' => 'GenuineIntel',
faf80f25
AD
149 'Cascadelake-Server' => 'GenuineIntel',
150 KnightsMill => 'GenuineIntel',
151
9052caba 152
8930da74
DM
153 # AMD CPUs
154 athlon => 'AuthenticAMD',
155 phenom => 'AuthenticAMD',
156 Opteron_G1 => 'AuthenticAMD',
157 Opteron_G2 => 'AuthenticAMD',
158 Opteron_G3 => 'AuthenticAMD',
159 Opteron_G4 => 'AuthenticAMD',
160 Opteron_G5 => 'AuthenticAMD',
2a850ee8
WB
161 EPYC => 'AuthenticAMD',
162 'EPYC-IBPB' => 'AuthenticAMD',
8930da74
DM
163
164 # generic types, use vendor from host node
165 host => 'default',
166 kvm32 => 'default',
167 kvm64 => 'default',
168 qemu32 => 'default',
169 qemu64 => 'default',
a446dbf4 170 max => 'default',
8930da74
DM
171};
172
3a040a0e
SR
173my @supported_cpu_flags = (
174 'pcid',
175 'spec-ctrl',
176 'ibpb',
177 'ssbd',
178 'virt-ssbd',
179 'amd-ssbd',
180 'amd-no-ssb',
181 'pdpe1gb',
182 'md-clear',
183 'hv-tlbflush',
cd98c467
PKS
184 'hv-evmcs',
185 'aes'
3a040a0e
SR
186);
187my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
049fc9eb 188
ff6ffe20 189my $cpu_fmt = {
16a91d65
WB
190 cputype => {
191 description => "Emulated CPU type.",
192 type => 'string',
7f694a71 193 enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
16a91d65
WB
194 default => 'kvm64',
195 default_key => 1,
196 },
197 hidden => {
198 description => "Do not identify as a KVM virtual machine.",
199 type => 'boolean',
200 optional => 1,
201 default => 0
202 },
2894c247
DC
203 'hv-vendor-id' => {
204 type => 'string',
205 pattern => qr/[a-zA-Z0-9]{1,12}/,
206 format_description => 'vendor-id',
fc5c194b 207 description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
2894c247
DC
208 optional => 1,
209 },
39fd79e2 210 flags => {
049fc9eb
FG
211 description => "List of additional CPU flags separated by ';'."
212 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
3a040a0e 213 . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
049fc9eb 214 format_description => '+FLAG[;-FLAG...]',
39fd79e2 215 type => 'string',
049fc9eb 216 pattern => qr/$cpu_flag(;$cpu_flag)*/,
39fd79e2 217 optional => 1,
39fd79e2 218 },
16a91d65
WB
219};
220
ec3582b5
WB
221my $watchdog_fmt = {
222 model => {
223 default_key => 1,
224 type => 'string',
225 enum => [qw(i6300esb ib700)],
226 description => "Watchdog type to emulate.",
227 default => 'i6300esb',
228 optional => 1,
229 },
230 action => {
231 type => 'string',
232 enum => [qw(reset shutdown poweroff pause debug none)],
233 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
234 optional => 1,
235 },
236};
237PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
238
9d66b397
SI
239my $agent_fmt = {
240 enabled => {
241 description => "Enable/disable Qemu GuestAgent.",
242 type => 'boolean',
243 default => 0,
244 default_key => 1,
245 },
246 fstrim_cloned_disks => {
247 description => "Run fstrim after cloning/moving a disk.",
248 type => 'boolean',
249 optional => 1,
250 default => 0
251 },
252};
253
55655ebc
DC
254my $vga_fmt = {
255 type => {
256 description => "Select the VGA type.",
257 type => 'string',
258 default => 'std',
259 optional => 1,
260 default_key => 1,
7c954c42 261 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
55655ebc
DC
262 },
263 memory => {
264 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
265 type => 'integer',
266 optional => 1,
267 minimum => 4,
268 maximum => 512,
269 },
270};
271
6dbcb073
DC
272my $ivshmem_fmt = {
273 size => {
274 type => 'integer',
275 minimum => 1,
276 description => "The size of the file in MB.",
277 },
278 name => {
279 type => 'string',
280 pattern => '[a-zA-Z0-9\-]+',
281 optional => 1,
282 format_description => 'string',
283 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
284 },
285};
286
1448547f
AL
287my $audio_fmt = {
288 device => {
289 type => 'string',
290 enum => [qw(ich9-intel-hda intel-hda AC97)],
291 description => "Configure an audio device."
292 },
293 driver => {
294 type => 'string',
295 enum => ['spice'],
296 default => 'spice',
297 optional => 1,
298 description => "Driver backend for the audio device."
299 },
300};
301
c4df18db
AL
302my $spice_enhancements_fmt = {
303 foldersharing => {
304 type => 'boolean',
305 optional => 1,
d282a24d 306 default => '0',
c4df18db
AL
307 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
308 },
309 videostreaming => {
310 type => 'string',
311 enum => ['off', 'all', 'filter'],
d282a24d 312 default => 'off',
c4df18db
AL
313 optional => 1,
314 description => "Enable video streaming. Uses compression for detected video streams."
315 },
316};
317
1e3baf05
DM
318my $confdesc = {
319 onboot => {
320 optional => 1,
321 type => 'boolean',
322 description => "Specifies whether a VM will be started during system bootup.",
323 default => 0,
324 },
325 autostart => {
326 optional => 1,
327 type => 'boolean',
328 description => "Automatic restart after crash (currently ignored).",
329 default => 0,
330 },
2ff09f52
DA
331 hotplug => {
332 optional => 1,
b3c2bdd1
DM
333 type => 'string', format => 'pve-hotplug-features',
334 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
335 default => 'network,disk,usb',
2ff09f52 336 },
1e3baf05
DM
337 reboot => {
338 optional => 1,
339 type => 'boolean',
340 description => "Allow reboot. If set to '0' the VM exit on reboot.",
341 default => 1,
342 },
343 lock => {
344 optional => 1,
345 type => 'string',
346 description => "Lock/unlock the VM.",
159719e5 347 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
1e3baf05
DM
348 },
349 cpulimit => {
350 optional => 1,
c6f773b8 351 type => 'number',
52261945
DM
352 description => "Limit of CPU usage.",
353 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
1e3baf05 354 minimum => 0,
c6f773b8 355 maximum => 128,
52261945 356 default => 0,
1e3baf05
DM
357 },
358 cpuunits => {
359 optional => 1,
360 type => 'integer',
52261945 361 description => "CPU weight for a VM.",
237239bf
PA
362 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
363 minimum => 2,
364 maximum => 262144,
613d76a1 365 default => 1024,
1e3baf05
DM
366 },
367 memory => {
368 optional => 1,
369 type => 'integer',
7878afeb 370 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
1e3baf05
DM
371 minimum => 16,
372 default => 512,
373 },
13a48620
DA
374 balloon => {
375 optional => 1,
376 type => 'integer',
8b1accf7
DM
377 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
378 minimum => 0,
379 },
380 shares => {
381 optional => 1,
382 type => 'integer',
82329cd5 383 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
8b1accf7
DM
384 minimum => 0,
385 maximum => 50000,
386 default => 1000,
13a48620 387 },
1e3baf05
DM
388 keyboard => {
389 optional => 1,
390 type => 'string',
f889aa0f 391 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
aea47dd6 392 "It should not be necessary to set it.",
e95fe75f 393 enum => PVE::Tools::kvmkeymaplist(),
aea47dd6 394 default => undef,
1e3baf05
DM
395 },
396 name => {
397 optional => 1,
7fabe17d 398 type => 'string', format => 'dns-name',
1e3baf05
DM
399 description => "Set a name for the VM. Only used on the configuration web interface.",
400 },
cdd20088
AD
401 scsihw => {
402 optional => 1,
403 type => 'string',
52261945 404 description => "SCSI controller model",
6731a4cf 405 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
cdd20088
AD
406 default => 'lsi',
407 },
1e3baf05
DM
408 description => {
409 optional => 1,
410 type => 'string',
0581fe4f 411 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
1e3baf05
DM
412 },
413 ostype => {
414 optional => 1,
415 type => 'string',
0cb9971e 416 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
52261945
DM
417 description => "Specify guest operating system.",
418 verbose_description => <<EODESC,
419Specify guest operating system. This is used to enable special
420optimization/features for specific operating systems:
421
422[horizontal]
423other;; unspecified OS
424wxp;; Microsoft Windows XP
425w2k;; Microsoft Windows 2000
426w2k3;; Microsoft Windows 2003
427w2k8;; Microsoft Windows 2008
428wvista;; Microsoft Windows Vista
429win7;; Microsoft Windows 7
44c2a647
TL
430win8;; Microsoft Windows 8/2012/2012r2
431win10;; Microsoft Windows 10/2016
52261945
DM
432l24;; Linux 2.4 Kernel
433l26;; Linux 2.6/3.X Kernel
434solaris;; Solaris/OpenSolaris/OpenIndiania kernel
1e3baf05
DM
435EODESC
436 },
437 boot => {
438 optional => 1,
439 type => 'string',
440 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
441 pattern => '[acdn]{1,4}',
32baffb4 442 default => 'cdn',
1e3baf05
DM
443 },
444 bootdisk => {
445 optional => 1,
446 type => 'string', format => 'pve-qm-bootdisk',
447 description => "Enable booting from specified disk.",
03e480fc 448 pattern => '(ide|sata|scsi|virtio)\d+',
1e3baf05
DM
449 },
450 smp => {
451 optional => 1,
452 type => 'integer',
453 description => "The number of CPUs. Please use option -sockets instead.",
454 minimum => 1,
455 default => 1,
456 },
457 sockets => {
458 optional => 1,
459 type => 'integer',
460 description => "The number of CPU sockets.",
461 minimum => 1,
462 default => 1,
463 },
464 cores => {
465 optional => 1,
466 type => 'integer',
467 description => "The number of cores per socket.",
468 minimum => 1,
469 default => 1,
470 },
8a010eae
AD
471 numa => {
472 optional => 1,
473 type => 'boolean',
1917695c 474 description => "Enable/disable NUMA.",
8a010eae
AD
475 default => 0,
476 },
7023f3ea
AD
477 hugepages => {
478 optional => 1,
479 type => 'string',
480 description => "Enable/disable hugepages memory.",
481 enum => [qw(any 2 1024)],
482 },
de9d1e55 483 vcpus => {
3bd18e48
AD
484 optional => 1,
485 type => 'integer',
de9d1e55 486 description => "Number of hotplugged vcpus.",
3bd18e48 487 minimum => 1,
de9d1e55 488 default => 0,
3bd18e48 489 },
1e3baf05
DM
490 acpi => {
491 optional => 1,
492 type => 'boolean',
493 description => "Enable/disable ACPI.",
494 default => 1,
495 },
bc84dcca 496 agent => {
ab6a046f 497 optional => 1,
9d66b397
SI
498 description => "Enable/disable Qemu GuestAgent and its properties.",
499 type => 'string',
500 format => $agent_fmt,
ab6a046f 501 },
1e3baf05
DM
502 kvm => {
503 optional => 1,
504 type => 'boolean',
505 description => "Enable/disable KVM hardware virtualization.",
506 default => 1,
507 },
508 tdf => {
509 optional => 1,
510 type => 'boolean',
8c559505
DM
511 description => "Enable/disable time drift fix.",
512 default => 0,
1e3baf05 513 },
19672434 514 localtime => {
1e3baf05
DM
515 optional => 1,
516 type => 'boolean',
517 description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
518 },
519 freeze => {
520 optional => 1,
521 type => 'boolean',
522 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
523 },
524 vga => {
525 optional => 1,
55655ebc
DC
526 type => 'string', format => $vga_fmt,
527 description => "Configure the VGA hardware.",
528 verbose_description => "Configure the VGA Hardware. If you want to use ".
529 "high resolution modes (>= 1280x1024x16) you may need to increase " .
530 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
531 "is 'std' for all OS types besides some Windows versions (XP and " .
532 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
533 "display server. For win* OS you can select how many independent " .
534 "displays you want, Linux guests can add displays them self.\n".
535 "You can also run without any graphic card, using a serial device as terminal.",
1e3baf05 536 },
0ea9541d
DM
537 watchdog => {
538 optional => 1,
539 type => 'string', format => 'pve-qm-watchdog',
52261945
DM
540 description => "Create a virtual hardware watchdog device.",
541 verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
1917695c
TL
542 " (by a guest action), the watchdog must be periodically polled " .
543 "by an agent inside the guest or else the watchdog will reset " .
544 "the guest (or execute the respective action specified)",
0ea9541d 545 },
1e3baf05
DM
546 startdate => {
547 optional => 1,
19672434 548 type => 'string',
1e3baf05
DM
549 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
550 description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
551 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
552 default => 'now',
553 },
43574f73 554 startup => get_standard_option('pve-startup-order'),
68eda3ab
AD
555 template => {
556 optional => 1,
557 type => 'boolean',
558 description => "Enable/disable Template.",
559 default => 0,
560 },
1e3baf05
DM
561 args => {
562 optional => 1,
563 type => 'string',
52261945
DM
564 description => "Arbitrary arguments passed to kvm.",
565 verbose_description => <<EODESCR,
c7a8aad6 566Arbitrary arguments passed to kvm, for example:
1e3baf05
DM
567
568args: -no-reboot -no-hpet
c7a8aad6
FG
569
570NOTE: this option is for experts only.
1e3baf05
DM
571EODESCR
572 },
573 tablet => {
574 optional => 1,
575 type => 'boolean',
576 default => 1,
52261945
DM
577 description => "Enable/disable the USB tablet device.",
578 verbose_description => "Enable/disable the USB tablet device. This device is " .
1917695c
TL
579 "usually needed to allow absolute mouse positioning with VNC. " .
580 "Else the mouse runs out of sync with normal VNC clients. " .
581 "If you're running lots of console-only guests on one host, " .
582 "you may consider disabling this to save some context switches. " .
583 "This is turned off by default if you use spice (-vga=qxl).",
1e3baf05
DM
584 },
585 migrate_speed => {
586 optional => 1,
587 type => 'integer',
588 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
589 minimum => 0,
590 default => 0,
591 },
592 migrate_downtime => {
593 optional => 1,
04432191 594 type => 'number',
1e3baf05
DM
595 description => "Set maximum tolerated downtime (in seconds) for migrations.",
596 minimum => 0,
04432191 597 default => 0.1,
1e3baf05
DM
598 },
599 cdrom => {
600 optional => 1,
b799312f 601 type => 'string', format => 'pve-qm-ide',
8485b9ba 602 typetext => '<volume>',
1e3baf05
DM
603 description => "This is an alias for option -ide2",
604 },
605 cpu => {
606 optional => 1,
607 description => "Emulated CPU type.",
608 type => 'string',
ff6ffe20 609 format => $cpu_fmt,
1e3baf05 610 },
b7ba6b79
DM
611 parent => get_standard_option('pve-snapshot-name', {
612 optional => 1,
613 description => "Parent snapshot name. This is used internally, and should not be modified.",
614 }),
982c7f12
DM
615 snaptime => {
616 optional => 1,
617 description => "Timestamp for snapshots.",
618 type => 'integer',
619 minimum => 0,
620 },
18bfb361
DM
621 vmstate => {
622 optional => 1,
623 type => 'string', format => 'pve-volume-id',
624 description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
625 },
253624c7
FG
626 vmstatestorage => get_standard_option('pve-storage-id', {
627 description => "Default storage for VM state volumes/files.",
628 optional => 1,
629 }),
c6737ef1
DC
630 runningmachine => get_standard_option('pve-qemu-machine', {
631 description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
632 }),
633 machine => get_standard_option('pve-qemu-machine'),
d731ecbe
WB
634 arch => {
635 description => "Virtual processor architecture. Defaults to the host.",
636 optional => 1,
637 type => 'string',
638 enum => [qw(x86_64 aarch64)],
639 },
2796e7d5
DM
640 smbios1 => {
641 description => "Specify SMBIOS type 1 fields.",
642 type => 'string', format => 'pve-qm-smbios1',
5d004b00 643 maxLength => 512,
2796e7d5
DM
644 optional => 1,
645 },
cb0e4540
AG
646 protection => {
647 optional => 1,
648 type => 'boolean',
52261945 649 description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
cb0e4540
AG
650 default => 0,
651 },
3edb45e7 652 bios => {
a783c78e 653 optional => 1,
3edb45e7
DM
654 type => 'string',
655 enum => [ qw(seabios ovmf) ],
656 description => "Select BIOS implementation.",
657 default => 'seabios',
a783c78e 658 },
6ee499ff
DC
659 vmgenid => {
660 type => 'string',
661 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
662 format_description => 'UUID',
f7ed64e7
TL
663 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
664 verbose_description => "The VM generation ID (vmgenid) device exposes a".
665 " 128-bit integer value identifier to the guest OS. This allows to".
666 " notify the guest operating system when the virtual machine is".
667 " executed with a different configuration (e.g. snapshot execution".
668 " or creation from a template). The guest operating system notices".
669 " the change, and is then able to react as appropriate by marking".
670 " its copies of distributed databases as dirty, re-initializing its".
671 " random number generator, etc.\n".
672 "Note that auto-creation only works when done throug API/CLI create".
673 " or update methods, but not when manually editing the config file.",
674 default => "1 (autogenerated)",
6ee499ff
DC
675 optional => 1,
676 },
9e784b11
DC
677 hookscript => {
678 type => 'string',
679 format => 'pve-volume-id',
680 optional => 1,
681 description => "Script that will be executed during various steps in the vms lifetime.",
682 },
6dbcb073
DC
683 ivshmem => {
684 type => 'string',
685 format => $ivshmem_fmt,
686 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
687 optional => 1,
2e7b5925
AL
688 },
689 audio0 => {
690 type => 'string',
1448547f 691 format => $audio_fmt,
194b65f1 692 description => "Configure a audio device, useful in combination with QXL/Spice.",
2e7b5925
AL
693 optional => 1
694 },
c4df18db
AL
695 spice_enhancements => {
696 type => 'string',
697 format => $spice_enhancements_fmt,
698 description => "Configure additional enhancements for SPICE.",
699 optional => 1
700 },
9ed7a77c
WB
701};
702
cb702ebe
DL
703my $cicustom_fmt = {
704 meta => {
705 type => 'string',
706 optional => 1,
707 description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
708 format => 'pve-volume-id',
709 format_description => 'volume',
710 },
711 network => {
712 type => 'string',
713 optional => 1,
714 description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
715 format => 'pve-volume-id',
716 format_description => 'volume',
717 },
718 user => {
719 type => 'string',
720 optional => 1,
721 description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
722 format => 'pve-volume-id',
723 format_description => 'volume',
724 },
725};
726PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
727
9ed7a77c 728my $confdesc_cloudinit = {
41cd94a0
WB
729 citype => {
730 optional => 1,
731 type => 'string',
498cdc36 732 description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
41cd94a0
WB
733 enum => ['configdrive2', 'nocloud'],
734 },
7b42f951
WB
735 ciuser => {
736 optional => 1,
737 type => 'string',
738 description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
739 },
740 cipassword => {
741 optional => 1,
742 type => 'string',
1d1c4e1c 743 description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
7b42f951 744 },
cb702ebe
DL
745 cicustom => {
746 optional => 1,
747 type => 'string',
748 description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
749 format => 'pve-qm-cicustom',
750 },
0c9a7596
AD
751 searchdomain => {
752 optional => 1,
753 type => 'string',
754 description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
755 },
756 nameserver => {
757 optional => 1,
758 type => 'string', format => 'address-list',
759 description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
760 },
761 sshkeys => {
762 optional => 1,
763 type => 'string',
764 format => 'urlencoded',
1d1c4e1c 765 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
0c9a7596 766 },
1e3baf05
DM
767};
768
769# what about other qemu settings ?
770#cpu => 'string',
771#machine => 'string',
772#fda => 'file',
773#fdb => 'file',
774#mtdblock => 'file',
775#sd => 'file',
776#pflash => 'file',
777#snapshot => 'bool',
778#bootp => 'file',
779##tftp => 'dir',
780##smb => 'dir',
781#kernel => 'file',
782#append => 'string',
783#initrd => 'file',
784##soundhw => 'string',
785
786while (my ($k, $v) = each %$confdesc) {
787 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
788}
789
790my $MAX_IDE_DISKS = 4;
f62db2a4 791my $MAX_SCSI_DISKS = 14;
a2650619 792my $MAX_VIRTIO_DISKS = 16;
cdb0931f 793my $MAX_SATA_DISKS = 6;
1e3baf05 794my $MAX_USB_DEVICES = 5;
5bdcf937 795my $MAX_NETS = 32;
c9db2240 796my $MAX_UNUSED_DISKS = 256;
c4e16381 797my $MAX_HOSTPCI_DEVICES = 16;
bae179aa 798my $MAX_SERIAL_PORTS = 4;
1989a89c 799my $MAX_PARALLEL_PORTS = 3;
2ed5d572
AD
800my $MAX_NUMA = 8;
801
ffc0d8c7
WB
802my $numa_fmt = {
803 cpus => {
804 type => "string",
805 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
52261945 806 description => "CPUs accessing this NUMA node.",
ffc0d8c7
WB
807 format_description => "id[-id];...",
808 },
809 memory => {
810 type => "number",
52261945 811 description => "Amount of memory this NUMA node provides.",
ffc0d8c7
WB
812 optional => 1,
813 },
814 hostnodes => {
815 type => "string",
816 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
52261945 817 description => "Host NUMA nodes to use.",
ffc0d8c7
WB
818 format_description => "id[-id];...",
819 optional => 1,
820 },
821 policy => {
822 type => 'string',
823 enum => [qw(preferred bind interleave)],
52261945 824 description => "NUMA allocation policy.",
ffc0d8c7
WB
825 optional => 1,
826 },
827};
828PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
2ed5d572
AD
829my $numadesc = {
830 optional => 1,
ffc0d8c7 831 type => 'string', format => $numa_fmt,
52261945 832 description => "NUMA topology.",
2ed5d572
AD
833};
834PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
835
836for (my $i = 0; $i < $MAX_NUMA; $i++) {
837 $confdesc->{"numa$i"} = $numadesc;
838}
1e3baf05
DM
839
840my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
55034103
KT
841 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
842 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
6b64503e 843my $nic_model_list_txt = join(' ', sort @$nic_model_list);
1e3baf05 844
52261945
DM
845my $net_fmt_bridge_descr = <<__EOD__;
846Bridge to attach the network device to. The Proxmox VE standard bridge
847is called 'vmbr0'.
848
849If you do not specify a bridge, we create a kvm user (NATed) network
850device, which provides DHCP and DNS services. The following addresses
851are used:
852
853 10.0.2.2 Gateway
854 10.0.2.3 DNS Server
855 10.0.2.4 SMB Server
856
857The DHCP server assign addresses to the guest starting from 10.0.2.15.
858__EOD__
859
cd9c34d1 860my $net_fmt = {
399d96db 861 macaddr => get_standard_option('mac-addr', {
52261945 862 description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
399d96db 863 }),
7f694a71
DM
864 model => {
865 type => 'string',
52261945 866 description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
7f694a71
DM
867 enum => $nic_model_list,
868 default_key => 1,
869 },
870 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
cd9c34d1
WB
871 bridge => {
872 type => 'string',
52261945 873 description => $net_fmt_bridge_descr,
cd9c34d1
WB
874 format_description => 'bridge',
875 optional => 1,
876 },
877 queues => {
878 type => 'integer',
879 minimum => 0, maximum => 16,
880 description => 'Number of packet queues to be used on the device.',
cd9c34d1
WB
881 optional => 1,
882 },
883 rate => {
884 type => 'number',
885 minimum => 0,
52261945 886 description => "Rate limit in mbps (megabytes per second) as floating point number.",
cd9c34d1
WB
887 optional => 1,
888 },
889 tag => {
890 type => 'integer',
9f41a659 891 minimum => 1, maximum => 4094,
cd9c34d1 892 description => 'VLAN tag to apply to packets on this interface.',
cd9c34d1
WB
893 optional => 1,
894 },
895 trunks => {
896 type => 'string',
897 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
898 description => 'VLAN trunks to pass through this interface.',
7f694a71 899 format_description => 'vlanid[;vlanid...]',
cd9c34d1
WB
900 optional => 1,
901 },
902 firewall => {
903 type => 'boolean',
904 description => 'Whether this interface should be protected by the firewall.',
cd9c34d1
WB
905 optional => 1,
906 },
907 link_down => {
908 type => 'boolean',
52261945 909 description => 'Whether this interface should be disconnected (like pulling the plug).',
cd9c34d1
WB
910 optional => 1,
911 },
912};
52261945 913
1e3baf05
DM
914my $netdesc = {
915 optional => 1,
7f694a71 916 type => 'string', format => $net_fmt,
52261945 917 description => "Specify network devices.",
1e3baf05 918};
52261945 919
1e3baf05
DM
920PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
921
0c9a7596
AD
922my $ipconfig_fmt = {
923 ip => {
924 type => 'string',
925 format => 'pve-ipv4-config',
926 format_description => 'IPv4Format/CIDR',
927 description => 'IPv4 address in CIDR format.',
928 optional => 1,
929 default => 'dhcp',
930 },
931 gw => {
932 type => 'string',
933 format => 'ipv4',
934 format_description => 'GatewayIPv4',
935 description => 'Default gateway for IPv4 traffic.',
936 optional => 1,
937 requires => 'ip',
938 },
939 ip6 => {
940 type => 'string',
941 format => 'pve-ipv6-config',
942 format_description => 'IPv6Format/CIDR',
943 description => 'IPv6 address in CIDR format.',
944 optional => 1,
945 default => 'dhcp',
946 },
947 gw6 => {
948 type => 'string',
949 format => 'ipv6',
950 format_description => 'GatewayIPv6',
951 description => 'Default gateway for IPv6 traffic.',
952 optional => 1,
953 requires => 'ip6',
954 },
955};
956PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
957my $ipconfigdesc = {
958 optional => 1,
959 type => 'string', format => 'pve-qm-ipconfig',
960 description => <<'EODESCR',
961cloud-init: Specify IP addresses and gateways for the corresponding interface.
962
963IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
964
965The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
966For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
967
968If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
969EODESCR
970};
971PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
972
1e3baf05
DM
973for (my $i = 0; $i < $MAX_NETS; $i++) {
974 $confdesc->{"net$i"} = $netdesc;
9ed7a77c
WB
975 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
976}
977
978foreach my $key (keys %$confdesc_cloudinit) {
979 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1e3baf05
DM
980}
981
ffa42b86
DC
982PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
983sub verify_volume_id_or_qm_path {
822c8a07
WB
984 my ($volid, $noerr) = @_;
985
ffa42b86
DC
986 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
987 return $volid;
988 }
989
990 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
822c8a07
WB
991 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
992 if ($@) {
993 return undef if $noerr;
994 die $@;
995 }
996 return $volid;
997}
998
1e3baf05 999my $drivename_hash;
19672434 1000
0541eeb8
WB
1001my %drivedesc_base = (
1002 volume => { alias => 'file' },
1003 file => {
93c0971c 1004 type => 'string',
ffa42b86 1005 format => 'pve-volume-id-or-qm-path',
0541eeb8
WB
1006 default_key => 1,
1007 format_description => 'volume',
1008 description => "The drive's backing volume.",
1009 },
1010 media => {
1011 type => 'string',
0541eeb8
WB
1012 enum => [qw(cdrom disk)],
1013 description => "The drive's media type.",
1014 default => 'disk',
1015 optional => 1
1016 },
1017 cyls => {
1018 type => 'integer',
0541eeb8
WB
1019 description => "Force the drive's physical geometry to have a specific cylinder count.",
1020 optional => 1
1021 },
1022 heads => {
1023 type => 'integer',
0541eeb8
WB
1024 description => "Force the drive's physical geometry to have a specific head count.",
1025 optional => 1
1026 },
1027 secs => {
1028 type => 'integer',
0541eeb8
WB
1029 description => "Force the drive's physical geometry to have a specific sector count.",
1030 optional => 1
1031 },
1032 trans => {
1033 type => 'string',
0541eeb8
WB
1034 enum => [qw(none lba auto)],
1035 description => "Force disk geometry bios translation mode.",
1036 optional => 1,
1037 },
1038 snapshot => {
1039 type => 'boolean',
bfb04cfc
WB
1040 description => "Controls qemu's snapshot mode feature."
1041 . " If activated, changes made to the disk are temporary and will"
1042 . " be discarded when the VM is shutdown.",
0541eeb8
WB
1043 optional => 1,
1044 },
1045 cache => {
1046 type => 'string',
0541eeb8
WB
1047 enum => [qw(none writethrough writeback unsafe directsync)],
1048 description => "The drive's cache mode",
1049 optional => 1,
1050 },
c7d2b650 1051 format => get_standard_option('pve-qm-image-format'),
0541eeb8 1052 size => {
47c28a68
WB
1053 type => 'string',
1054 format => 'disk-size',
7f694a71 1055 format_description => 'DiskSize',
0541eeb8
WB
1056 description => "Disk size. This is purely informational and has no effect.",
1057 optional => 1,
1058 },
1059 backup => {
1060 type => 'boolean',
0541eeb8
WB
1061 description => "Whether the drive should be included when making backups.",
1062 optional => 1,
1063 },
8557d01f 1064 replicate => {
9edac22f 1065 type => 'boolean',
3ab7663a 1066 description => 'Whether the drive should considered for replication jobs.',
9edac22f
WL
1067 optional => 1,
1068 default => 1,
1069 },
6e9d2550
AD
1070 rerror => {
1071 type => 'string',
1072 enum => [qw(ignore report stop)],
1073 description => 'Read error action.',
1074 optional => 1,
1075 },
0541eeb8
WB
1076 werror => {
1077 type => 'string',
0541eeb8
WB
1078 enum => [qw(enospc ignore report stop)],
1079 description => 'Write error action.',
1080 optional => 1,
1081 },
1082 aio => {
1083 type => 'string',
0541eeb8
WB
1084 enum => [qw(native threads)],
1085 description => 'AIO type to use.',
1086 optional => 1,
1087 },
1088 discard => {
1089 type => 'string',
0541eeb8
WB
1090 enum => [qw(ignore on)],
1091 description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1092 optional => 1,
1093 },
1094 detect_zeroes => {
1095 type => 'boolean',
1096 description => 'Controls whether to detect and try to optimize writes of zeroes.',
1097 optional => 1,
1098 },
1099 serial => {
1100 type => 'string',
46630a5f 1101 format => 'urlencoded',
0541eeb8 1102 format_description => 'serial',
ba8fc5d1
WB
1103 maxLength => 20*3, # *3 since it's %xx url enoded
1104 description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
0541eeb8 1105 optional => 1,
ec82e3ee
CH
1106 },
1107 shared => {
1108 type => 'boolean',
1109 description => 'Mark this locally-managed volume as available on all nodes',
1110 verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1111 optional => 1,
1112 default => 0,
0541eeb8
WB
1113 }
1114);
1115
0541eeb8
WB
1116my %iothread_fmt = ( iothread => {
1117 type => 'boolean',
0541eeb8
WB
1118 description => "Whether to use iothreads for this drive",
1119 optional => 1,
1120});
1121
1122my %model_fmt = (
1123 model => {
1124 type => 'string',
46630a5f 1125 format => 'urlencoded',
0541eeb8 1126 format_description => 'model',
ba8fc5d1
WB
1127 maxLength => 40*3, # *3 since it's %xx url enoded
1128 description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
0541eeb8
WB
1129 optional => 1,
1130 },
1131);
1132
1133my %queues_fmt = (
1134 queues => {
1135 type => 'integer',
0541eeb8
WB
1136 description => "Number of queues.",
1137 minimum => 2,
1138 optional => 1
1139 }
1140);
1141
8e3c33ab
FG
1142my %scsiblock_fmt = (
1143 scsiblock => {
1144 type => 'boolean',
1145 description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1146 optional => 1,
1147 default => 0,
1148 },
1149);
1150
6c875f9f
NC
1151my %ssd_fmt = (
1152 ssd => {
1153 type => 'boolean',
1154 description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1155 optional => 1,
1156 },
1157);
1158
e741c516
CE
1159my %wwn_fmt = (
1160 wwn => {
1161 type => 'string',
1162 pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1163 format_description => 'wwn',
1164 description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1165 optional => 1,
1166 },
1167);
1168
0541eeb8 1169my $add_throttle_desc = sub {
9196a8ec
WB
1170 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1171 my $d = {
0541eeb8 1172 type => $type,
7f694a71 1173 format_description => $unit,
9196a8ec 1174 description => "Maximum $what in $longunit.",
0541eeb8
WB
1175 optional => 1,
1176 };
9196a8ec
WB
1177 $d->{minimum} = $minimum if defined($minimum);
1178 $drivedesc_base{$key} = $d;
0541eeb8
WB
1179};
1180# throughput: (leaky bucket)
d3f3f1b3
DM
1181$add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1182$add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1183$add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1184$add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1185$add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1186$add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1187$add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1188$add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1189$add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
0541eeb8
WB
1190
1191# pools: (pool of IO before throttling starts taking effect)
d3f3f1b3
DM
1192$add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1193$add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1194$add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1195$add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1196$add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1197$add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
9196a8ec
WB
1198
1199# burst lengths
fb8e95a2
WB
1200$add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1201$add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1202$add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203$add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1204$add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1205$add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206
1207# legacy support
1208$drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1209$drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1210$drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1211$drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
0541eeb8
WB
1212
1213my $ide_fmt = {
1214 %drivedesc_base,
0541eeb8 1215 %model_fmt,
6c875f9f 1216 %ssd_fmt,
e741c516 1217 %wwn_fmt,
0541eeb8 1218};
b799312f 1219PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
0541eeb8 1220
1e3baf05
DM
1221my $idedesc = {
1222 optional => 1,
0541eeb8 1223 type => 'string', format => $ide_fmt,
3c770faa 1224 description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1e3baf05
DM
1225};
1226PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1227
0541eeb8
WB
1228my $scsi_fmt = {
1229 %drivedesc_base,
1230 %iothread_fmt,
1231 %queues_fmt,
8e3c33ab 1232 %scsiblock_fmt,
6c875f9f 1233 %ssd_fmt,
e741c516 1234 %wwn_fmt,
0541eeb8 1235};
1e3baf05
DM
1236my $scsidesc = {
1237 optional => 1,
0541eeb8 1238 type => 'string', format => $scsi_fmt,
3c770faa 1239 description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1e3baf05
DM
1240};
1241PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1242
0541eeb8
WB
1243my $sata_fmt = {
1244 %drivedesc_base,
6c875f9f 1245 %ssd_fmt,
e741c516 1246 %wwn_fmt,
0541eeb8 1247};
cdb0931f
DA
1248my $satadesc = {
1249 optional => 1,
0541eeb8 1250 type => 'string', format => $sata_fmt,
3c770faa 1251 description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
cdb0931f
DA
1252};
1253PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1254
0541eeb8
WB
1255my $virtio_fmt = {
1256 %drivedesc_base,
1257 %iothread_fmt,
0541eeb8 1258};
1e3baf05
DM
1259my $virtiodesc = {
1260 optional => 1,
0541eeb8 1261 type => 'string', format => $virtio_fmt,
3c770faa 1262 description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1e3baf05
DM
1263};
1264PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1265
0541eeb8
WB
1266my $alldrive_fmt = {
1267 %drivedesc_base,
0541eeb8
WB
1268 %iothread_fmt,
1269 %model_fmt,
1270 %queues_fmt,
8e3c33ab 1271 %scsiblock_fmt,
6c875f9f 1272 %ssd_fmt,
e741c516 1273 %wwn_fmt,
0541eeb8
WB
1274};
1275
6470743f
DC
1276my $efidisk_fmt = {
1277 volume => { alias => 'file' },
1278 file => {
1279 type => 'string',
1280 format => 'pve-volume-id-or-qm-path',
1281 default_key => 1,
1282 format_description => 'volume',
1283 description => "The drive's backing volume.",
1284 },
c7d2b650 1285 format => get_standard_option('pve-qm-image-format'),
6470743f
DC
1286 size => {
1287 type => 'string',
1288 format => 'disk-size',
1289 format_description => 'DiskSize',
1290 description => "Disk size. This is purely informational and has no effect.",
1291 optional => 1,
1292 },
1293};
1294
1295my $efidisk_desc = {
1296 optional => 1,
1297 type => 'string', format => $efidisk_fmt,
1298 description => "Configure a Disk for storing EFI vars",
1299};
1300
1301PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1302
ff6ffe20 1303my $usb_fmt = {
a6b9aee4
DC
1304 host => {
1305 default_key => 1,
1306 type => 'string', format => 'pve-qm-usb-device',
1307 format_description => 'HOSTUSBDEVICE|spice',
52261945
DM
1308 description => <<EODESCR,
1309The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1310
1311 'bus-port(.port)*' (decimal numbers) or
1312 'vendor_id:product_id' (hexadeciaml numbers) or
1313 'spice'
1314
1315You can use the 'lsusb -t' command to list existing usb devices.
1316
1317NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1318
1319The value 'spice' can be used to add a usb redirection devices for spice.
1320EODESCR
a6b9aee4
DC
1321 },
1322 usb3 => {
1323 optional => 1,
1324 type => 'boolean',
97ef5356 1325 description => "Specifies whether if given host option is a USB3 device or port.",
52261945 1326 default => 0,
a6b9aee4
DC
1327 },
1328};
1329
1e3baf05
DM
1330my $usbdesc = {
1331 optional => 1,
ff6ffe20 1332 type => 'string', format => $usb_fmt,
52261945 1333 description => "Configure an USB device (n is 0 to 4).",
1e3baf05
DM
1334};
1335PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1336
2fd24788 1337my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1f4f447b
WB
1338my $hostpci_fmt = {
1339 host => {
1340 default_key => 1,
1341 type => 'string',
1342 pattern => qr/$PCIRE(;$PCIRE)*/,
1343 format_description => 'HOSTPCIID[;HOSTPCIID2...]',
52261945 1344 description => <<EODESCR,
370b05e7 1345Host PCI device pass through. The PCI ID of a host's PCI device or a list
52261945
DM
1346of PCI virtual functions of the host. HOSTPCIID syntax is:
1347
1348'bus:dev.func' (hexadecimal numbers)
1349
1350You can us the 'lspci' command to list existing PCI devices.
52261945 1351EODESCR
1f4f447b
WB
1352 },
1353 rombar => {
1354 type => 'boolean',
52261945 1355 description => "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1f4f447b
WB
1356 optional => 1,
1357 default => 1,
1358 },
456a6fec
AD
1359 romfile => {
1360 type => 'string',
1361 pattern => '[^,;]+',
1362 format_description => 'string',
1363 description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1364 optional => 1,
1365 },
1f4f447b
WB
1366 pcie => {
1367 type => 'boolean',
52261945 1368 description => "Choose the PCI-express bus (needs the 'q35' machine model).",
1f4f447b
WB
1369 optional => 1,
1370 default => 0,
1371 },
1372 'x-vga' => {
1373 type => 'boolean',
52261945 1374 description => "Enable vfio-vga device support.",
1f4f447b
WB
1375 optional => 1,
1376 default => 0,
1377 },
6ab45bd7
DC
1378 'mdev' => {
1379 type => 'string',
1380 format_description => 'string',
1381 pattern => '[^/\.:]+',
1382 optional => 1,
1383 description => <<EODESCR
1384The type of mediated device to use.
1385An instance of this type will be created on startup of the VM and
1386will be cleaned up when the VM stops.
1387EODESCR
1388 }
1f4f447b
WB
1389};
1390PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1391
040b06b7
DA
1392my $hostpcidesc = {
1393 optional => 1,
1394 type => 'string', format => 'pve-qm-hostpci',
52261945 1395 description => "Map host PCI devices into guest.",
faab5306
DM
1396 verbose_description => <<EODESCR,
1397Map host PCI devices into guest.
1398
370b05e7 1399NOTE: This option allows direct access to host hardware. So it is no longer
faab5306
DM
1400possible to migrate such machines - use with special care.
1401
1402CAUTION: Experimental! User reported problems with this option.
1403EODESCR
040b06b7
DA
1404};
1405PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1406
bae179aa
DA
1407my $serialdesc = {
1408 optional => 1,
ca0cef26 1409 type => 'string',
1b0b51ed 1410 pattern => '(/dev/.+|socket)',
52261945
DM
1411 description => "Create a serial device inside the VM (n is 0 to 3)",
1412 verbose_description => <<EODESCR,
52261945
DM
1413Create a serial device inside the VM (n is 0 to 3), and pass through a
1414host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1415host side (use 'qm terminal' to open a terminal connection).
bae179aa 1416
8a61e0fd 1417NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
bae179aa 1418
52261945 1419CAUTION: Experimental! User reported problems with this option.
bae179aa
DA
1420EODESCR
1421};
bae179aa 1422
1989a89c
DA
1423my $paralleldesc= {
1424 optional => 1,
ca0cef26 1425 type => 'string',
9ecc8431 1426 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
52261945
DM
1427 description => "Map host parallel devices (n is 0 to 2).",
1428 verbose_description => <<EODESCR,
19672434 1429Map host parallel devices (n is 0 to 2).
1989a89c 1430
8a61e0fd 1431NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1989a89c 1432
52261945 1433CAUTION: Experimental! User reported problems with this option.
1989a89c
DA
1434EODESCR
1435};
1989a89c
DA
1436
1437for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1438 $confdesc->{"parallel$i"} = $paralleldesc;
1439}
1440
bae179aa
DA
1441for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1442 $confdesc->{"serial$i"} = $serialdesc;
1443}
1444
040b06b7
DA
1445for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1446 $confdesc->{"hostpci$i"} = $hostpcidesc;
1447}
1e3baf05
DM
1448
1449for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1450 $drivename_hash->{"ide$i"} = 1;
1451 $confdesc->{"ide$i"} = $idedesc;
1452}
1453
cdb0931f
DA
1454for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1455 $drivename_hash->{"sata$i"} = 1;
1456 $confdesc->{"sata$i"} = $satadesc;
1457}
1458
1e3baf05
DM
1459for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1460 $drivename_hash->{"scsi$i"} = 1;
1461 $confdesc->{"scsi$i"} = $scsidesc ;
1462}
1463
1464for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1465 $drivename_hash->{"virtio$i"} = 1;
1466 $confdesc->{"virtio$i"} = $virtiodesc;
1467}
1468
6470743f
DC
1469$drivename_hash->{efidisk0} = 1;
1470$confdesc->{efidisk0} = $efidisk_desc;
1471
1e3baf05
DM
1472for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1473 $confdesc->{"usb$i"} = $usbdesc;
1474}
1475
1476my $unuseddesc = {
1477 optional => 1,
1478 type => 'string', format => 'pve-volume-id',
52261945 1479 description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1e3baf05
DM
1480};
1481
1482for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1483 $confdesc->{"unused$i"} = $unuseddesc;
1484}
1485
1486my $kvm_api_version = 0;
1487
1488sub kvm_version {
1e3baf05
DM
1489 return $kvm_api_version if $kvm_api_version;
1490
646f2df4
WB
1491 open my $fh, '<', '/dev/kvm'
1492 or return undef;
1e3baf05 1493
646f2df4
WB
1494 # 0xae00 => KVM_GET_API_VERSION
1495 $kvm_api_version = ioctl($fh, 0xae00, 0);
1e3baf05 1496
646f2df4 1497 return $kvm_api_version;
1e3baf05
DM
1498}
1499
1476b99f
DC
1500my $kvm_user_version = {};
1501my $kvm_mtime = {};
1e3baf05
DM
1502
1503sub kvm_user_version {
1476b99f 1504 my ($binary) = @_;
1e3baf05 1505
1476b99f
DC
1506 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1507 my $st = stat($binary);
1e3baf05 1508
1476b99f
DC
1509 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1510 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1511 $cachedmtime == $st->mtime;
1512
1513 $kvm_user_version->{$binary} = 'unknown';
1514 $kvm_mtime->{$binary} = $st->mtime;
1e3baf05 1515
09b11429
TL
1516 my $code = sub {
1517 my $line = shift;
1518 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1476b99f 1519 $kvm_user_version->{$binary} = $2;
09b11429
TL
1520 }
1521 };
19672434 1522
1476b99f 1523 eval { run_command([$binary, '--version'], outfunc => $code); };
09b11429 1524 warn $@ if $@;
1e3baf05 1525
1476b99f 1526 return $kvm_user_version->{$binary};
1e3baf05
DM
1527
1528}
1529
db70021b
TL
1530sub kernel_has_vhost_net {
1531 return -c '/dev/vhost-net';
1532}
1e3baf05 1533
74479ee9 1534sub valid_drive_names {
1e3baf05 1535 # order is important - used to autoselect boot disk
19672434 1536 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1e3baf05 1537 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
cdb0931f 1538 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
6470743f
DC
1539 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1540 'efidisk0');
1e3baf05
DM
1541}
1542
74479ee9 1543sub is_valid_drivename {
1e3baf05
DM
1544 my $dev = shift;
1545
6b64503e 1546 return defined($drivename_hash->{$dev});
1e3baf05
DM
1547}
1548
1549sub option_exists {
1550 my $key = shift;
1551 return defined($confdesc->{$key});
19672434 1552}
1e3baf05
DM
1553
1554sub nic_models {
1555 return $nic_model_list;
1556}
1557
1558sub os_list_description {
1559
1560 return {
1561 other => 'Other',
1562 wxp => 'Windows XP',
1563 w2k => 'Windows 2000',
1564 w2k3 =>, 'Windows 2003',
1565 w2k8 => 'Windows 2008',
1566 wvista => 'Windows Vista',
1567 win7 => 'Windows 7',
a70ebde3 1568 win8 => 'Windows 8/2012',
0cb9971e 1569 win10 => 'Windows 10/2016',
1e3baf05
DM
1570 l24 => 'Linux 2.4',
1571 l26 => 'Linux 2.6',
19672434 1572 };
1e3baf05
DM
1573}
1574
1e3baf05
DM
1575my $cdrom_path;
1576
1577sub get_cdrom_path {
1578
1579 return $cdrom_path if $cdrom_path;
1580
1581 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1582 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1583 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1584}
1585
1586sub get_iso_path {
1587 my ($storecfg, $vmid, $cdrom) = @_;
1588
1589 if ($cdrom eq 'cdrom') {
1590 return get_cdrom_path();
1591 } elsif ($cdrom eq 'none') {
1592 return '';
1593 } elsif ($cdrom =~ m|^/|) {
1594 return $cdrom;
1595 } else {
6b64503e 1596 return PVE::Storage::path($storecfg, $cdrom);
1e3baf05
DM
1597 }
1598}
1599
1600# try to convert old style file names to volume IDs
1601sub filename_to_volume_id {
1602 my ($vmid, $file, $media) = @_;
1603
0c9a7596 1604 if (!($file eq 'none' || $file eq 'cdrom' ||
1e3baf05 1605 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
19672434 1606
1e3baf05 1607 return undef if $file =~ m|/|;
19672434 1608
1e3baf05
DM
1609 if ($media && $media eq 'cdrom') {
1610 $file = "local:iso/$file";
1611 } else {
1612 $file = "local:$vmid/$file";
1613 }
1614 }
1615
1616 return $file;
1617}
1618
1619sub verify_media_type {
1620 my ($opt, $vtype, $media) = @_;
1621
1622 return if !$media;
1623
1624 my $etype;
1625 if ($media eq 'disk') {
a125592c 1626 $etype = 'images';
1e3baf05
DM
1627 } elsif ($media eq 'cdrom') {
1628 $etype = 'iso';
1629 } else {
1630 die "internal error";
1631 }
1632
1633 return if ($vtype eq $etype);
19672434 1634
1e3baf05
DM
1635 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1636}
1637
1638sub cleanup_drive_path {
1639 my ($opt, $storecfg, $drive) = @_;
1640
1641 # try to convert filesystem paths to volume IDs
1642
1643 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1644 ($drive->{file} !~ m|^/dev/.+|) &&
1645 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
19672434 1646 ($drive->{file} !~ m/^\d+$/)) {
1e3baf05
DM
1647 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1648 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1649 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1650 verify_media_type($opt, $vtype, $drive->{media});
1651 $drive->{file} = $volid;
1652 }
1653
1654 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1655}
1656
b3c2bdd1
DM
1657sub parse_hotplug_features {
1658 my ($data) = @_;
1659
1660 my $res = {};
1661
1662 return $res if $data eq '0';
a1b7d579 1663
b3c2bdd1
DM
1664 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1665
45827685 1666 foreach my $feature (PVE::Tools::split_list($data)) {
b3c2bdd1
DM
1667 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1668 $res->{$1} = 1;
1669 } else {
596a0a20 1670 die "invalid hotplug feature '$feature'\n";
b3c2bdd1
DM
1671 }
1672 }
1673 return $res;
1674}
1675
1676PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1677sub pve_verify_hotplug_features {
1678 my ($value, $noerr) = @_;
1679
1680 return $value if parse_hotplug_features($value);
1681
1682 return undef if $noerr;
1683
1684 die "unable to parse hotplug option\n";
1685}
1686
1e3baf05
DM
1687# ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1688# [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
036e0e2b 1689# [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
6e47c3b4
WB
1690# [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1691# [,iothread=on][,serial=serial][,model=model]
1e3baf05
DM
1692
1693sub parse_drive {
1694 my ($key, $data) = @_;
1695
0541eeb8 1696 my ($interface, $index);
19672434 1697
0541eeb8
WB
1698 if ($key =~ m/^([^\d]+)(\d+)$/) {
1699 $interface = $1;
1700 $index = $2;
1e3baf05
DM
1701 } else {
1702 return undef;
1703 }
1704
0541eeb8
WB
1705 my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1706 : $confdesc->{$key}->{format};
1707 if (!$desc) {
1708 warn "invalid drive key: $key\n";
1709 return undef;
1710 }
1711 my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1712 return undef if !$res;
1713 $res->{interface} = $interface;
1714 $res->{index} = $index;
1715
1716 my $error = 0;
1717 foreach my $opt (qw(bps bps_rd bps_wr)) {
1718 if (my $bps = defined(delete $res->{$opt})) {
1719 if (defined($res->{"m$opt"})) {
1720 warn "both $opt and m$opt specified\n";
1721 ++$error;
1722 next;
1e3baf05 1723 }
0541eeb8 1724 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1e3baf05
DM
1725 }
1726 }
9196a8ec
WB
1727
1728 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1729 for my $requirement (
fb8e95a2
WB
1730 [mbps_max => 'mbps'],
1731 [mbps_rd_max => 'mbps_rd'],
1732 [mbps_wr_max => 'mbps_wr'],
1733 [miops_max => 'miops'],
1734 [miops_rd_max => 'miops_rd'],
1735 [miops_wr_max => 'miops_wr'],
9196a8ec
WB
1736 [bps_max_length => 'mbps_max'],
1737 [bps_rd_max_length => 'mbps_rd_max'],
1738 [bps_wr_max_length => 'mbps_wr_max'],
1739 [iops_max_length => 'iops_max'],
1740 [iops_rd_max_length => 'iops_rd_max'],
1741 [iops_wr_max_length => 'iops_wr_max']) {
1742 my ($option, $requires) = @$requirement;
1743 if ($res->{$option} && !$res->{$requires}) {
1744 warn "$option requires $requires\n";
1745 ++$error;
1746 }
1747 }
1748
0541eeb8 1749 return undef if $error;
be190583 1750
9bf371a6
DM
1751 return undef if $res->{mbps_rd} && $res->{mbps};
1752 return undef if $res->{mbps_wr} && $res->{mbps};
affd2f88
AD
1753 return undef if $res->{iops_rd} && $res->{iops};
1754 return undef if $res->{iops_wr} && $res->{iops};
74edd76b 1755
1e3baf05
DM
1756 if ($res->{media} && ($res->{media} eq 'cdrom')) {
1757 return undef if $res->{snapshot} || $res->{trans} || $res->{format};
19672434 1758 return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1e3baf05
DM
1759 return undef if $res->{interface} eq 'virtio';
1760 }
1761
0541eeb8
WB
1762 if (my $size = $res->{size}) {
1763 return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1e3baf05
DM
1764 }
1765
1766 return $res;
1767}
1768
1e3baf05
DM
1769sub print_drive {
1770 my ($vmid, $drive) = @_;
0541eeb8
WB
1771 my $data = { %$drive };
1772 delete $data->{$_} for qw(index interface);
1773 return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1e3baf05
DM
1774}
1775
28ef82d3
DM
1776sub scsi_inquiry {
1777 my($fh, $noerr) = @_;
1778
1779 my $SG_IO = 0x2285;
1780 my $SG_GET_VERSION_NUM = 0x2282;
1781
1782 my $versionbuf = "\x00" x 8;
1783 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1784 if (!$ret) {
1785 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1786 return undef;
1787 }
97d62eb7 1788 my $version = unpack("I", $versionbuf);
28ef82d3
DM
1789 if ($version < 30000) {
1790 die "scsi generic interface too old\n" if !$noerr;
1791 return undef;
1792 }
97d62eb7 1793
28ef82d3
DM
1794 my $buf = "\x00" x 36;
1795 my $sensebuf = "\x00" x 8;
f334aa3e 1796 my $cmd = pack("C x3 C x1", 0x12, 36);
97d62eb7 1797
28ef82d3
DM
1798 # see /usr/include/scsi/sg.h
1799 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1800
97d62eb7
DM
1801 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1802 length($sensebuf), 0, length($buf), $buf,
28ef82d3
DM
1803 $cmd, $sensebuf, 6000);
1804
1805 $ret = ioctl($fh, $SG_IO, $packet);
1806 if (!$ret) {
1807 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1808 return undef;
1809 }
97d62eb7 1810
28ef82d3
DM
1811 my @res = unpack($sg_io_hdr_t, $packet);
1812 if ($res[17] || $res[18]) {
1813 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1814 return undef;
1815 }
1816
1817 my $res = {};
09984754 1818 (my $byte0, my $byte1, $res->{vendor},
28ef82d3
DM
1819 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1820
09984754
DM
1821 $res->{removable} = $byte1 & 128 ? 1 : 0;
1822 $res->{type} = $byte0 & 31;
1823
28ef82d3
DM
1824 return $res;
1825}
1826
1827sub path_is_scsi {
1828 my ($path) = @_;
1829
1830 my $fh = IO::File->new("+<$path") || return undef;
1831 my $res = scsi_inquiry($fh, 1);
1832 close($fh);
1833
1834 return $res;
1835}
1836
db656e5f
DM
1837sub machine_type_is_q35 {
1838 my ($conf) = @_;
b467f79a 1839
db656e5f
DM
1840 return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1841}
1842
1843sub print_tabletdevice_full {
d559309f 1844 my ($conf, $arch) = @_;
b467f79a 1845
db656e5f
DM
1846 my $q35 = machine_type_is_q35($conf);
1847
1848 # we use uhci for old VMs because tablet driver was buggy in older qemu
d559309f
WB
1849 my $usbbus;
1850 if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1851 $usbbus = 'ehci';
1852 } else {
1853 $usbbus = 'uhci';
1854 }
b467f79a 1855
db656e5f
DM
1856 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1857}
1858
d559309f
WB
1859sub print_keyboarddevice_full {
1860 my ($conf, $arch, $machine) = @_;
1861
1862 return undef if $arch ne 'aarch64';
1863
1864 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1865}
1866
ca916ecc 1867sub print_drivedevice_full {
d559309f 1868 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
ca916ecc
DA
1869
1870 my $device = '';
1871 my $maxdev = 0;
19672434 1872
ca916ecc 1873 if ($drive->{interface} eq 'virtio') {
d559309f 1874 my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
2ed36a41 1875 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
51f492cd 1876 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
2ed36a41 1877 } elsif ($drive->{interface} eq 'scsi') {
6731a4cf 1878
ee034f5c 1879 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
2ed36a41
DM
1880 my $unit = $drive->{index} % $maxdev;
1881 my $devicetype = 'hd';
69bcf246
WB
1882 my $path = '';
1883 if (drive_is_cdrom($drive)) {
1884 $devicetype = 'cd';
29b19529 1885 } else {
69bcf246
WB
1886 if ($drive->{file} =~ m|^/|) {
1887 $path = $drive->{file};
1888 if (my $info = path_is_scsi($path)) {
8e3c33ab 1889 if ($info->{type} == 0 && $drive->{scsiblock}) {
69bcf246
WB
1890 $devicetype = 'block';
1891 } elsif ($info->{type} == 1) { # tape
1892 $devicetype = 'generic';
1893 }
1894 }
1895 } else {
1896 $path = PVE::Storage::path($storecfg, $drive->{file});
1897 }
1898
1899 if($path =~ m/^iscsi\:\/\//){
1900 $devicetype = 'generic';
1901 }
1902 }
1903
1904 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1905 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1906 } else {
1907 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908 }
cdd20088 1909
6c875f9f
NC
1910 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1911 $device .= ",rotation_rate=1";
1912 }
e741c516 1913 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
6c875f9f
NC
1914
1915 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1916 my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
2ed36a41
DM
1917 my $controller = int($drive->{index} / $maxdev);
1918 my $unit = $drive->{index} % $maxdev;
1919 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1920
6c875f9f
NC
1921 $device = "ide-$devicetype";
1922 if ($drive->{interface} eq 'ide') {
1923 $device .= ",bus=ide.$controller,unit=$unit";
1924 } else {
1925 $device .= ",bus=ahci$controller.$unit";
1926 }
1927 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1928
1929 if ($devicetype eq 'hd') {
1930 if (my $model = $drive->{model}) {
1931 $model = URI::Escape::uri_unescape($model);
1932 $device .= ",model=$model";
1933 }
1934 if ($drive->{ssd}) {
1935 $device .= ",rotation_rate=1";
1936 }
0f2812c2 1937 }
e741c516 1938 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
2ed36a41
DM
1939 } elsif ($drive->{interface} eq 'usb') {
1940 die "implement me";
1941 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1942 } else {
1943 die "unsupported interface type";
ca916ecc
DA
1944 }
1945
3b408e82
DM
1946 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1947
a70e7e6c
TL
1948 if (my $serial = $drive->{serial}) {
1949 $serial = URI::Escape::uri_unescape($serial);
1950 $device .= ",serial=$serial";
1951 }
1952
1953
ca916ecc
DA
1954 return $device;
1955}
1956
15b21acc 1957sub get_initiator_name {
46f58b5f 1958 my $initiator;
15b21acc 1959
46f58b5f
DM
1960 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1961 while (defined(my $line = <$fh>)) {
1962 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
15b21acc
MR
1963 $initiator = $1;
1964 last;
1965 }
46f58b5f
DM
1966 $fh->close();
1967
15b21acc
MR
1968 return $initiator;
1969}
1970
1e3baf05
DM
1971sub print_drive_full {
1972 my ($storecfg, $vmid, $drive) = @_;
1973
d81f0f09
DM
1974 my $path;
1975 my $volid = $drive->{file};
1976 my $format;
370b05e7 1977
d81f0f09
DM
1978 if (drive_is_cdrom($drive)) {
1979 $path = get_iso_path($storecfg, $vmid, $volid);
1980 } else {
1981 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1982 if ($storeid) {
1983 $path = PVE::Storage::path($storecfg, $volid);
1984 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1985 $format = qemu_img_format($scfg, $volname);
1986 } else {
1987 $path = $volid;
5b61bff2 1988 $format = "raw";
d81f0f09
DM
1989 }
1990 }
1991
1e3baf05 1992 my $opts = '';
8a267708 1993 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1e3baf05 1994 foreach my $o (@qemu_drive_options) {
5fc74861 1995 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
19672434 1996 }
8a267708
DC
1997
1998 # snapshot only accepts on|off
1999 if (defined($drive->{snapshot})) {
2000 my $v = $drive->{snapshot} ? 'on' : 'off';
2001 $opts .= ",snapshot=$v";
2002 }
2003
fb8e95a2
WB
2004 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2005 my ($dir, $qmpname) = @$type;
2006 if (my $v = $drive->{"mbps$dir"}) {
2007 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2008 }
2009 if (my $v = $drive->{"mbps${dir}_max"}) {
2010 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2011 }
2012 if (my $v = $drive->{"bps${dir}_max_length"}) {
2013 $opts .= ",throttling.bps$qmpname-max-length=$v";
2014 }
2015 if (my $v = $drive->{"iops${dir}"}) {
2016 $opts .= ",throttling.iops$qmpname=$v";
2017 }
2018 if (my $v = $drive->{"iops${dir}_max"}) {
8aca1654 2019 $opts .= ",throttling.iops$qmpname-max=$v";
fb8e95a2
WB
2020 }
2021 if (my $v = $drive->{"iops${dir}_max_length"}) {
8aca1654 2022 $opts .= ",throttling.iops$qmpname-max-length=$v";
fb8e95a2
WB
2023 }
2024 }
2025
d81f0f09
DM
2026 $opts .= ",format=$format" if $format && !$drive->{format};
2027
b2ee900e
WB
2028 my $cache_direct = 0;
2029
2030 if (my $cache = $drive->{cache}) {
2031 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2032 } elsif (!drive_is_cdrom($drive)) {
2033 $opts .= ",cache=none";
2034 $cache_direct = 1;
2035 }
2036
2037 # aio native works only with O_DIRECT
2038 if (!$drive->{aio}) {
2039 if($cache_direct) {
2040 $opts .= ",aio=native";
2041 } else {
2042 $opts .= ",aio=threads";
2043 }
2044 }
11490cf2 2045
6e47c3b4
WB
2046 if (!drive_is_cdrom($drive)) {
2047 my $detectzeroes;
7d4e30f3 2048 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
6e47c3b4
WB
2049 $detectzeroes = 'off';
2050 } elsif ($drive->{discard}) {
2051 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2052 } else {
2053 # This used to be our default with discard not being specified:
2054 $detectzeroes = 'on';
2055 }
2056 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2057 }
f1e05305 2058
1e3baf05
DM
2059 my $pathinfo = $path ? "file=$path," : '';
2060
3ebfcc86 2061 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1e3baf05
DM
2062}
2063
cc4d6182 2064sub print_netdevice_full {
d559309f 2065 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
cc4d6182
DA
2066
2067 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2068
2069 my $device = $net->{model};
2070 if ($net->{model} eq 'virtio') {
2071 $device = 'virtio-net-pci';
2072 };
2073
d559309f 2074 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
5e2068d2 2075 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
a9410357
AD
2076 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2077 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2078 my $vectors = $net->{queues} * 2 + 2;
2079 $tmpstr .= ",vectors=$vectors,mq=on";
2080 }
cc4d6182 2081 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
ba9e1000
DM
2082
2083 if ($use_old_bios_files) {
2084 my $romfile;
2085 if ($device eq 'virtio-net-pci') {
2086 $romfile = 'pxe-virtio.rom';
2087 } elsif ($device eq 'e1000') {
2088 $romfile = 'pxe-e1000.rom';
2089 } elsif ($device eq 'ne2k') {
2090 $romfile = 'pxe-ne2k_pci.rom';
2091 } elsif ($device eq 'pcnet') {
2092 $romfile = 'pxe-pcnet.rom';
2093 } elsif ($device eq 'rtl8139') {
2094 $romfile = 'pxe-rtl8139.rom';
2095 }
2096 $tmpstr .= ",romfile=$romfile" if $romfile;
2097 }
2098
cc4d6182
DA
2099 return $tmpstr;
2100}
2101
2102sub print_netdev_full {
d559309f 2103 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
cc4d6182
DA
2104
2105 my $i = '';
2106 if ($netid =~ m/^net(\d+)$/) {
2107 $i = int($1);
2108 }
2109
2110 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2111
2112 my $ifname = "tap${vmid}i$i";
2113
2114 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2115 die "interface name '$ifname' is too long (max 15 character)\n"
2116 if length($ifname) >= 16;
2117
2118 my $vhostparam = '';
6f0cb675 2119 if (is_native($arch)) {
db70021b 2120 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
6f0cb675 2121 }
cc4d6182
DA
2122
2123 my $vmname = $conf->{name} || "vm$vmid";
2124
a9410357 2125 my $netdev = "";
208ba94e 2126 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
a9410357 2127
cc4d6182 2128 if ($net->{bridge}) {
208ba94e 2129 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
cc4d6182 2130 } else {
a9410357 2131 $netdev = "type=user,id=$netid,hostname=$vmname";
cc4d6182 2132 }
a9410357
AD
2133
2134 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2135
2136 return $netdev;
cc4d6182 2137}
1e3baf05 2138
0efb537e
AD
2139
2140sub print_cpu_device {
2141 my ($conf, $id) = @_;
2142
74c02ef7
PA
2143 my $kvm = $conf->{kvm} // 1;
2144 my $cpu = $kvm ? "kvm64" : "qemu64";
0efb537e
AD
2145 if (my $cputype = $conf->{cpu}) {
2146 my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2147 or die "Cannot parse cpu description: $cputype\n";
2148 $cpu = $cpuconf->{cputype};
2149 }
2150
0efb537e
AD
2151 my $cores = $conf->{cores} || 1;
2152
2153 my $current_core = ($id - 1) % $cores;
7032e08c 2154 my $current_socket = int(($id - 1 - $current_core)/$cores);
0efb537e
AD
2155
2156 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2157}
2158
55655ebc
DC
2159my $vga_map = {
2160 'cirrus' => 'cirrus-vga',
2161 'std' => 'VGA',
2162 'vmware' => 'vmware-svga',
2163 'virtio' => 'virtio-vga',
2164};
2165
2166sub print_vga_device {
d559309f 2167 my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
55655ebc
DC
2168
2169 my $type = $vga_map->{$vga->{type}};
86c9fafe 2170 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
d559309f
WB
2171 $type = 'virtio-gpu';
2172 }
55655ebc
DC
2173 my $vgamem_mb = $vga->{memory};
2174 if ($qxlnum) {
2175 $type = $id ? 'qxl' : 'qxl-vga';
2176 }
2177 die "no devicetype for $vga->{type}\n" if !$type;
2178
2179 my $memory = "";
2180 if ($vgamem_mb) {
2181 if ($vga->{type} eq 'virtio') {
2182 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2183 $memory = ",max_hostmem=$bytes";
2184 } elsif ($qxlnum) {
2185 # from https://www.spice-space.org/multiple-monitors.html
2186 $memory = ",vgamem_mb=$vga->{memory}";
2187 my $ram = $vgamem_mb * 4;
2188 my $vram = $vgamem_mb * 2;
2189 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2190 } else {
2191 $memory = ",vgamem_mb=$vga->{memory}";
2192 }
2193 } elsif ($qxlnum && $id) {
2194 $memory = ",ram_size=67108864,vram_size=33554432";
2195 }
2196
2197 my $q35 = machine_type_is_q35($conf);
2198 my $vgaid = "vga" . ($id // '');
2199 my $pciaddr;
daadd5a4 2200
55655ebc 2201 if ($q35 && $vgaid eq 'vga') {
daadd5a4 2202 # the first display uses pcie.0 bus on q35 machines
d559309f 2203 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
55655ebc 2204 } else {
d559309f 2205 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
55655ebc
DC
2206 }
2207
2208 return "$type,id=${vgaid}${memory}${pciaddr}";
2209}
2210
9ed7a77c
WB
2211sub drive_is_cloudinit {
2212 my ($drive) = @_;
2213 return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2214}
2215
1e3baf05 2216sub drive_is_cdrom {
9c52f5ed
WB
2217 my ($drive, $exclude_cloudinit) = @_;
2218
9ed7a77c 2219 return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1e3baf05
DM
2220
2221 return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2222
2223}
2224
ffc0d8c7
WB
2225sub parse_number_sets {
2226 my ($set) = @_;
2227 my $res = [];
2228 foreach my $part (split(/;/, $set)) {
2229 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2230 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2231 push @$res, [ $1, $2 ];
2ed5d572 2232 } else {
ffc0d8c7 2233 die "invalid range: $part\n";
2ed5d572
AD
2234 }
2235 }
ffc0d8c7
WB
2236 return $res;
2237}
2ed5d572 2238
ffc0d8c7
WB
2239sub parse_numa {
2240 my ($data) = @_;
2241
2242 my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2243 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2244 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2ed5d572
AD
2245 return $res;
2246}
2247
040b06b7
DA
2248sub parse_hostpci {
2249 my ($value) = @_;
2250
2251 return undef if !$value;
2252
1f4f447b 2253 my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
0cea6a01 2254
1f4f447b
WB
2255 my @idlist = split(/;/, $res->{host});
2256 delete $res->{host};
2257 foreach my $id (@idlist) {
2fd24788
DC
2258 if ($id =~ m/\./) { # full id 00:00.1
2259 push @{$res->{pciid}}, {
2260 id => $id,
2261 };
2262 } else { # partial id 00:00
2263 $res->{pciid} = PVE::SysFSTools::lspci($id);
0cea6a01 2264 }
040b06b7 2265 }
040b06b7
DA
2266 return $res;
2267}
2268
1e3baf05
DM
2269# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2270sub parse_net {
2271 my ($data) = @_;
2272
cd9c34d1
WB
2273 my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2274 if ($@) {
2275 warn $@;
2276 return undef;
1e3baf05 2277 }
b5b99790
WB
2278 if (!defined($res->{macaddr})) {
2279 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2280 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2281 }
0c9a7596
AD
2282 return $res;
2283}
2284
2285# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2286sub parse_ipconfig {
2287 my ($data) = @_;
2288
2289 my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2290 if ($@) {
2291 warn $@;
2292 return undef;
2293 }
2294
2295 if ($res->{gw} && !$res->{ip}) {
2296 warn 'gateway specified without specifying an IP address';
2297 return undef;
2298 }
2299 if ($res->{gw6} && !$res->{ip6}) {
2300 warn 'IPv6 gateway specified without specifying an IPv6 address';
2301 return undef;
2302 }
2303 if ($res->{gw} && $res->{ip} eq 'dhcp') {
2304 warn 'gateway specified together with DHCP';
2305 return undef;
2306 }
2307 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2308 # gw6 + auto/dhcp
2309 warn "IPv6 gateway specified together with $res->{ip6} address";
2310 return undef;
2311 }
2312
2313 if (!$res->{ip} && !$res->{ip6}) {
2314 return { ip => 'dhcp', ip6 => 'dhcp' };
2315 }
2316
1e3baf05
DM
2317 return $res;
2318}
2319
2320sub print_net {
2321 my $net = shift;
2322
cd9c34d1 2323 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1e3baf05
DM
2324}
2325
2326sub add_random_macs {
2327 my ($settings) = @_;
2328
2329 foreach my $opt (keys %$settings) {
2330 next if $opt !~ m/^net(\d+)$/;
2331 my $net = parse_net($settings->{$opt});
2332 next if !$net;
2333 $settings->{$opt} = print_net($net);
2334 }
2335}
2336
055d554d
DM
2337sub vm_is_volid_owner {
2338 my ($storecfg, $vmid, $volid) = @_;
2339
2340 if ($volid !~ m|^/|) {
2341 my ($path, $owner);
2342 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2343 if ($owner && ($owner == $vmid)) {
2344 return 1;
2345 }
2346 }
2347
2348 return undef;
2349}
2350
3dc38fbb
WB
2351sub split_flagged_list {
2352 my $text = shift || '';
2353 $text =~ s/[,;]/ /g;
2354 $text =~ s/^\s+//;
2355 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2356}
2357
2358sub join_flagged_list {
2359 my ($how, $lst) = @_;
2360 join $how, map { $lst->{$_} . $_ } keys %$lst;
2361}
2362
055d554d 2363sub vmconfig_delete_pending_option {
3dc38fbb 2364 my ($conf, $key, $force) = @_;
055d554d
DM
2365
2366 delete $conf->{pending}->{$key};
3dc38fbb
WB
2367 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2368 $pending_delete_hash->{$key} = $force ? '!' : '';
2369 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
055d554d
DM
2370}
2371
2372sub vmconfig_undelete_pending_option {
2373 my ($conf, $key) = @_;
2374
3dc38fbb 2375 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
055d554d
DM
2376 delete $pending_delete_hash->{$key};
2377
3dc38fbb
WB
2378 if (%$pending_delete_hash) {
2379 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
055d554d
DM
2380 } else {
2381 delete $conf->{pending}->{delete};
2382 }
2383}
2384
2385sub vmconfig_register_unused_drive {
2386 my ($storecfg, $vmid, $conf, $drive) = @_;
2387
2d9ddec5
WB
2388 if (drive_is_cloudinit($drive)) {
2389 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2390 warn $@ if $@;
2391 } elsif (!drive_is_cdrom($drive)) {
055d554d
DM
2392 my $volid = $drive->{file};
2393 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
8793d495 2394 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
055d554d
DM
2395 }
2396 }
2397}
2398
c750e90a
DM
2399sub vmconfig_cleanup_pending {
2400 my ($conf) = @_;
2401
2402 # remove pending changes when nothing changed
2403 my $changes;
2404 foreach my $opt (keys %{$conf->{pending}}) {
2405 if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq $conf->{$opt})) {
2406 $changes = 1;
2407 delete $conf->{pending}->{$opt};
2408 }
2409 }
2410
3dc38fbb 2411 my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
c750e90a 2412 my $pending_delete_hash = {};
3dc38fbb 2413 while (my ($opt, $force) = each %$current_delete_hash) {
c750e90a 2414 if (defined($conf->{$opt})) {
3dc38fbb 2415 $pending_delete_hash->{$opt} = $force;
c750e90a
DM
2416 } else {
2417 $changes = 1;
2418 }
2419 }
2420
3dc38fbb
WB
2421 if (%$pending_delete_hash) {
2422 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
c750e90a
DM
2423 } else {
2424 delete $conf->{pending}->{delete};
2425 }
2426
2427 return $changes;
2428}
2429
1f30ac3a 2430# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
ff6ffe20 2431my $smbios1_fmt = {
bd27e851
WB
2432 uuid => {
2433 type => 'string',
2434 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2435 format_description => 'UUID',
52261945 2436 description => "Set SMBIOS1 UUID.",
bd27e851
WB
2437 optional => 1,
2438 },
2439 version => {
2440 type => 'string',
1f30ac3a
CE
2441 pattern => '[A-Za-z0-9+\/]+={0,2}',
2442 format_description => 'Base64 encoded string',
52261945 2443 description => "Set SMBIOS1 version.",
bd27e851
WB
2444 optional => 1,
2445 },
2446 serial => {
2447 type => 'string',
1f30ac3a
CE
2448 pattern => '[A-Za-z0-9+\/]+={0,2}',
2449 format_description => 'Base64 encoded string',
52261945 2450 description => "Set SMBIOS1 serial number.",
bd27e851
WB
2451 optional => 1,
2452 },
2453 manufacturer => {
2454 type => 'string',
1f30ac3a
CE
2455 pattern => '[A-Za-z0-9+\/]+={0,2}',
2456 format_description => 'Base64 encoded string',
52261945 2457 description => "Set SMBIOS1 manufacturer.",
bd27e851
WB
2458 optional => 1,
2459 },
2460 product => {
2461 type => 'string',
1f30ac3a
CE
2462 pattern => '[A-Za-z0-9+\/]+={0,2}',
2463 format_description => 'Base64 encoded string',
52261945 2464 description => "Set SMBIOS1 product ID.",
bd27e851
WB
2465 optional => 1,
2466 },
2467 sku => {
2468 type => 'string',
1f30ac3a
CE
2469 pattern => '[A-Za-z0-9+\/]+={0,2}',
2470 format_description => 'Base64 encoded string',
52261945 2471 description => "Set SMBIOS1 SKU string.",
bd27e851
WB
2472 optional => 1,
2473 },
2474 family => {
2475 type => 'string',
1f30ac3a
CE
2476 pattern => '[A-Za-z0-9+\/]+={0,2}',
2477 format_description => 'Base64 encoded string',
52261945 2478 description => "Set SMBIOS1 family string.",
bd27e851
WB
2479 optional => 1,
2480 },
1f30ac3a
CE
2481 base64 => {
2482 type => 'boolean',
2483 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2484 optional => 1,
2485 },
2796e7d5
DM
2486};
2487
2796e7d5
DM
2488sub parse_smbios1 {
2489 my ($data) = @_;
2490
ff6ffe20 2491 my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
bd27e851 2492 warn $@ if $@;
2796e7d5
DM
2493 return $res;
2494}
2495
cd11416f
DM
2496sub print_smbios1 {
2497 my ($smbios1) = @_;
ff6ffe20 2498 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
cd11416f
DM
2499}
2500
ff6ffe20 2501PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2796e7d5 2502
1e3baf05
DM
2503PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2504sub verify_bootdisk {
2505 my ($value, $noerr) = @_;
2506
74479ee9 2507 return $value if is_valid_drivename($value);
1e3baf05
DM
2508
2509 return undef if $noerr;
2510
2511 die "invalid boot disk '$value'\n";
2512}
2513
0ea9541d
DM
2514sub parse_watchdog {
2515 my ($value) = @_;
2516
2517 return undef if !$value;
2518
ec3582b5
WB
2519 my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2520 warn $@ if $@;
0ea9541d
DM
2521 return $res;
2522}
2523
9d66b397
SI
2524sub parse_guest_agent {
2525 my ($value) = @_;
2526
2527 return {} if !defined($value->{agent});
2528
2529 my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2530 warn $@ if $@;
2531
2532 # if the agent is disabled ignore the other potentially set properties
2533 return {} if !$res->{enabled};
2534 return $res;
2535}
2536
55655ebc
DC
2537sub parse_vga {
2538 my ($value) = @_;
2539
2540 return {} if !$value;
2541 my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2542 warn $@ if $@;
2543 return $res;
2544}
2545
1e3baf05
DM
2546PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2547sub verify_usb_device {
2548 my ($value, $noerr) = @_;
2549
2550 return $value if parse_usb_device($value);
2551
2552 return undef if $noerr;
19672434 2553
1e3baf05
DM
2554 die "unable to parse usb device\n";
2555}
2556
1e3baf05
DM
2557# add JSON properties for create and set function
2558sub json_config_properties {
2559 my $prop = shift;
2560
2561 foreach my $opt (keys %$confdesc) {
c6737ef1 2562 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1e3baf05
DM
2563 $prop->{$opt} = $confdesc->{$opt};
2564 }
2565
2566 return $prop;
2567}
2568
d41121fd
DM
2569# return copy of $confdesc_cloudinit to generate documentation
2570sub cloudinit_config_properties {
2571
2572 return dclone($confdesc_cloudinit);
2573}
2574
1e3baf05
DM
2575sub check_type {
2576 my ($key, $value) = @_;
2577
2578 die "unknown setting '$key'\n" if !$confdesc->{$key};
2579
2580 my $type = $confdesc->{$key}->{type};
2581
6b64503e 2582 if (!defined($value)) {
1e3baf05
DM
2583 die "got undefined value\n";
2584 }
2585
2586 if ($value =~ m/[\n\r]/) {
2587 die "property contains a line feed\n";
2588 }
2589
2590 if ($type eq 'boolean') {
19672434
DM
2591 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2592 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2593 die "type check ('boolean') failed - got '$value'\n";
1e3baf05
DM
2594 } elsif ($type eq 'integer') {
2595 return int($1) if $value =~ m/^(\d+)$/;
2596 die "type check ('integer') failed - got '$value'\n";
04432191
AD
2597 } elsif ($type eq 'number') {
2598 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2599 die "type check ('number') failed - got '$value'\n";
1e3baf05
DM
2600 } elsif ($type eq 'string') {
2601 if (my $fmt = $confdesc->{$key}->{format}) {
1e3baf05 2602 PVE::JSONSchema::check_format($fmt, $value);
19672434
DM
2603 return $value;
2604 }
1e3baf05 2605 $value =~ s/^\"(.*)\"$/$1/;
19672434 2606 return $value;
1e3baf05
DM
2607 } else {
2608 die "internal error"
2609 }
2610}
2611
1e3baf05
DM
2612sub touch_config {
2613 my ($vmid) = @_;
2614
ffda963f 2615 my $conf = PVE::QemuConfig->config_file($vmid);
1e3baf05
DM
2616 utime undef, undef, $conf;
2617}
2618
1e3baf05 2619sub destroy_vm {
15cc8784 2620 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1e3baf05 2621
ffda963f 2622 my $conffile = PVE::QemuConfig->config_file($vmid);
1e3baf05 2623
ffda963f 2624 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 2625
ffda963f 2626 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 2627
5e67a2d2
DC
2628 if ($conf->{template}) {
2629 # check if any base image is still used by a linked clone
2630 foreach_drive($conf, sub {
2631 my ($ds, $drive) = @_;
2632
2633 return if drive_is_cdrom($drive);
2634
2635 my $volid = $drive->{file};
2636
2637 return if !$volid || $volid =~ m|^/|;
2638
2639 die "base volume '$volid' is still in use by linked cloned\n"
2640 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2641
2642 });
2643 }
2644
19672434 2645 # only remove disks owned by this VM
1e3baf05
DM
2646 foreach_drive($conf, sub {
2647 my ($ds, $drive) = @_;
2648
9c52f5ed 2649 return if drive_is_cdrom($drive, 1);
1e3baf05
DM
2650
2651 my $volid = $drive->{file};
ed221350 2652
ff1a2432 2653 return if !$volid || $volid =~ m|^/|;
1e3baf05 2654
6b64503e 2655 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
ff1a2432 2656 return if !$path || !$owner || ($owner != $vmid);
1e3baf05 2657
31b52247
FG
2658 eval {
2659 PVE::Storage::vdisk_free($storecfg, $volid);
2660 };
2661 warn "Could not remove disk '$volid', check manually: $@" if $@;
2662
1e3baf05 2663 });
19672434 2664
a6af7b3e 2665 if ($keep_empty_config) {
9c502e26 2666 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
a6af7b3e
DM
2667 } else {
2668 unlink $conffile;
2669 }
1e3baf05
DM
2670
2671 # also remove unused disk
2672 eval {
6b64503e 2673 my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1e3baf05
DM
2674
2675 eval {
6b64503e 2676 PVE::Storage::foreach_volid($dl, sub {
1e3baf05 2677 my ($volid, $sid, $volname, $d) = @_;
6b64503e 2678 PVE::Storage::vdisk_free($storecfg, $volid);
1e3baf05
DM
2679 });
2680 };
2681 warn $@ if $@;
2682
2683 };
2684 warn $@ if $@;
2685}
2686
1e3baf05
DM
2687sub parse_vm_config {
2688 my ($filename, $raw) = @_;
2689
2690 return undef if !defined($raw);
2691
554ac7e7 2692 my $res = {
fc1ddcdc 2693 digest => Digest::SHA::sha1_hex($raw),
0d18dcfc 2694 snapshots => {},
0d732d16 2695 pending => {},
554ac7e7 2696 };
1e3baf05 2697
19672434 2698 $filename =~ m|/qemu-server/(\d+)\.conf$|
1e3baf05
DM
2699 || die "got strange filename '$filename'";
2700
2701 my $vmid = $1;
2702
0d18dcfc 2703 my $conf = $res;
b0ec896e 2704 my $descr;
e297c490 2705 my $section = '';
0581fe4f 2706
0d18dcfc
DM
2707 my @lines = split(/\n/, $raw);
2708 foreach my $line (@lines) {
1e3baf05 2709 next if $line =~ m/^\s*$/;
be190583 2710
eab09f4e 2711 if ($line =~ m/^\[PENDING\]\s*$/i) {
e297c490 2712 $section = 'pending';
b0ec896e
DM
2713 if (defined($descr)) {
2714 $descr =~ s/\s+$//;
2715 $conf->{description} = $descr;
2716 }
2717 $descr = undef;
e297c490 2718 $conf = $res->{$section} = {};
eab09f4e
AD
2719 next;
2720
0d732d16 2721 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
e297c490 2722 $section = $1;
b0ec896e
DM
2723 if (defined($descr)) {
2724 $descr =~ s/\s+$//;
2725 $conf->{description} = $descr;
2726 }
2727 $descr = undef;
e297c490 2728 $conf = $res->{snapshots}->{$section} = {};
0d18dcfc
DM
2729 next;
2730 }
1e3baf05 2731
0581fe4f 2732 if ($line =~ m/^\#(.*)\s*$/) {
b0ec896e 2733 $descr = '' if !defined($descr);
0581fe4f
DM
2734 $descr .= PVE::Tools::decode_text($1) . "\n";
2735 next;
2736 }
2737
1e3baf05 2738 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
b0ec896e 2739 $descr = '' if !defined($descr);
0581fe4f 2740 $descr .= PVE::Tools::decode_text($2);
0d18dcfc
DM
2741 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2742 $conf->{snapstate} = $1;
1e3baf05
DM
2743 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2744 my $key = $1;
2745 my $value = $2;
0d18dcfc 2746 $conf->{$key} = $value;
ef824322 2747 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
e297c490 2748 my $value = $1;
ef824322
DM
2749 if ($section eq 'pending') {
2750 $conf->{delete} = $value; # we parse this later
2751 } else {
2752 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
eab09f4e 2753 }
15cf7698 2754 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
1e3baf05
DM
2755 my $key = $1;
2756 my $value = $2;
2757 eval { $value = check_type($key, $value); };
2758 if ($@) {
2759 warn "vm $vmid - unable to parse value of '$key' - $@";
2760 } else {
b799312f 2761 $key = 'ide2' if $key eq 'cdrom';
1e3baf05 2762 my $fmt = $confdesc->{$key}->{format};
b799312f 2763 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
1e3baf05
DM
2764 my $v = parse_drive($key, $value);
2765 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2766 $v->{file} = $volid;
6b64503e 2767 $value = print_drive($vmid, $v);
1e3baf05
DM
2768 } else {
2769 warn "vm $vmid - unable to parse value of '$key'\n";
2770 next;
2771 }
2772 }
2773
b799312f 2774 $conf->{$key} = $value;
1e3baf05
DM
2775 }
2776 }
2777 }
2778
b0ec896e
DM
2779 if (defined($descr)) {
2780 $descr =~ s/\s+$//;
2781 $conf->{description} = $descr;
2782 }
0d18dcfc 2783 delete $res->{snapstate}; # just to be sure
1e3baf05
DM
2784
2785 return $res;
2786}
2787
1858638f
DM
2788sub write_vm_config {
2789 my ($filename, $conf) = @_;
1e3baf05 2790
0d18dcfc
DM
2791 delete $conf->{snapstate}; # just to be sure
2792
1858638f
DM
2793 if ($conf->{cdrom}) {
2794 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2795 $conf->{ide2} = $conf->{cdrom};
2796 delete $conf->{cdrom};
2797 }
1e3baf05
DM
2798
2799 # we do not use 'smp' any longer
1858638f
DM
2800 if ($conf->{sockets}) {
2801 delete $conf->{smp};
2802 } elsif ($conf->{smp}) {
2803 $conf->{sockets} = $conf->{smp};
2804 delete $conf->{cores};
2805 delete $conf->{smp};
1e3baf05
DM
2806 }
2807
ee2f90b1 2808 my $used_volids = {};
0d18dcfc 2809
ee2f90b1 2810 my $cleanup_config = sub {
ef824322 2811 my ($cref, $pending, $snapname) = @_;
1858638f 2812
ee2f90b1
DM
2813 foreach my $key (keys %$cref) {
2814 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
ef824322 2815 $key eq 'snapstate' || $key eq 'pending';
ee2f90b1 2816 my $value = $cref->{$key};
ef824322
DM
2817 if ($key eq 'delete') {
2818 die "propertry 'delete' is only allowed in [PENDING]\n"
2819 if !$pending;
2820 # fixme: check syntax?
2821 next;
2822 }
ee2f90b1
DM
2823 eval { $value = check_type($key, $value); };
2824 die "unable to parse value of '$key' - $@" if $@;
1858638f 2825
ee2f90b1
DM
2826 $cref->{$key} = $value;
2827
74479ee9 2828 if (!$snapname && is_valid_drivename($key)) {
ed221350 2829 my $drive = parse_drive($key, $value);
ee2f90b1
DM
2830 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2831 }
1e3baf05 2832 }
ee2f90b1
DM
2833 };
2834
2835 &$cleanup_config($conf);
ef824322
DM
2836
2837 &$cleanup_config($conf->{pending}, 1);
2838
ee2f90b1 2839 foreach my $snapname (keys %{$conf->{snapshots}}) {
ef824322
DM
2840 die "internal error" if $snapname eq 'pending';
2841 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
1e3baf05
DM
2842 }
2843
1858638f
DM
2844 # remove 'unusedX' settings if we re-add a volume
2845 foreach my $key (keys %$conf) {
2846 my $value = $conf->{$key};
ee2f90b1 2847 if ($key =~ m/^unused/ && $used_volids->{$value}) {
1858638f 2848 delete $conf->{$key};
1e3baf05 2849 }
1858638f 2850 }
be190583 2851
0d18dcfc 2852 my $generate_raw_config = sub {
b0ec896e 2853 my ($conf, $pending) = @_;
0581fe4f 2854
0d18dcfc
DM
2855 my $raw = '';
2856
2857 # add description as comment to top of file
b0ec896e
DM
2858 if (defined(my $descr = $conf->{description})) {
2859 if ($descr) {
2860 foreach my $cl (split(/\n/, $descr)) {
2861 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2862 }
2863 } else {
2864 $raw .= "#\n" if $pending;
2865 }
0d18dcfc
DM
2866 }
2867
2868 foreach my $key (sort keys %$conf) {
ef824322 2869 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
0d18dcfc
DM
2870 $raw .= "$key: $conf->{$key}\n";
2871 }
2872 return $raw;
2873 };
0581fe4f 2874
0d18dcfc 2875 my $raw = &$generate_raw_config($conf);
ef824322
DM
2876
2877 if (scalar(keys %{$conf->{pending}})){
2878 $raw .= "\n[PENDING]\n";
b0ec896e 2879 $raw .= &$generate_raw_config($conf->{pending}, 1);
ef824322
DM
2880 }
2881
0d18dcfc
DM
2882 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2883 $raw .= "\n[$snapname]\n";
2884 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1858638f 2885 }
1e3baf05 2886
1858638f
DM
2887 return $raw;
2888}
1e3baf05 2889
19672434 2890sub load_defaults {
1e3baf05
DM
2891
2892 my $res = {};
2893
2894 # we use static defaults from our JSON schema configuration
2895 foreach my $key (keys %$confdesc) {
2896 if (defined(my $default = $confdesc->{$key}->{default})) {
2897 $res->{$key} = $default;
2898 }
2899 }
19672434 2900
1e3baf05
DM
2901 return $res;
2902}
2903
2904sub config_list {
2905 my $vmlist = PVE::Cluster::get_vmlist();
2906 my $res = {};
2907 return $res if !$vmlist || !$vmlist->{ids};
2908 my $ids = $vmlist->{ids};
2909
1e3baf05
DM
2910 foreach my $vmid (keys %$ids) {
2911 my $d = $ids->{$vmid};
2912 next if !$d->{node} || $d->{node} ne $nodename;
5ee957cc 2913 next if !$d->{type} || $d->{type} ne 'qemu';
1e3baf05
DM
2914 $res->{$vmid}->{exists} = 1;
2915 }
2916 return $res;
2917}
2918
64e13401
DM
2919# test if VM uses local resources (to prevent migration)
2920sub check_local_resources {
2921 my ($conf, $noerr) = @_;
2922
ca6abacf 2923 my @loc_res = ();
19672434 2924
ca6abacf
TM
2925 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2926 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
64e13401 2927
ca6abacf 2928 push @loc_res, "ivshmem" if $conf->{ivshmem};
6dbcb073 2929
0d29ab3b 2930 foreach my $k (keys %$conf) {
a9ce7583 2931 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
d44712fc
EK
2932 # sockets are safe: they will recreated be on the target side post-migrate
2933 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
ca6abacf 2934 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
64e13401
DM
2935 }
2936
ca6abacf 2937 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
64e13401 2938
ca6abacf 2939 return \@loc_res;
64e13401
DM
2940}
2941
719893a9 2942# check if used storages are available on all nodes (use by migrate)
47152e2e
DM
2943sub check_storage_availability {
2944 my ($storecfg, $conf, $node) = @_;
2945
2946 foreach_drive($conf, sub {
2947 my ($ds, $drive) = @_;
2948
2949 my $volid = $drive->{file};
2950 return if !$volid;
2951
2952 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2953 return if !$sid;
2954
2955 # check if storage is available on both nodes
2956 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2957 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2958 });
2959}
2960
719893a9
DM
2961# list nodes where all VM images are available (used by has_feature API)
2962sub shared_nodes {
2963 my ($conf, $storecfg) = @_;
2964
2965 my $nodelist = PVE::Cluster::get_nodelist();
2966 my $nodehash = { map { $_ => 1 } @$nodelist };
2967 my $nodename = PVE::INotify::nodename();
be190583 2968
719893a9
DM
2969 foreach_drive($conf, sub {
2970 my ($ds, $drive) = @_;
2971
2972 my $volid = $drive->{file};
2973 return if !$volid;
2974
2975 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2976 if ($storeid) {
2977 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978 if ($scfg->{disable}) {
2979 $nodehash = {};
2980 } elsif (my $avail = $scfg->{nodes}) {
2981 foreach my $node (keys %$nodehash) {
2982 delete $nodehash->{$node} if !$avail->{$node};
2983 }
2984 } elsif (!$scfg->{shared}) {
2985 foreach my $node (keys %$nodehash) {
2986 delete $nodehash->{$node} if $node ne $nodename
2987 }
2988 }
2989 }
2990 });
2991
2992 return $nodehash
2993}
2994
f25852c2
TM
2995sub check_local_storage_availability {
2996 my ($conf, $storecfg) = @_;
2997
2998 my $nodelist = PVE::Cluster::get_nodelist();
2999 my $nodehash = { map { $_ => {} } @$nodelist };
3000
3001 foreach_drive($conf, sub {
3002 my ($ds, $drive) = @_;
3003
3004 my $volid = $drive->{file};
3005 return if !$volid;
3006
3007 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3008 if ($storeid) {
3009 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3010
3011 if ($scfg->{disable}) {
3012 foreach my $node (keys %$nodehash) {
32075a2c 3013 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
f25852c2
TM
3014 }
3015 } elsif (my $avail = $scfg->{nodes}) {
3016 foreach my $node (keys %$nodehash) {
3017 if (!$avail->{$node}) {
32075a2c 3018 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
f25852c2
TM
3019 }
3020 }
3021 }
3022 }
3023 });
3024
32075a2c
TL
3025 foreach my $node (values %$nodehash) {
3026 if (my $unavail = $node->{unavailable_storages}) {
3027 $node->{unavailable_storages} = [ sort keys %$unavail ];
3028 }
3029 }
3030
f25852c2
TM
3031 return $nodehash
3032}
3033
1e3baf05
DM
3034sub check_cmdline {
3035 my ($pidfile, $pid) = @_;
3036
6b64503e
DM
3037 my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3038 if (defined($fh)) {
1e3baf05
DM
3039 my $line = <$fh>;
3040 $fh->close;
3041 return undef if !$line;
6b64503e 3042 my @param = split(/\0/, $line);
1e3baf05
DM
3043
3044 my $cmd = $param[0];
6908fd9b 3045 return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
1e3baf05
DM
3046
3047 for (my $i = 0; $i < scalar (@param); $i++) {
3048 my $p = $param[$i];
3049 next if !$p;
3050 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3051 my $p = $param[$i+1];
3052 return 1 if $p && ($p eq $pidfile);
3053 return undef;
3054 }
3055 }
3056 }
3057 return undef;
3058}
3059
3060sub check_running {
7e8dcf2c 3061 my ($vmid, $nocheck, $node) = @_;
1e3baf05 3062
ffda963f 3063 my $filename = PVE::QemuConfig->config_file($vmid, $node);
1e3baf05
DM
3064
3065 die "unable to find configuration file for VM $vmid - no such machine\n"
e6c3b671 3066 if !$nocheck && ! -f $filename;
1e3baf05 3067
e6c3b671 3068 my $pidfile = pidfile_name($vmid);
1e3baf05 3069
e6c3b671
DM
3070 if (my $fd = IO::File->new("<$pidfile")) {
3071 my $st = stat($fd);
1e3baf05 3072 my $line = <$fd>;
6b64503e 3073 close($fd);
1e3baf05
DM
3074
3075 my $mtime = $st->mtime;
3076 if ($mtime > time()) {
3077 warn "file '$filename' modified in future\n";
3078 }
3079
3080 if ($line =~ m/^(\d+)$/) {
3081 my $pid = $1;
e6c3b671
DM
3082 if (check_cmdline($pidfile, $pid)) {
3083 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3084 return $pid;
3085 }
3086 }
1e3baf05
DM
3087 }
3088 }
3089
3090 return undef;
3091}
3092
3093sub vzlist {
19672434 3094
1e3baf05
DM
3095 my $vzlist = config_list();
3096
6b64503e 3097 my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
1e3baf05 3098
19672434 3099 while (defined(my $de = $fd->read)) {
1e3baf05
DM
3100 next if $de !~ m/^(\d+)\.pid$/;
3101 my $vmid = $1;
6b64503e
DM
3102 next if !defined($vzlist->{$vmid});
3103 if (my $pid = check_running($vmid)) {
1e3baf05
DM
3104 $vzlist->{$vmid}->{pid} = $pid;
3105 }
3106 }
3107
3108 return $vzlist;
3109}
3110
1e3baf05
DM
3111sub disksize {
3112 my ($storecfg, $conf) = @_;
3113
3114 my $bootdisk = $conf->{bootdisk};
3115 return undef if !$bootdisk;
74479ee9 3116 return undef if !is_valid_drivename($bootdisk);
1e3baf05
DM
3117
3118 return undef if !$conf->{$bootdisk};
3119
3120 my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3121 return undef if !defined($drive);
3122
3123 return undef if drive_is_cdrom($drive);
3124
3125 my $volid = $drive->{file};
3126 return undef if !$volid;
3127
24afaca0 3128 return $drive->{size};
1e3baf05
DM
3129}
3130
b1a70cab
DM
3131our $vmstatus_return_properties = {
3132 vmid => get_standard_option('pve-vmid'),
3133 status => {
3134 description => "Qemu process status.",
3135 type => 'string',
3136 enum => ['stopped', 'running'],
3137 },
3138 maxmem => {
3139 description => "Maximum memory in bytes.",
3140 type => 'integer',
3141 optional => 1,
3142 renderer => 'bytes',
3143 },
3144 maxdisk => {
3145 description => "Root disk size in bytes.",
3146 type => 'integer',
3147 optional => 1,
3148 renderer => 'bytes',
3149 },
3150 name => {
3151 description => "VM name.",
3152 type => 'string',
3153 optional => 1,
3154 },
3155 qmpstatus => {
3156 description => "Qemu QMP agent status.",
3157 type => 'string',
3158 optional => 1,
3159 },
3160 pid => {
3161 description => "PID of running qemu process.",
3162 type => 'integer',
3163 optional => 1,
3164 },
3165 uptime => {
3166 description => "Uptime.",
3167 type => 'integer',
3168 optional => 1,
3169 renderer => 'duration',
3170 },
3171 cpus => {
3172 description => "Maximum usable CPUs.",
3173 type => 'number',
3174 optional => 1,
3175 },
e6ed61b4 3176 lock => {
11efdfa5 3177 description => "The current config lock, if any.",
e6ed61b4
DC
3178 type => 'string',
3179 optional => 1,
3180 }
b1a70cab
DM
3181};
3182
1e3baf05
DM
3183my $last_proc_pid_stat;
3184
03a33f30
DM
3185# get VM status information
3186# This must be fast and should not block ($full == false)
3187# We only query KVM using QMP if $full == true (this can be slow)
1e3baf05 3188sub vmstatus {
03a33f30 3189 my ($opt_vmid, $full) = @_;
1e3baf05
DM
3190
3191 my $res = {};
3192
19672434 3193 my $storecfg = PVE::Storage::config();
1e3baf05
DM
3194
3195 my $list = vzlist();
3618ee99
EK
3196 my $defaults = load_defaults();
3197
694fcad4 3198 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
1e3baf05 3199
ae4915a2
DM
3200 my $cpucount = $cpuinfo->{cpus} || 1;
3201
1e3baf05
DM
3202 foreach my $vmid (keys %$list) {
3203 next if $opt_vmid && ($vmid ne $opt_vmid);
3204
ffda963f 3205 my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
1e3baf05
DM
3206 my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3207
b1a70cab 3208 my $d = { vmid => $vmid };
1e3baf05
DM
3209 $d->{pid} = $list->{$vmid}->{pid};
3210
3211 # fixme: better status?
3212 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3213
af990afe
DM
3214 my $size = disksize($storecfg, $conf);
3215 if (defined($size)) {
3216 $d->{disk} = 0; # no info available
1e3baf05
DM
3217 $d->{maxdisk} = $size;
3218 } else {
3219 $d->{disk} = 0;
3220 $d->{maxdisk} = 0;
3221 }
3222
3618ee99
EK
3223 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3224 * ($conf->{cores} || $defaults->{cores});
ae4915a2 3225 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
d7c8364b 3226 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
ae4915a2 3227
1e3baf05 3228 $d->{name} = $conf->{name} || "VM $vmid";
3618ee99
EK
3229 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3230 : $defaults->{memory}*(1024*1024);
1e3baf05 3231
8b1accf7 3232 if ($conf->{balloon}) {
4bdb0514 3233 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3618ee99
EK
3234 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3235 : $defaults->{shares};
8b1accf7
DM
3236 }
3237
1e3baf05
DM
3238 $d->{uptime} = 0;
3239 $d->{cpu} = 0;
1e3baf05
DM
3240 $d->{mem} = 0;
3241
3242 $d->{netout} = 0;
3243 $d->{netin} = 0;
3244
3245 $d->{diskread} = 0;
3246 $d->{diskwrite} = 0;
3247
ffda963f 3248 $d->{template} = PVE::QemuConfig->is_template($conf);
4d8c851b 3249
8107b378 3250 $d->{serial} = 1 if conf_has_serial($conf);
e6ed61b4 3251 $d->{lock} = $conf->{lock} if $conf->{lock};
8107b378 3252
1e3baf05
DM
3253 $res->{$vmid} = $d;
3254 }
3255
3256 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3257 foreach my $dev (keys %$netdev) {
3258 next if $dev !~ m/^tap([1-9]\d*)i/;
3259 my $vmid = $1;
3260 my $d = $res->{$vmid};
3261 next if !$d;
19672434 3262
1e3baf05
DM
3263 $d->{netout} += $netdev->{$dev}->{receive};
3264 $d->{netin} += $netdev->{$dev}->{transmit};
604ea644
AD
3265
3266 if ($full) {
3267 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3268 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3269 }
3270
1e3baf05
DM
3271 }
3272
1e3baf05
DM
3273 my $ctime = gettimeofday;
3274
3275 foreach my $vmid (keys %$list) {
3276
3277 my $d = $res->{$vmid};
3278 my $pid = $d->{pid};
3279 next if !$pid;
3280
694fcad4
DM
3281 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3282 next if !$pstat; # not running
19672434 3283
694fcad4 3284 my $used = $pstat->{utime} + $pstat->{stime};
1e3baf05 3285
694fcad4 3286 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
1e3baf05 3287
694fcad4 3288 if ($pstat->{vsize}) {
6b64503e 3289 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
1e3baf05
DM
3290 }
3291
3292 my $old = $last_proc_pid_stat->{$pid};
3293 if (!$old) {
19672434
DM
3294 $last_proc_pid_stat->{$pid} = {
3295 time => $ctime,
1e3baf05
DM
3296 used => $used,
3297 cpu => 0,
1e3baf05
DM
3298 };
3299 next;
3300 }
3301
7f0b5beb 3302 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
1e3baf05
DM
3303
3304 if ($dtime > 1000) {
3305 my $dutime = $used - $old->{used};
3306
ae4915a2 3307 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
1e3baf05 3308 $last_proc_pid_stat->{$pid} = {
19672434 3309 time => $ctime,
1e3baf05
DM
3310 used => $used,
3311 cpu => $d->{cpu},
1e3baf05
DM
3312 };
3313 } else {
3314 $d->{cpu} = $old->{cpu};
1e3baf05
DM
3315 }
3316 }
3317
f5eb281a 3318 return $res if !$full;
03a33f30
DM
3319
3320 my $qmpclient = PVE::QMPClient->new();
3321
64e7fcf2
DM
3322 my $ballooncb = sub {
3323 my ($vmid, $resp) = @_;
3324
3325 my $info = $resp->{'return'};
38babf81
DM
3326 return if !$info->{max_mem};
3327
64e7fcf2
DM
3328 my $d = $res->{$vmid};
3329
38babf81
DM
3330 # use memory assigned to VM
3331 $d->{maxmem} = $info->{max_mem};
3332 $d->{balloon} = $info->{actual};
3333
3334 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3335 $d->{mem} = $info->{total_mem} - $info->{free_mem};
3336 $d->{freemem} = $info->{free_mem};
64e7fcf2
DM
3337 }
3338
604ea644 3339 $d->{ballooninfo} = $info;
64e7fcf2
DM
3340 };
3341
03a33f30
DM
3342 my $blockstatscb = sub {
3343 my ($vmid, $resp) = @_;
3344 my $data = $resp->{'return'} || [];
3345 my $totalrdbytes = 0;
3346 my $totalwrbytes = 0;
604ea644 3347
03a33f30
DM
3348 for my $blockstat (@$data) {
3349 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3350 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
604ea644
AD
3351
3352 $blockstat->{device} =~ s/drive-//;
3353 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
03a33f30
DM
3354 }
3355 $res->{$vmid}->{diskread} = $totalrdbytes;
3356 $res->{$vmid}->{diskwrite} = $totalwrbytes;
3357 };
3358
3359 my $statuscb = sub {
3360 my ($vmid, $resp) = @_;
64e7fcf2 3361
03a33f30 3362 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
64e7fcf2
DM
3363 # this fails if ballon driver is not loaded, so this must be
3364 # the last commnand (following command are aborted if this fails).
38babf81 3365 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
03a33f30
DM
3366
3367 my $status = 'unknown';
3368 if (!defined($status = $resp->{'return'}->{status})) {
3369 warn "unable to get VM status\n";
3370 return;
3371 }
3372
3373 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3374 };
3375
3376 foreach my $vmid (keys %$list) {
3377 next if $opt_vmid && ($vmid ne $opt_vmid);
3378 next if !$res->{$vmid}->{pid}; # not running
3379 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3380 }
3381
b017fbda 3382 $qmpclient->queue_execute(undef, 2);
03a33f30
DM
3383
3384 foreach my $vmid (keys %$list) {
3385 next if $opt_vmid && ($vmid ne $opt_vmid);
3386 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3387 }
3388
1e3baf05
DM
3389 return $res;
3390}
3391
3392sub foreach_drive {
b74ff047 3393 my ($conf, $func, @param) = @_;
1e3baf05 3394
74479ee9
FG
3395 foreach my $ds (valid_drive_names()) {
3396 next if !defined($conf->{$ds});
1e3baf05 3397
6b64503e 3398 my $drive = parse_drive($ds, $conf->{$ds});
1e3baf05
DM
3399 next if !$drive;
3400
b74ff047 3401 &$func($ds, $drive, @param);
1e3baf05
DM
3402 }
3403}
3404
d5769dc2 3405sub foreach_volid {
b6adff33 3406 my ($conf, $func, @param) = @_;
be190583 3407
d5769dc2
DM
3408 my $volhash = {};
3409
3410 my $test_volid = sub {
c272ba8d 3411 my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
d5769dc2
DM
3412
3413 return if !$volid;
be190583 3414
392f8b5d
DM
3415 $volhash->{$volid}->{cdrom} //= 1;
3416 $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3417
3418 $volhash->{$volid}->{replicate} //= 0;
3419 $volhash->{$volid}->{replicate} = 1 if $replicate;
39019f75 3420
ec82e3ee
CH
3421 $volhash->{$volid}->{shared} //= 0;
3422 $volhash->{$volid}->{shared} = 1 if $shared;
3423
39019f75
DM
3424 $volhash->{$volid}->{referenced_in_config} //= 0;
3425 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3426
3427 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3428 if defined($snapname);
c272ba8d 3429 $volhash->{$volid}->{size} = $size if $size;
d5769dc2
DM
3430 };
3431
ed221350 3432 foreach_drive($conf, sub {
d5769dc2 3433 my ($ds, $drive) = @_;
c272ba8d 3434 $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
d5769dc2
DM
3435 });
3436
3437 foreach my $snapname (keys %{$conf->{snapshots}}) {
3438 my $snap = $conf->{snapshots}->{$snapname};
39019f75 3439 $test_volid->($snap->{vmstate}, 0, 1, $snapname);
ed221350 3440 foreach_drive($snap, sub {
d5769dc2 3441 my ($ds, $drive) = @_;
ec82e3ee 3442 $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
d5769dc2
DM
3443 });
3444 }
3445
3446 foreach my $volid (keys %$volhash) {
b6adff33 3447 &$func($volid, $volhash->{$volid}, @param);
d5769dc2
DM
3448 }
3449}
3450
8107b378
DC
3451sub conf_has_serial {
3452 my ($conf) = @_;
3453
3454 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3455 if ($conf->{"serial$i"}) {
3456 return 1;
3457 }
3458 }
3459
3460 return 0;
3461}
3462
d5535a00
TL
3463sub conf_has_audio {
3464 my ($conf, $id) = @_;
3465
3466 $id //= 0;
3467 my $audio = $conf->{"audio$id"};
3468 return undef if !defined($audio);
3469
3470 my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3471 my $audiodriver = $audioproperties->{driver} // 'spice';
3472
3473 return {
3474 dev => $audioproperties->{device},
b0f96836 3475 dev_id => "audiodev$id",
d5535a00
TL
3476 backend => $audiodriver,
3477 backend_id => "$audiodriver-backend${id}",
3478 };
3479}
3480
86b8228b
DM
3481sub vga_conf_has_spice {
3482 my ($vga) = @_;
3483
55655ebc
DC
3484 my $vgaconf = parse_vga($vga);
3485 my $vgatype = $vgaconf->{type};
3486 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
590e698c
DM
3487
3488 return $1 || 1;
86b8228b
DM
3489}
3490
d731ecbe
WB
3491my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3492sub get_host_arch() {
3493 $host_arch = (POSIX::uname())[4] if !$host_arch;
3494 return $host_arch;
3495}
3496
3497sub is_native($) {
3498 my ($arch) = @_;
3499 return get_host_arch() eq $arch;
3500}
3501
3502my $default_machines = {
3503 x86_64 => 'pc',
3504 aarch64 => 'virt',
3505};
3506
3507sub get_basic_machine_info {
3508 my ($conf, $forcemachine) = @_;
3509
3510 my $arch = $conf->{arch} // get_host_arch();
3511 my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3512 return ($arch, $machine);
3513}
3514
96ed3574
WB
3515sub get_ovmf_files($) {
3516 my ($arch) = @_;
3517
3518 my $ovmf = $OVMF->{$arch}
3519 or die "no OVMF images known for architecture '$arch'\n";
3520
3521 return @$ovmf;
3522}
3523
6908fd9b
WB
3524my $Arch2Qemu = {
3525 aarch64 => '/usr/bin/qemu-system-aarch64',
3526 x86_64 => '/usr/bin/qemu-system-x86_64',
3527};
3528sub get_command_for_arch($) {
3529 my ($arch) = @_;
3530 return '/usr/bin/kvm' if is_native($arch);
3531
3532 my $cmd = $Arch2Qemu->{$arch}
3533 or die "don't know how to emulate architecture '$arch'\n";
3534 return $cmd;
3535}
3536
4fc262bd
WB
3537sub get_cpu_options {
3538 my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3539
3540 my $cpuFlags = [];
3541 my $ostype = $conf->{ostype};
3542
3543 my $cpu = $kvm ? "kvm64" : "qemu64";
0f27a91d
WB
3544 if ($arch eq 'aarch64') {
3545 $cpu = 'cortex-a57';
3546 }
2894c247 3547 my $hv_vendor_id;
4fc262bd
WB
3548 if (my $cputype = $conf->{cpu}) {
3549 my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3550 or die "Cannot parse cpu description: $cputype\n";
3551 $cpu = $cpuconf->{cputype};
3552 $kvm_off = 1 if $cpuconf->{hidden};
2894c247 3553 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
4fc262bd
WB
3554
3555 if (defined(my $flags = $cpuconf->{flags})) {
3556 push @$cpuFlags, split(";", $flags);
3557 }
3558 }
3559
1ea63c15 3560 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
4fc262bd
WB
3561
3562 push @$cpuFlags , '-x2apic'
3563 if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
1ea63c15 3569 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
4fc262bd
WB
3570
3571 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573 }
3574
2894c247 3575 add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
4fc262bd 3576
1ea63c15 3577 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
4fc262bd
WB
3578
3579 push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3582 push @$cpuFlags, "vendor=${cpu_vendor}"
3583 if $cpu_vendor ne 'default';
3584 } elsif ($arch ne 'aarch64') {
3585 die "internal error"; # should not happen
3586 }
3587
3588 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3589
3590 return ('-cpu', $cpu);
3591}
3592
1e3baf05 3593sub config_to_command {
67812f9c 3594 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
1e3baf05
DM
3595
3596 my $cmd = [];
8c559505
DM
3597 my $globalFlags = [];
3598 my $machineFlags = [];
3599 my $rtcFlags = [];
5bdcf937 3600 my $devices = [];
b78ebef7 3601 my $pciaddr = '';
5bdcf937 3602 my $bridges = {};
1e3baf05 3603 my $vernum = 0; # unknown
b42d3cf9 3604 my $ostype = $conf->{ostype};
4317f69f 3605 my $winversion = windows_version($ostype);
d731ecbe
WB
3606 my $kvm = $conf->{kvm};
3607
3608 my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
1476b99f
DC
3609 my $kvm_binary = get_command_for_arch($arch);
3610 my $kvmver = kvm_user_version($kvm_binary);
d731ecbe 3611 $kvm //= 1 if is_native($arch);
4317f69f 3612
d731ecbe
WB
3613 if ($kvm) {
3614 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3615 if !defined kvm_version();
3616 }
bfcd9b7e 3617
a3c52213
DM
3618 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3619 $vernum = $1*1000000+$2*1000;
3620 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
1e3baf05
DM
3621 $vernum = $1*1000000+$2*1000+$3;
3622 }
3623
a3c52213 3624 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
1e3baf05 3625
db656e5f 3626 my $q35 = machine_type_is_q35($conf);
4d3f29ed 3627 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
249c4a6c
AD
3628 my $use_old_bios_files = undef;
3629 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
db656e5f 3630
f08e17c7
AD
3631 my $cpuunits = defined($conf->{cpuunits}) ?
3632 $conf->{cpuunits} : $defaults->{cpuunits};
3633
1476b99f 3634 push @$cmd, $kvm_binary;
1e3baf05
DM
3635
3636 push @$cmd, '-id', $vmid;
3637
e4d4cda1
HR
3638 my $vmname = $conf->{name} || "vm$vmid";
3639
3640 push @$cmd, '-name', $vmname;
3641
1e3baf05
DM
3642 my $use_virtio = 0;
3643
c971c4f2
AD
3644 my $qmpsocket = qmp_socket($vmid);
3645 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3646 push @$cmd, '-mon', "chardev=qmp,mode=control";
3647
71bd73b5 3648 if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
b4496b9e 3649 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
71bd73b5
DC
3650 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3651 }
1e3baf05 3652
6b64503e 3653 push @$cmd, '-pidfile' , pidfile_name($vmid);
19672434 3654
1e3baf05
DM
3655 push @$cmd, '-daemonize';
3656
2796e7d5 3657 if ($conf->{smbios1}) {
1f30ac3a
CE
3658 my $smbios_conf = parse_smbios1($conf->{smbios1});
3659 if ($smbios_conf->{base64}) {
3660 # Do not pass base64 flag to qemu
3661 delete $smbios_conf->{base64};
3662 my $smbios_string = "";
3663 foreach my $key (keys %$smbios_conf) {
3664 my $value;
3665 if ($key eq "uuid") {
3666 $value = $smbios_conf->{uuid}
3667 } else {
3668 $value = decode_base64($smbios_conf->{$key});
3669 }
3670 # qemu accepts any binary data, only commas need escaping by double comma
3671 $value =~ s/,/,,/g;
3672 $smbios_string .= "," . $key . "=" . $value if $value;
3673 }
3674 push @$cmd, '-smbios', "type=1" . $smbios_string;
3675 } else {
3676 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3677 }
2796e7d5
DM
3678 }
3679
6ee499ff
DC
3680 if ($conf->{vmgenid}) {
3681 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3682 }
3683
96ed3574 3684 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3edb45e7 3685 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
96ed3574 3686 die "uefi base image not found\n" if ! -f $ovmf_code;
2ddc0a5c 3687
4dcce9ee 3688 my $path;
13bca7b4 3689 my $format;
4dcce9ee
TL
3690 if (my $efidisk = $conf->{efidisk0}) {
3691 my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2ddc0a5c 3692 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
13bca7b4 3693 $format = $d->{format};
2ddc0a5c
DC
3694 if ($storeid) {
3695 $path = PVE::Storage::path($storecfg, $d->{file});
13bca7b4
WB
3696 if (!defined($format)) {
3697 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3698 $format = qemu_img_format($scfg, $volname);
3699 }
2ddc0a5c
DC
3700 } else {
3701 $path = $d->{file};
13bca7b4
WB
3702 die "efidisk format must be specified\n"
3703 if !defined($format);
2ddc0a5c 3704 }
2ddc0a5c 3705 } else {
4dcce9ee
TL
3706 warn "no efidisk configured! Using temporary efivars disk.\n";
3707 $path = "/tmp/$vmid-ovmf.fd";
96ed3574 3708 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
13bca7b4 3709 $format = 'raw';
2ddc0a5c 3710 }
4dcce9ee 3711
96ed3574 3712 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
2bfbee03 3713 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
a783c78e
AD
3714 }
3715
7583d156
DC
3716 # load q35 config
3717 if ($q35) {
3718 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719 if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3720 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721 } else {
3722 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723 }
3724 }
da8b4189 3725
d40e5e18 3726 # add usb controllers
d559309f 3727 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
d40e5e18 3728 push @$devices, @usbcontrollers if @usbcontrollers;
55655ebc 3729 my $vga = parse_vga($conf->{vga});
2fa3151e 3730
55655ebc
DC
3731 my $qxlnum = vga_conf_has_spice($conf->{vga});
3732 $vga->{type} = 'qxl' if $qxlnum;
2fa3151e 3733
55655ebc 3734 if (!$vga->{type}) {
869ad4a7
WB
3735 if ($arch eq 'aarch64') {
3736 $vga->{type} = 'virtio';
3737 } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
55655ebc 3738 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
a2a5cd64 3739 } else {
55655ebc 3740 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
a2a5cd64 3741 }
5acbfe9e
DM
3742 }
3743
1e3baf05 3744 # enable absolute mouse coordinates (needed by vnc)
5acbfe9e
DM
3745 my $tablet;
3746 if (defined($conf->{tablet})) {
3747 $tablet = $conf->{tablet};
3748 } else {
3749 $tablet = $defaults->{tablet};
590e698c 3750 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
55655ebc 3751 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
5acbfe9e
DM
3752 }
3753
d559309f
WB
3754 if ($tablet) {
3755 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3756 my $kbd = print_keyboarddevice_full($conf, $arch);
3757 push @$devices, '-device', $kbd if defined($kbd);
3758 }
b467f79a 3759
16a91d65 3760 my $kvm_off = 0;
4317f69f
AD
3761 my $gpu_passthrough;
3762
1e3baf05 3763 # host pci devices
040b06b7 3764 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
dae0c8e5
TL
3765 my $id = "hostpci$i";
3766 my $d = parse_hostpci($conf->{$id});
2e3b7e2a
AD
3767 next if !$d;
3768
dae0c8e5 3769 if (my $pcie = $d->{pcie}) {
2e3b7e2a 3770 die "q35 machine model is not enabled" if !$q35;
739ba340
DC
3771 # win7 wants to have the pcie devices directly on the pcie bus
3772 # instead of in the root port
3773 if ($winversion == 7) {
dae0c8e5 3774 $pciaddr = print_pcie_addr("${id}bus0");
739ba340 3775 } else {
c4e16381 3776 # add more root ports if needed, 4 are present by default
dae0c8e5 3777 # by pve-q35 cfgs, rest added here on demand.
c4e16381
AL
3778 if ($i > 3) {
3779 push @$devices, '-device', print_pcie_root_port($i);
3780 }
dae0c8e5 3781 $pciaddr = print_pcie_addr($id);
739ba340 3782 }
bd772c2e 3783 } else {
dae0c8e5 3784 $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
2e3b7e2a
AD
3785 }
3786
1f4f447b
WB
3787 my $xvga = '';
3788 if ($d->{'x-vga'}) {
dae0c8e5 3789 $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
16a91d65 3790 $kvm_off = 1;
bfc0bb81 3791 $vga->{type} = 'none' if !defined($conf->{vga});
4317f69f 3792 $gpu_passthrough = 1;
137483c0 3793 }
dae0c8e5 3794
4543ecf0
AD
3795 my $pcidevices = $d->{pciid};
3796 my $multifunction = 1 if @$pcidevices > 1;
dae0c8e5 3797
6ab45bd7
DC
3798 my $sysfspath;
3799 if ($d->{mdev} && scalar(@$pcidevices) == 1) {
dae0c8e5 3800 my $pci_id = $pcidevices->[0]->{id};
6ab45bd7 3801 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
dae0c8e5 3802 $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
6ab45bd7 3803 } elsif ($d->{mdev}) {
dae0c8e5 3804 warn "ignoring mediated device '$id' with multifunction device\n";
6ab45bd7 3805 }
2e3b7e2a 3806
4543ecf0 3807 my $j=0;
dae0c8e5 3808 foreach my $pcidevice (@$pcidevices) {
6ab45bd7 3809 my $devicestr = "vfio-pci";
dae0c8e5 3810
6ab45bd7
DC
3811 if ($sysfspath) {
3812 $devicestr .= ",sysfsdev=$sysfspath";
3813 } else {
2fd24788 3814 $devicestr .= ",host=$pcidevice->{id}";
6ab45bd7 3815 }
4543ecf0 3816
dae0c8e5
TL
3817 my $mf_addr = $multifunction ? ".$j" : '';
3818 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3819
3820 if ($j == 0) {
3821 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3822 $devicestr .= "$xvga";
4543ecf0 3823 $devicestr .= ",multifunction=on" if $multifunction;
dae0c8e5 3824 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
4543ecf0
AD
3825 }
3826
3827 push @$devices, '-device', $devicestr;
3828 $j++;
3829 }
1e3baf05
DM
3830 }
3831
3832 # usb devices
ae36393d 3833 my $usb_dev_features = {};
733234be 3834 $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
ae36393d
AL
3835
3836 my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
d40e5e18 3837 push @$devices, @usbdevices if @usbdevices;
1e3baf05 3838 # serial devices
bae179aa 3839 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
34978be3 3840 if (my $path = $conf->{"serial$i"}) {
9f9d2fb2
DM
3841 if ($path eq 'socket') {
3842 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3843 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
91b01bbb
WB
3844 # On aarch64, serial0 is the UART device. Qemu only allows
3845 # connecting UART devices via the '-serial' command line, as
3846 # the device has a fixed slot on the hardware...
3847 if ($arch eq 'aarch64' && $i == 0) {
3848 push @$devices, '-serial', "chardev:serial$i";
3849 } else {
3850 push @$devices, '-device', "isa-serial,chardev=serial$i";
3851 }
9f9d2fb2
DM
3852 } else {
3853 die "no such serial device\n" if ! -c $path;
3854 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3855 push @$devices, '-device', "isa-serial,chardev=serial$i";
3856 }
34978be3 3857 }
1e3baf05
DM
3858 }
3859
3860 # parallel devices
1989a89c 3861 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
34978be3 3862 if (my $path = $conf->{"parallel$i"}) {
19672434 3863 die "no such parallel device\n" if ! -c $path;
32e69805 3864 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
4c5dbaf6 3865 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
5bdcf937 3866 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
34978be3 3867 }
1e3baf05
DM
3868 }
3869
d5535a00
TL
3870 if (my $audio = conf_has_audio($conf)) {
3871
2e7b5925
AL
3872 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3873
d5535a00
TL
3874 my $id = $audio->{dev_id};
3875 if ($audio->{dev} eq 'AC97') {
3876 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3877 } elsif ($audio->{dev} =~ /intel\-hda$/) {
3878 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3879 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3880 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
b3703d39 3881 } else {
d5535a00 3882 die "unkown audio device '$audio->{dev}', implement me!";
2e7b5925 3883 }
1448547f 3884
d5535a00 3885 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2e7b5925 3886 }
19672434 3887
1e3baf05
DM
3888 my $sockets = 1;
3889 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3890 $sockets = $conf->{sockets} if $conf->{sockets};
3891
3892 my $cores = $conf->{cores} || 1;
3bd18e48 3893
de9d1e55 3894 my $maxcpus = $sockets * $cores;
76267728 3895
de9d1e55 3896 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
76267728 3897
de9d1e55
AD
3898 my $allowed_vcpus = $cpuinfo->{cpus};
3899
6965d5d1 3900 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
de9d1e55
AD
3901 if ($allowed_vcpus < $maxcpus);
3902
69c81430 3903 if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
1e3baf05 3904
69c81430
AD
3905 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3906 for (my $i = 2; $i <= $vcpus; $i++) {
3907 my $cpustr = print_cpu_device($conf,$i);
3908 push @$cmd, '-device', $cpustr;
3909 }
3910
3911 } else {
3912
3913 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3914 }
1e3baf05
DM
3915 push @$cmd, '-nodefaults';
3916
32baffb4 3917 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3b408e82 3918
0888fdce
DM
3919 my $bootindex_hash = {};
3920 my $i = 1;
3921 foreach my $o (split(//, $bootorder)) {
3922 $bootindex_hash->{$o} = $i*100;
3923 $i++;
afdb31d5 3924 }
3b408e82 3925
dbea4415 3926 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
1e3baf05 3927
6b64503e 3928 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
1e3baf05 3929
6b64503e 3930 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
1e3baf05 3931
84902837 3932 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
d559309f 3933 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
b7be4ba9 3934 my $socket = vnc_socket($vmid);
dc62a7fa 3935 push @$cmd, '-vnc', "unix:$socket,password";
b7be4ba9 3936 } else {
55655ebc 3937 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
b7be4ba9
AD
3938 push @$cmd, '-nographic';
3939 }
3940
1e3baf05 3941 # time drift fix
6b64503e 3942 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
1e3baf05 3943
8c559505 3944 my $useLocaltime = $conf->{localtime};
1e3baf05 3945
4317f69f
AD
3946 if ($winversion >= 5) { # windows
3947 $useLocaltime = 1 if !defined($conf->{localtime});
7a131888 3948
4317f69f
AD
3949 # use time drift fix when acpi is enabled
3950 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3951 $tdf = 1 if !defined($conf->{tdf});
462e8d19 3952 }
4317f69f 3953 }
462e8d19 3954
4317f69f
AD
3955 if ($winversion >= 6) {
3956 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3957 push @$cmd, '-no-hpet';
1e3baf05
DM
3958 }
3959
8c559505
DM
3960 push @$rtcFlags, 'driftfix=slew' if $tdf;
3961
74c02ef7 3962 if (!$kvm) {
8c559505 3963 push @$machineFlags, 'accel=tcg';
7f0b5beb 3964 }
1e3baf05 3965
952958bc
DM
3966 if ($machine_type) {
3967 push @$machineFlags, "type=${machine_type}";
3bafc510
DM
3968 }
3969
85f0511d 3970 if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
8c559505
DM
3971 push @$rtcFlags, "base=$conf->{startdate}";
3972 } elsif ($useLocaltime) {
3973 push @$rtcFlags, 'base=localtime';
3974 }
1e3baf05 3975
4fc262bd 3976 push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
519ed28c 3977
0567a4d5 3978 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
370b05e7 3979
1e3baf05
DM
3980 push @$cmd, '-S' if $conf->{freeze};
3981
b20df606 3982 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
1e3baf05 3983
9d66b397 3984 if (parse_guest_agent($conf)->{enabled}) {
7a6c2150 3985 my $qgasocket = qmp_socket($vmid, 1);
d559309f 3986 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
ab6a046f
AD
3987 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3988 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3989 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3990 }
3991
1d794448 3992 my $spice_port;
2fa3151e 3993
590e698c
DM
3994 if ($qxlnum) {
3995 if ($qxlnum > 1) {
ac087616 3996 if ($winversion){
590e698c 3997 for(my $i = 1; $i < $qxlnum; $i++){
d559309f 3998 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
590e698c
DM
3999 }
4000 } else {
4001 # assume other OS works like Linux
55655ebc
DC
4002 my ($ram, $vram) = ("134217728", "67108864");
4003 if ($vga->{memory}) {
4004 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4005 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4006 }
4007 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4008 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
2fa3151e
AD
4009 }
4010 }
4011
d559309f 4012 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
95a4b4a9 4013
af0eba7e
WB
4014 my $nodename = PVE::INotify::nodename();
4015 my $pfamily = PVE::Tools::get_host_address_family($nodename);
91152441
WB
4016 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4017 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4d316a63
AL
4018
4019 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4020 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4021 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4022
91152441
WB
4023 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4024 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
943340a6 4025
caab114a
TL
4026 my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4027 if ($spice_enhancement->{foldersharing}) {
4028 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4029 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4030 }
c4df18db 4031
caab114a
TL
4032 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4033 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4034 push @$devices, '-spice', "$spice_opts";
1011b570
DM
4035 }
4036
8d9ae0d2
DM
4037 # enable balloon by default, unless explicitly disabled
4038 if (!defined($conf->{balloon}) || $conf->{balloon}) {
d559309f 4039 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
8d9ae0d2
DM
4040 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4041 }
1e3baf05 4042
0ea9541d
DM
4043 if ($conf->{watchdog}) {
4044 my $wdopts = parse_watchdog($conf->{watchdog});
d559309f 4045 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
0a40e8ea 4046 my $watchdog = $wdopts->{model} || 'i6300esb';
5bdcf937
AD
4047 push @$devices, '-device', "$watchdog$pciaddr";
4048 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
0ea9541d
DM
4049 }
4050
1e3baf05 4051 my $vollist = [];
941e0c42 4052 my $scsicontroller = {};
26ee04b6 4053 my $ahcicontroller = {};
cdd20088 4054 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
1e3baf05 4055
5881b913
DM
4056 # Add iscsi initiator name if available
4057 if (my $initiator = get_initiator_name()) {
4058 push @$devices, '-iscsi', "initiator-name=$initiator";
4059 }
4060
1e3baf05
DM
4061 foreach_drive($conf, sub {
4062 my ($ds, $drive) = @_;
4063
ff1a2432 4064 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
1e3baf05 4065 push @$vollist, $drive->{file};
ff1a2432 4066 }
afdb31d5 4067
4dcce9ee
TL
4068 # ignore efidisk here, already added in bios/fw handling code above
4069 return if $drive->{interface} eq 'efidisk';
4070
1e3baf05 4071 $use_virtio = 1 if $ds =~ m/^virtio/;
3b408e82
DM
4072
4073 if (drive_is_cdrom ($drive)) {
4074 if ($bootindex_hash->{d}) {
4075 $drive->{bootindex} = $bootindex_hash->{d};
4076 $bootindex_hash->{d} += 1;
4077 }
4078 } else {
4079 if ($bootindex_hash->{c}) {
4080 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4081 $bootindex_hash->{c} += 1;
4082 }
4083 }
4084
51f492cd
AD
4085 if($drive->{interface} eq 'virtio'){
4086 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4087 }
4088
941e0c42 4089 if ($drive->{interface} eq 'scsi') {
cdd20088 4090
ee034f5c 4091 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
6731a4cf 4092
d559309f 4093 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
a1b7d579 4094 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
fc8b40fd
AD
4095
4096 my $iothread = '';
4097 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4098 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4099 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
e7a5104d
DC
4100 } elsif ($drive->{iothread}) {
4101 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
fc8b40fd
AD
4102 }
4103
6e11f143
AD
4104 my $queues = '';
4105 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4106 $queues = ",num_queues=$drive->{queues}";
370b05e7 4107 }
6e11f143
AD
4108
4109 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
cdd20088 4110 $scsicontroller->{$controller}=1;
941e0c42 4111 }
3b408e82 4112
26ee04b6
DA
4113 if ($drive->{interface} eq 'sata') {
4114 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
d559309f 4115 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
5bdcf937 4116 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
26ee04b6
DA
4117 $ahcicontroller->{$controller}=1;
4118 }
46f58b5f 4119
15b21acc
MR
4120 my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4121 push @$devices, '-drive',$drive_cmd;
d559309f 4122 push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
1e3baf05
DM
4123 });
4124
cc4d6182 4125 for (my $i = 0; $i < $MAX_NETS; $i++) {
5f0c4c32 4126 next if !$conf->{"net$i"};
cc4d6182
DA
4127 my $d = parse_net($conf->{"net$i"});
4128 next if !$d;
1e3baf05 4129
cc4d6182 4130 $use_virtio = 1 if $d->{model} eq 'virtio';
1e3baf05 4131
cc4d6182
DA
4132 if ($bootindex_hash->{n}) {
4133 $d->{bootindex} = $bootindex_hash->{n};
4134 $bootindex_hash->{n} += 1;
4135 }
1e3baf05 4136
d559309f 4137 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
5bdcf937
AD
4138 push @$devices, '-netdev', $netdevfull;
4139
d559309f 4140 my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
5bdcf937
AD
4141 push @$devices, '-device', $netdevicefull;
4142 }
1e3baf05 4143
6dbcb073
DC
4144 if ($conf->{ivshmem}) {
4145 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
e3c27a6a 4146
6dbcb073
DC
4147 my $bus;
4148 if ($q35) {
4149 $bus = print_pcie_addr("ivshmem");
4150 } else {
4151 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4152 }
e3c27a6a
TL
4153
4154 my $ivshmem_name = $ivshmem->{name} // $vmid;
4155 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4156
6dbcb073
DC
4157 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4158 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4159 }
4160
db656e5f
DM
4161 if (!$q35) {
4162 # add pci bridges
fc79e813
AD
4163 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4164 $bridges->{1} = 1;
4165 $bridges->{2} = 1;
4166 }
4167
6731a4cf
AD
4168 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4169
311e9293 4170 for my $k (sort {$b cmp $a} keys %$bridges) {
d559309f 4171 $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
f8e83f05
AD
4172 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4173 }
19672434
DM
4174 }
4175
5bdcf937 4176 push @$cmd, @$devices;
be190583 4177 push @$cmd, '-rtc', join(',', @$rtcFlags)
8c559505 4178 if scalar(@$rtcFlags);
be190583 4179 push @$cmd, '-machine', join(',', @$machineFlags)
8c559505
DM
4180 if scalar(@$machineFlags);
4181 push @$cmd, '-global', join(',', @$globalFlags)
4182 if scalar(@$globalFlags);
4183
7ceade4c
DC
4184 if (my $vmstate = $conf->{vmstate}) {
4185 my $statepath = PVE::Storage::path($storecfg, $vmstate);
8f899d73 4186 push @$vollist, $statepath;
7ceade4c
DC
4187 push @$cmd, '-loadstate', $statepath;
4188 }
4189
76350670
DC
4190 # add custom args
4191 if ($conf->{args}) {
4192 my $aa = PVE::Tools::split_args($conf->{args});
4193 push @$cmd, @$aa;
4194 }
4195
1d794448 4196 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
1e3baf05 4197}
19672434 4198
1e3baf05
DM
4199sub vnc_socket {
4200 my ($vmid) = @_;
4201 return "${var_run_tmpdir}/$vmid.vnc";
4202}
4203
943340a6 4204sub spice_port {
1011b570 4205 my ($vmid) = @_;
943340a6 4206
1d794448 4207 my $res = vm_mon_cmd($vmid, 'query-spice');
943340a6
DM
4208
4209 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
1011b570
DM
4210}
4211
c971c4f2 4212sub qmp_socket {
71bd73b5 4213 my ($vmid, $qga, $name) = @_;
693d12a2 4214 my $sockettype = $qga ? 'qga' : 'qmp';
71bd73b5
DC
4215 my $ext = $name ? '-'.$name : '';
4216 return "${var_run_tmpdir}/$vmid$ext.$sockettype";
c971c4f2
AD
4217}
4218
1e3baf05
DM
4219sub pidfile_name {
4220 my ($vmid) = @_;
4221 return "${var_run_tmpdir}/$vmid.pid";
4222}
4223
86fdcfb2
DA
4224sub vm_devices_list {
4225 my ($vmid) = @_;
4226
ceea9078 4227 my $res = vm_mon_cmd($vmid, 'query-pci');
f721624b 4228 my $devices_to_check = [];
ceea9078
DM
4229 my $devices = {};
4230 foreach my $pcibus (@$res) {
f721624b
DC
4231 push @$devices_to_check, @{$pcibus->{devices}},
4232 }
4233
4234 while (@$devices_to_check) {
4235 my $to_check = [];
4236 for my $d (@$devices_to_check) {
4237 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4238 next if !$d->{'pci_bridge'};
4239
4240 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4241 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
f78cc802 4242 }
f721624b 4243 $devices_to_check = $to_check;
f78cc802
AD
4244 }
4245
4246 my $resblock = vm_mon_cmd($vmid, 'query-block');
4247 foreach my $block (@$resblock) {
4248 if($block->{device} =~ m/^drive-(\S+)/){
4249 $devices->{$1} = 1;
1dc4f496
DM
4250 }
4251 }
86fdcfb2 4252
3d7389fe
DM
4253 my $resmice = vm_mon_cmd($vmid, 'query-mice');
4254 foreach my $mice (@$resmice) {
4255 if ($mice->{name} eq 'QEMU HID Tablet') {
4256 $devices->{tablet} = 1;
4257 last;
4258 }
4259 }
4260
deb091c5
DC
4261 # for usb devices there is no query-usb
4262 # but we can iterate over the entries in
4263 # qom-list path=/machine/peripheral
4264 my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4265 foreach my $per (@$resperipheral) {
4266 if ($per->{name} =~ m/^usb\d+$/) {
4267 $devices->{$per->{name}} = 1;
4268 }
4269 }
4270
1dc4f496 4271 return $devices;
86fdcfb2
DA
4272}
4273
ec21aa11 4274sub vm_deviceplug {
d559309f 4275 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
ae57f6b3 4276
db656e5f
DM
4277 my $q35 = machine_type_is_q35($conf);
4278
95d6343b
DA
4279 my $devices_list = vm_devices_list($vmid);
4280 return 1 if defined($devices_list->{$deviceid});
4281
d559309f 4282 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
fee46675 4283
3d7389fe 4284 if ($deviceid eq 'tablet') {
fee46675 4285
d559309f
WB
4286 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4287
4288 } elsif ($deviceid eq 'keyboard') {
4289
4290 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3d7389fe 4291
4eb68604
DC
4292 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4293
f745762b
DC
4294 die "usb hotplug currently not reliable\n";
4295 # since we can't reliably hot unplug all added usb devices
4296 # and usb passthrough disables live migration
4297 # we disable usb hotplugging for now
4eb68604
DC
4298 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4299
fee46675 4300 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
40f28a9f 4301
22de899a
AD
4302 qemu_iothread_add($vmid, $deviceid, $device);
4303
fee46675 4304 qemu_driveadd($storecfg, $vmid, $device);
d559309f 4305 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
fee46675 4306
5e5dcb73 4307 qemu_deviceadd($vmid, $devicefull);
fee46675
DM
4308 eval { qemu_deviceaddverify($vmid, $deviceid); };
4309 if (my $err = $@) {
63c2da2f
DM
4310 eval { qemu_drivedel($vmid, $deviceid); };
4311 warn $@ if $@;
fee46675 4312 die $err;
5e5dcb73 4313 }
cfc817c7 4314
2733141c 4315 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
fee46675 4316
fc8b40fd 4317
cdd20088 4318 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
d559309f 4319 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
a1b7d579 4320 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
2733141c
AD
4321
4322 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
fee46675 4323
fc8b40fd
AD
4324 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4325 qemu_iothread_add($vmid, $deviceid, $device);
4326 $devicefull .= ",iothread=iothread-$deviceid";
4327 }
4328
6e11f143
AD
4329 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4330 $devicefull .= ",num_queues=$device->{queues}";
4331 }
4332
cfc817c7 4333 qemu_deviceadd($vmid, $devicefull);
fee46675 4334 qemu_deviceaddverify($vmid, $deviceid);
cfc817c7 4335
fee46675
DM
4336 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4337
d559309f 4338 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
fee46675 4339 qemu_driveadd($storecfg, $vmid, $device);
a1b7d579 4340
d559309f 4341 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
fee46675
DM
4342 eval { qemu_deviceadd($vmid, $devicefull); };
4343 if (my $err = $@) {
63c2da2f
DM
4344 eval { qemu_drivedel($vmid, $deviceid); };
4345 warn $@ if $@;
fee46675 4346 die $err;
a4f091a0 4347 }
a4f091a0 4348
fee46675
DM
4349 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4350
95d3be58 4351 return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
8718099c 4352
95d3be58
DC
4353 my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4354 my $use_old_bios_files = undef;
4355 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
8718099c 4356
95d3be58
DC
4357 my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4358 qemu_deviceadd($vmid, $netdevicefull);
79046fd1
DC
4359 eval {
4360 qemu_deviceaddverify($vmid, $deviceid);
4361 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4362 };
fee46675
DM
4363 if (my $err = $@) {
4364 eval { qemu_netdevdel($vmid, $deviceid); };
4365 warn $@ if $@;
4366 die $err;
95d3be58 4367 }
2630d2a9 4368
fee46675 4369 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
b467f79a 4370
40f28a9f 4371 my $bridgeid = $2;
d559309f 4372 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
40f28a9f 4373 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
a1b7d579 4374
40f28a9f 4375 qemu_deviceadd($vmid, $devicefull);
fee46675
DM
4376 qemu_deviceaddverify($vmid, $deviceid);
4377
4378 } else {
a1b7d579 4379 die "can't hotplug device '$deviceid'\n";
40f28a9f
AD
4380 }
4381
5e5dcb73 4382 return 1;
a4dea331
DA
4383}
4384
3eec5767 4385# fixme: this should raise exceptions on error!
ec21aa11 4386sub vm_deviceunplug {
f19d1c47 4387 my ($vmid, $conf, $deviceid) = @_;
873c2d69 4388
95d6343b
DA
4389 my $devices_list = vm_devices_list($vmid);
4390 return 1 if !defined($devices_list->{$deviceid});
4391
63c2da2f
DM
4392 die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4393
d559309f 4394 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
63c2da2f 4395
3d7389fe 4396 qemu_devicedel($vmid, $deviceid);
3d7389fe 4397
4eb68604
DC
4398 } elsif ($deviceid =~ m/^usb\d+$/) {
4399
f745762b
DC
4400 die "usb hotplug currently not reliable\n";
4401 # when unplugging usb devices this way,
4402 # there may be remaining usb controllers/hubs
4403 # so we disable it for now
4eb68604
DC
4404 qemu_devicedel($vmid, $deviceid);
4405 qemu_devicedelverify($vmid, $deviceid);
4406
63c2da2f 4407 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
f19d1c47 4408
5e5dcb73 4409 qemu_devicedel($vmid, $deviceid);
63c2da2f
DM
4410 qemu_devicedelverify($vmid, $deviceid);
4411 qemu_drivedel($vmid, $deviceid);
22de899a
AD
4412 qemu_iothread_del($conf, $vmid, $deviceid);
4413
2733141c 4414 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
a1b7d579 4415
63c2da2f 4416 qemu_devicedel($vmid, $deviceid);
8ce30dde 4417 qemu_devicedelverify($vmid, $deviceid);
fc8b40fd 4418 qemu_iothread_del($conf, $vmid, $deviceid);
a1b7d579 4419
63c2da2f 4420 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
cfc817c7 4421
63c2da2f
DM
4422 qemu_devicedel($vmid, $deviceid);
4423 qemu_drivedel($vmid, $deviceid);
a1b7d579 4424 qemu_deletescsihw($conf, $vmid, $deviceid);
8ce30dde 4425
63c2da2f 4426 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
a4f091a0 4427
2630d2a9 4428 qemu_devicedel($vmid, $deviceid);
63c2da2f
DM
4429 qemu_devicedelverify($vmid, $deviceid);
4430 qemu_netdevdel($vmid, $deviceid);
4431
4432 } else {
4433 die "can't unplug device '$deviceid'\n";
2630d2a9
DA
4434 }
4435
5e5dcb73
DA
4436 return 1;
4437}
4438
4439sub qemu_deviceadd {
4440 my ($vmid, $devicefull) = @_;
873c2d69 4441
d695b5b7
AD
4442 $devicefull = "driver=".$devicefull;
4443 my %options = split(/[=,]/, $devicefull);
f19d1c47 4444
d695b5b7 4445 vm_mon_cmd($vmid, "device_add" , %options);
5e5dcb73 4446}
afdb31d5 4447
5e5dcb73 4448sub qemu_devicedel {
fee46675 4449 my ($vmid, $deviceid) = @_;
63c2da2f 4450
5a77d8c1 4451 my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
5e5dcb73
DA
4452}
4453
22de899a
AD
4454sub qemu_iothread_add {
4455 my($vmid, $deviceid, $device) = @_;
4456
4457 if ($device->{iothread}) {
4458 my $iothreads = vm_iothreads_list($vmid);
4459 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4460 }
4461}
4462
4463sub qemu_iothread_del {
4464 my($conf, $vmid, $deviceid) = @_;
4465
7820eae4
DC
4466 my $confid = $deviceid;
4467 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4468 $confid = 'scsi' . $1;
4469 }
4470 my $device = parse_drive($confid, $conf->{$confid});
22de899a
AD
4471 if ($device->{iothread}) {
4472 my $iothreads = vm_iothreads_list($vmid);
4473 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4474 }
4475}
4476
4d3f29ed
AD
4477sub qemu_objectadd {
4478 my($vmid, $objectid, $qomtype) = @_;
4479
4480 vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4481
4482 return 1;
4483}
4484
4485sub qemu_objectdel {
4486 my($vmid, $objectid) = @_;
4487
4488 vm_mon_cmd($vmid, "object-del", id => $objectid);
4489
4490 return 1;
4491}
4492
5e5dcb73 4493sub qemu_driveadd {
fee46675 4494 my ($storecfg, $vmid, $device) = @_;
5e5dcb73
DA
4495
4496 my $drive = print_drive_full($storecfg, $vmid, $device);
7a69fc3c 4497 $drive =~ s/\\/\\\\/g;
8ead5ec7 4498 my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
fee46675 4499
5e5dcb73 4500 # If the command succeeds qemu prints: "OK"
fee46675
DM
4501 return 1 if $ret =~ m/OK/s;
4502
4503 die "adding drive failed: $ret\n";
5e5dcb73 4504}
afdb31d5 4505
5e5dcb73
DA
4506sub qemu_drivedel {
4507 my($vmid, $deviceid) = @_;
873c2d69 4508
7b7c6d1b 4509 my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
5e5dcb73 4510 $ret =~ s/^\s+//;
a1b7d579 4511
63c2da2f 4512 return 1 if $ret eq "";
a1b7d579 4513
63c2da2f 4514 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
a1b7d579
DM
4515 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4516
63c2da2f 4517 die "deleting drive $deviceid failed : $ret\n";
5e5dcb73 4518}
f19d1c47 4519
5e5dcb73 4520sub qemu_deviceaddverify {
fee46675 4521 my ($vmid, $deviceid) = @_;
873c2d69 4522
5e5dcb73
DA
4523 for (my $i = 0; $i <= 5; $i++) {
4524 my $devices_list = vm_devices_list($vmid);
4525 return 1 if defined($devices_list->{$deviceid});
4526 sleep 1;
afdb31d5 4527 }
fee46675
DM
4528
4529 die "error on hotplug device '$deviceid'\n";
5e5dcb73 4530}
afdb31d5 4531
5e5dcb73
DA
4532
4533sub qemu_devicedelverify {
63c2da2f
DM
4534 my ($vmid, $deviceid) = @_;
4535
a1b7d579 4536 # need to verify that the device is correctly removed as device_del
63c2da2f 4537 # is async and empty return is not reliable
5e5dcb73 4538
5e5dcb73
DA
4539 for (my $i = 0; $i <= 5; $i++) {
4540 my $devices_list = vm_devices_list($vmid);
4541 return 1 if !defined($devices_list->{$deviceid});
4542 sleep 1;
afdb31d5 4543 }
63c2da2f
DM
4544
4545 die "error on hot-unplugging device '$deviceid'\n";
873c2d69
DA
4546}
4547
cdd20088 4548sub qemu_findorcreatescsihw {
d559309f 4549 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
cfc817c7 4550
ee034f5c 4551 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
2733141c
AD
4552
4553 my $scsihwid="$controller_prefix$controller";
cfc817c7
DA
4554 my $devices_list = vm_devices_list($vmid);
4555
cdd20088 4556 if(!defined($devices_list->{$scsihwid})) {
d559309f 4557 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
cfc817c7 4558 }
fee46675 4559
cfc817c7
DA
4560 return 1;
4561}
4562
8ce30dde
AD
4563sub qemu_deletescsihw {
4564 my ($conf, $vmid, $opt) = @_;
4565
4566 my $device = parse_drive($opt, $conf->{$opt});
4567
a1511b3c 4568 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
2733141c
AD
4569 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4570 return 1;
4571 }
4572
ee034f5c 4573 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
8ce30dde
AD
4574
4575 my $devices_list = vm_devices_list($vmid);
4576 foreach my $opt (keys %{$devices_list}) {
74479ee9 4577 if (PVE::QemuServer::is_valid_drivename($opt)) {
8ce30dde
AD
4578 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4579 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4580 return 1;
4581 }
4582 }
4583 }
4584
4585 my $scsihwid="scsihw$controller";
4586
4587 vm_deviceunplug($vmid, $conf, $scsihwid);
4588
4589 return 1;
4590}
4591
281fedb3 4592sub qemu_add_pci_bridge {
d559309f 4593 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
40f28a9f
AD
4594
4595 my $bridges = {};
281fedb3
DM
4596
4597 my $bridgeid;
4598
d559309f 4599 print_pci_addr($device, $bridges, $arch, $machine_type);
40f28a9f
AD
4600
4601 while (my ($k, $v) = each %$bridges) {
4602 $bridgeid = $k;
4603 }
fee46675 4604 return 1 if !defined($bridgeid) || $bridgeid < 1;
281fedb3 4605
40f28a9f
AD
4606 my $bridge = "pci.$bridgeid";
4607 my $devices_list = vm_devices_list($vmid);
4608
281fedb3 4609 if (!defined($devices_list->{$bridge})) {
d559309f 4610 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
40f28a9f 4611 }
281fedb3 4612
40f28a9f
AD
4613 return 1;
4614}
4615
25088687
DM
4616sub qemu_set_link_status {
4617 my ($vmid, $device, $up) = @_;
4618
a1b7d579 4619 vm_mon_cmd($vmid, "set_link", name => $device,
25088687
DM
4620 up => $up ? JSON::true : JSON::false);
4621}
4622
2630d2a9 4623sub qemu_netdevadd {
d559309f 4624 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
2630d2a9 4625
d559309f 4626 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
73aa03b8 4627 my %options = split(/[=,]/, $netdev);
2630d2a9 4628
73aa03b8
AD
4629 vm_mon_cmd($vmid, "netdev_add", %options);
4630 return 1;
2630d2a9
DA
4631}
4632
4633sub qemu_netdevdel {
4634 my ($vmid, $deviceid) = @_;
4635
89c1e0f4 4636 vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
2630d2a9
DA
4637}
4638
16521d63 4639sub qemu_usb_hotplug {
d559309f 4640 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
16521d63
DC
4641
4642 return if !$device;
4643
4644 # remove the old one first
4645 vm_deviceunplug($vmid, $conf, $deviceid);
4646
4647 # check if xhci controller is necessary and available
4648 if ($device->{usb3}) {
4649
4650 my $devicelist = vm_devices_list($vmid);
4651
4652 if (!$devicelist->{xhci}) {
d559309f 4653 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
16521d63
DC
4654 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4655 }
4656 }
4657 my $d = parse_usb_device($device->{host});
4658 $d->{usb3} = $device->{usb3};
4659
4660 # add the new one
d559309f 4661 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
16521d63
DC
4662}
4663
838776ab 4664sub qemu_cpu_hotplug {
8edc9c08 4665 my ($vmid, $conf, $vcpus) = @_;
838776ab 4666
1e881b75
AD
4667 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4668
8edc9c08
AD
4669 my $sockets = 1;
4670 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671 $sockets = $conf->{sockets} if $conf->{sockets};
4672 my $cores = $conf->{cores} || 1;
4673 my $maxcpus = $sockets * $cores;
838776ab 4674
8edc9c08 4675 $vcpus = $maxcpus if !$vcpus;
3a11fadb 4676
8edc9c08
AD
4677 die "you can't add more vcpus than maxcpus\n"
4678 if $vcpus > $maxcpus;
3a11fadb 4679
8edc9c08 4680 my $currentvcpus = $conf->{vcpus} || $maxcpus;
1e881b75 4681
eba3e64d 4682 if ($vcpus < $currentvcpus) {
1e881b75
AD
4683
4684 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4685
4686 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687 qemu_devicedel($vmid, "cpu$i");
4688 my $retry = 0;
4689 my $currentrunningvcpus = undef;
4690 while (1) {
4691 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4692 last if scalar(@{$currentrunningvcpus}) == $i-1;
961af8a3 4693 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
1e881b75
AD
4694 $retry++;
4695 sleep 1;
4696 }
4697 #update conf after each succesfull cpu unplug
4698 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699 PVE::QemuConfig->write_config($vmid, $conf);
4700 }
4701 } else {
961af8a3 4702 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
1e881b75
AD
4703 }
4704
4705 return;
4706 }
838776ab 4707
8edc9c08 4708 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
961af8a3 4709 die "vcpus in running vm does not match its configuration\n"
8edc9c08 4710 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
838776ab 4711
eba3e64d
AD
4712 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4713
4714 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715 my $cpustr = print_cpu_device($conf, $i);
4716 qemu_deviceadd($vmid, $cpustr);
4717
4718 my $retry = 0;
4719 my $currentrunningvcpus = undef;
4720 while (1) {
4721 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4722 last if scalar(@{$currentrunningvcpus}) == $i;
961af8a3 4723 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
eba3e64d
AD
4724 sleep 1;
4725 $retry++;
4726 }
4727 #update conf after each succesfull cpu hotplug
4728 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729 PVE::QemuConfig->write_config($vmid, $conf);
4730 }
4731 } else {
4732
4733 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4735 }
838776ab
AD
4736 }
4737}
4738
affd2f88 4739sub qemu_block_set_io_throttle {
277ca170
WB
4740 my ($vmid, $deviceid,
4741 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
9196a8ec
WB
4742 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
affd2f88 4745
f3f323a3
AD
4746 return if !check_running($vmid) ;
4747
277ca170
WB
4748 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749 bps => int($bps),
4750 bps_rd => int($bps_rd),
4751 bps_wr => int($bps_wr),
4752 iops => int($iops),
4753 iops_rd => int($iops_rd),
4754 iops_wr => int($iops_wr),
4755 bps_max => int($bps_max),
4756 bps_rd_max => int($bps_rd_max),
4757 bps_wr_max => int($bps_wr_max),
4758 iops_max => int($iops_max),
4759 iops_rd_max => int($iops_rd_max),
9196a8ec
WB
4760 iops_wr_max => int($iops_wr_max),
4761 bps_max_length => int($bps_max_length),
4762 bps_rd_max_length => int($bps_rd_max_length),
4763 bps_wr_max_length => int($bps_wr_max_length),
4764 iops_max_length => int($iops_max_length),
4765 iops_rd_max_length => int($iops_rd_max_length),
4766 iops_wr_max_length => int($iops_wr_max_length),
277ca170 4767 );
f3f323a3 4768
affd2f88
AD
4769}
4770
f5eb281a 4771# old code, only used to shutdown old VM after update
dab36e1e
DM
4772sub __read_avail {
4773 my ($fh, $timeout) = @_;
4774
4775 my $sel = new IO::Select;
4776 $sel->add($fh);
4777
4778 my $res = '';
4779 my $buf;
4780
4781 my @ready;
4782 while (scalar (@ready = $sel->can_read($timeout))) {
4783 my $count;
4784 if ($count = $fh->sysread($buf, 8192)) {
4785 if ($buf =~ /^(.*)\(qemu\) $/s) {
4786 $res .= $1;
4787 last;
4788 } else {
4789 $res .= $buf;
4790 }
4791 } else {
4792 if (!defined($count)) {
4793 die "$!\n";
4794 }
4795 last;
4796 }
4797 }
4798
4799 die "monitor read timeout\n" if !scalar(@ready);
f5eb281a 4800
dab36e1e
DM
4801 return $res;
4802}
4803
c1175c92
AD
4804sub qemu_block_resize {
4805 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4806
ed221350 4807 my $running = check_running($vmid);
c1175c92 4808
7246e8f9 4809 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
c1175c92
AD
4810
4811 return if !$running;
4812
4813 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4814
4815}
4816
1ab0057c
AD
4817sub qemu_volume_snapshot {
4818 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4819
ed221350 4820 my $running = check_running($vmid);
1ab0057c 4821
e5eaa028 4822 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
eba2b721 4823 vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
e5eaa028
WL
4824 } else {
4825 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4826 }
1ab0057c
AD
4827}
4828
fc46aff9
AD
4829sub qemu_volume_snapshot_delete {
4830 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4831
ed221350 4832 my $running = check_running($vmid);
fc46aff9 4833
a2f1554b
AD
4834 if($running) {
4835
4836 $running = undef;
4837 my $conf = PVE::QemuConfig->load_config($vmid);
4838 foreach_drive($conf, sub {
4839 my ($ds, $drive) = @_;
4840 $running = 1 if $drive->{file} eq $volid;
4841 });
4842 }
4843
1ef7592f 4844 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
eba2b721 4845 vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
1ef7592f
AD
4846 } else {
4847 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4848 }
fc46aff9
AD
4849}
4850
264e519f
DM
4851sub set_migration_caps {
4852 my ($vmid) = @_;
a89fded1 4853
8b8345f3 4854 my $cap_ref = [];
a89fded1
AD
4855
4856 my $enabled_cap = {
8b8345f3 4857 "auto-converge" => 1,
0b0a47e8 4858 "xbzrle" => 1,
8b8345f3
DM
4859 "x-rdma-pin-all" => 0,
4860 "zero-blocks" => 0,
b62532e4 4861 "compress" => 0
a89fded1
AD
4862 };
4863
8b8345f3 4864 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
a89fded1 4865
8b8345f3 4866 for my $supported_capability (@$supported_capabilities) {
b463a3ce
SP
4867 push @$cap_ref, {
4868 capability => $supported_capability->{capability},
22430fa2
DM
4869 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4870 };
a89fded1
AD
4871 }
4872
8b8345f3
DM
4873 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4874}
a89fded1 4875
81d95ae1 4876my $fast_plug_option = {
7498eb64 4877 'lock' => 1,
81d95ae1 4878 'name' => 1,
a1b7d579 4879 'onboot' => 1,
81d95ae1
DM
4880 'shares' => 1,
4881 'startup' => 1,
b0ec896e 4882 'description' => 1,
ec647db4 4883 'protection' => 1,
8cad5e9b 4884 'vmstatestorage' => 1,
9e784b11 4885 'hookscript' => 1,
81d95ae1
DM
4886};
4887
3a11fadb
DM
4888# hotplug changes in [PENDING]
4889# $selection hash can be used to only apply specified options, for
4890# example: { cores => 1 } (only apply changed 'cores')
4891# $errors ref is used to return error messages
c427973b 4892sub vmconfig_hotplug_pending {
3a11fadb 4893 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
c427973b 4894
8e90138a 4895 my $defaults = load_defaults();
d559309f 4896 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
c427973b
DM
4897
4898 # commit values which do not have any impact on running VM first
3a11fadb
DM
4899 # Note: those option cannot raise errors, we we do not care about
4900 # $selection and always apply them.
4901
4902 my $add_error = sub {
4903 my ($opt, $msg) = @_;
4904 $errors->{$opt} = "hotplug problem - $msg";
4905 };
c427973b
DM
4906
4907 my $changes = 0;
4908 foreach my $opt (keys %{$conf->{pending}}) { # add/change
81d95ae1 4909 if ($fast_plug_option->{$opt}) {
c427973b
DM
4910 $conf->{$opt} = $conf->{pending}->{$opt};
4911 delete $conf->{pending}->{$opt};
4912 $changes = 1;
4913 }
4914 }
4915
4916 if ($changes) {
ffda963f
FG
4917 PVE::QemuConfig->write_config($vmid, $conf);
4918 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
c427973b
DM
4919 }
4920
b3c2bdd1 4921 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
c427973b 4922
3dc38fbb
WB
4923 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4924 while (my ($opt, $force) = each %$pending_delete_hash) {
3a11fadb 4925 next if $selection && !$selection->{$opt};
3a11fadb 4926 eval {
51a6f637
AD
4927 if ($opt eq 'hotplug') {
4928 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4929 } elsif ($opt eq 'tablet') {
b3c2bdd1 4930 die "skip\n" if !$hotplug_features->{usb};
3a11fadb 4931 if ($defaults->{tablet}) {
d559309f
WB
4932 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4933 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4934 if $arch eq 'aarch64';
3a11fadb 4935 } else {
d559309f
WB
4936 vm_deviceunplug($vmid, $conf, 'tablet');
4937 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
3a11fadb 4938 }
4eb68604 4939 } elsif ($opt =~ m/^usb\d+/) {
f745762b
DC
4940 die "skip\n";
4941 # since we cannot reliably hot unplug usb devices
4942 # we are disabling it
4eb68604
DC
4943 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4944 vm_deviceunplug($vmid, $conf, $opt);
8edc9c08 4945 } elsif ($opt eq 'vcpus') {
b3c2bdd1 4946 die "skip\n" if !$hotplug_features->{cpu};
8edc9c08 4947 qemu_cpu_hotplug($vmid, $conf, undef);
9c2f7069 4948 } elsif ($opt eq 'balloon') {
81d95ae1 4949 # enable balloon device is not hotpluggable
75b51053
DC
4950 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4951 # here we reset the ballooning value to memory
4952 my $balloon = $conf->{memory} || $defaults->{memory};
4953 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
81d95ae1
DM
4954 } elsif ($fast_plug_option->{$opt}) {
4955 # do nothing
3eec5767 4956 } elsif ($opt =~ m/^net(\d+)$/) {
b3c2bdd1 4957 die "skip\n" if !$hotplug_features->{network};
3eec5767 4958 vm_deviceunplug($vmid, $conf, $opt);
74479ee9 4959 } elsif (is_valid_drivename($opt)) {
b3c2bdd1 4960 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
19120f99 4961 vm_deviceunplug($vmid, $conf, $opt);
3dc38fbb 4962 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4d3f29ed
AD
4963 } elsif ($opt =~ m/^memory$/) {
4964 die "skip\n" if !$hotplug_features->{memory};
6779f1ac 4965 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
c8effec3
AD
4966 } elsif ($opt eq 'cpuunits') {
4967 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
58be00f1
AD
4968 } elsif ($opt eq 'cpulimit') {
4969 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3d7389fe 4970 } else {
e56beeda 4971 die "skip\n";
3d7389fe 4972 }
3a11fadb
DM
4973 };
4974 if (my $err = $@) {
e56beeda
DM
4975 &$add_error($opt, $err) if $err ne "skip\n";
4976 } else {
3a11fadb
DM
4977 # save new config if hotplug was successful
4978 delete $conf->{$opt};
4979 vmconfig_undelete_pending_option($conf, $opt);
ffda963f
FG
4980 PVE::QemuConfig->write_config($vmid, $conf);
4981 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3d7389fe 4982 }
3d7389fe
DM
4983 }
4984
9ed7a77c
WB
4985 my $apply_pending_cloudinit;
4986 $apply_pending_cloudinit = sub {
4987 my ($key, $value) = @_;
4988 $apply_pending_cloudinit = sub {}; # once is enough
4989
4990 my @cloudinit_opts = keys %$confdesc_cloudinit;
4991 foreach my $opt (keys %{$conf->{pending}}) {
4992 next if !grep { $_ eq $opt } @cloudinit_opts;
4993 $conf->{$opt} = delete $conf->{pending}->{$opt};
4994 }
4995
4996 my $new_conf = { %$conf };
4997 $new_conf->{$key} = $value;
4998 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4999 };
5000
3d7389fe 5001 foreach my $opt (keys %{$conf->{pending}}) {
3a11fadb 5002 next if $selection && !$selection->{$opt};
3d7389fe 5003 my $value = $conf->{pending}->{$opt};
3a11fadb 5004 eval {
51a6f637
AD
5005 if ($opt eq 'hotplug') {
5006 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5007 } elsif ($opt eq 'tablet') {
b3c2bdd1 5008 die "skip\n" if !$hotplug_features->{usb};
3a11fadb 5009 if ($value == 1) {
d559309f
WB
5010 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5011 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5012 if $arch eq 'aarch64';
3a11fadb 5013 } elsif ($value == 0) {
d559309f
WB
5014 vm_deviceunplug($vmid, $conf, 'tablet');
5015 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
3a11fadb 5016 }
4eb68604 5017 } elsif ($opt =~ m/^usb\d+$/) {
f745762b
DC
5018 die "skip\n";
5019 # since we cannot reliably hot unplug usb devices
5020 # we are disabling it
4eb68604
DC
5021 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5022 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5023 die "skip\n" if !$d;
d559309f 5024 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
8edc9c08 5025 } elsif ($opt eq 'vcpus') {
b3c2bdd1 5026 die "skip\n" if !$hotplug_features->{cpu};
3a11fadb
DM
5027 qemu_cpu_hotplug($vmid, $conf, $value);
5028 } elsif ($opt eq 'balloon') {
81d95ae1 5029 # enable/disable balloning device is not hotpluggable
8fe689e7 5030 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
a1b7d579 5031 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
81d95ae1
DM
5032 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5033
3a11fadb 5034 # allow manual ballooning if shares is set to zero
4cc1efa6 5035 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
9c2f7069
AD
5036 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5037 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038 }
a1b7d579 5039 } elsif ($opt =~ m/^net(\d+)$/) {
3eec5767 5040 # some changes can be done without hotplug
a1b7d579 5041 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
d559309f 5042 $vmid, $opt, $value, $arch, $machine_type);
74479ee9 5043 } elsif (is_valid_drivename($opt)) {
a05cff86 5044 # some changes can be done without hotplug
9ed7a77c
WB
5045 my $drive = parse_drive($opt, $value);
5046 if (drive_is_cloudinit($drive)) {
5047 &$apply_pending_cloudinit($opt, $value);
5048 }
b3c2bdd1 5049 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
d559309f 5050 $vmid, $opt, $value, 1, $arch, $machine_type);
4d3f29ed
AD
5051 } elsif ($opt =~ m/^memory$/) { #dimms
5052 die "skip\n" if !$hotplug_features->{memory};
6779f1ac 5053 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
c8effec3
AD
5054 } elsif ($opt eq 'cpuunits') {
5055 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
58be00f1 5056 } elsif ($opt eq 'cpulimit') {
c6f773b8 5057 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
58be00f1 5058 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3a11fadb 5059 } else {
e56beeda 5060 die "skip\n"; # skip non-hot-pluggable options
3d7389fe 5061 }
3a11fadb
DM
5062 };
5063 if (my $err = $@) {
e56beeda
DM
5064 &$add_error($opt, $err) if $err ne "skip\n";
5065 } else {
3a11fadb
DM
5066 # save new config if hotplug was successful
5067 $conf->{$opt} = $value;
5068 delete $conf->{pending}->{$opt};
ffda963f
FG
5069 PVE::QemuConfig->write_config($vmid, $conf);
5070 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3d7389fe 5071 }
3d7389fe 5072 }
c427973b 5073}
055d554d 5074
3dc38fbb
WB
5075sub try_deallocate_drive {
5076 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5077
5078 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5079 my $volid = $drive->{file};
5080 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5081 my $sid = PVE::Storage::parse_volume_id($volid);
5082 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
cee01bcb
WB
5083
5084 # check if the disk is really unused
cee01bcb 5085 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
77019edf 5086 if is_volume_in_use($storecfg, $conf, $key, $volid);
cee01bcb 5087 PVE::Storage::vdisk_free($storecfg, $volid);
3dc38fbb 5088 return 1;
40b977f3
WL
5089 } else {
5090 # If vm is not owner of this disk remove from config
5091 return 1;
3dc38fbb
WB
5092 }
5093 }
5094
5095 return undef;
5096}
5097
5098sub vmconfig_delete_or_detach_drive {
5099 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5100
5101 my $drive = parse_drive($opt, $conf->{$opt});
5102
5103 my $rpcenv = PVE::RPCEnvironment::get();
5104 my $authuser = $rpcenv->get_user();
5105
5106 if ($force) {
5107 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5108 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5109 } else {
5110 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5111 }
5112}
5113
055d554d 5114sub vmconfig_apply_pending {
3a11fadb 5115 my ($vmid, $conf, $storecfg) = @_;
c427973b
DM
5116
5117 # cold plug
055d554d 5118
3dc38fbb
WB
5119 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5120 while (my ($opt, $force) = each %$pending_delete_hash) {
055d554d 5121 die "internal error" if $opt =~ m/^unused/;
ffda963f 5122 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
5123 if (!defined($conf->{$opt})) {
5124 vmconfig_undelete_pending_option($conf, $opt);
ffda963f 5125 PVE::QemuConfig->write_config($vmid, $conf);
74479ee9 5126 } elsif (is_valid_drivename($opt)) {
3dc38fbb 5127 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
055d554d
DM
5128 vmconfig_undelete_pending_option($conf, $opt);
5129 delete $conf->{$opt};
ffda963f 5130 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM
5131 } else {
5132 vmconfig_undelete_pending_option($conf, $opt);
5133 delete $conf->{$opt};
ffda963f 5134 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM
5135 }
5136 }
5137
ffda963f 5138 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
5139
5140 foreach my $opt (keys %{$conf->{pending}}) { # add/change
ffda963f 5141 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
5142
5143 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5144 # skip if nothing changed
74479ee9 5145 } elsif (is_valid_drivename($opt)) {
055d554d
DM
5146 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5147 if defined($conf->{$opt});
5148 $conf->{$opt} = $conf->{pending}->{$opt};
5149 } else {
5150 $conf->{$opt} = $conf->{pending}->{$opt};
5151 }
5152
5153 delete $conf->{pending}->{$opt};
ffda963f 5154 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM
5155 }
5156}
5157
3eec5767
DM
5158my $safe_num_ne = sub {
5159 my ($a, $b) = @_;
5160
5161 return 0 if !defined($a) && !defined($b);
5162 return 1 if !defined($a);
5163 return 1 if !defined($b);
5164
5165 return $a != $b;
5166};
5167
5168my $safe_string_ne = sub {
5169 my ($a, $b) = @_;
5170
5171 return 0 if !defined($a) && !defined($b);
5172 return 1 if !defined($a);
5173 return 1 if !defined($b);
5174
5175 return $a ne $b;
5176};
5177
5178sub vmconfig_update_net {
d559309f 5179 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
3eec5767
DM
5180
5181 my $newnet = parse_net($value);
5182
5183 if ($conf->{$opt}) {
5184 my $oldnet = parse_net($conf->{$opt});
5185
5186 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5187 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5188 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5189 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5190
5191 # for non online change, we try to hot-unplug
7196b757 5192 die "skip\n" if !$hotplug;
3eec5767
DM
5193 vm_deviceunplug($vmid, $conf, $opt);
5194 } else {
5195
5196 die "internal error" if $opt !~ m/net(\d+)/;
5197 my $iface = "tap${vmid}i$1";
a1b7d579 5198
25088687
DM
5199 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5200 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
16d08ecf 5201 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
25088687 5202 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
3eec5767 5203 PVE::Network::tap_unplug($iface);
4f4fbeb0
WB
5204 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5205 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5206 # Rate can be applied on its own but any change above needs to
5207 # include the rate in tap_plug since OVS resets everything.
5208 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
3eec5767 5209 }
38c590d9 5210
25088687
DM
5211 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5212 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5213 }
5214
38c590d9 5215 return 1;
3eec5767
DM
5216 }
5217 }
a1b7d579 5218
7196b757 5219 if ($hotplug) {
d559309f 5220 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
38c590d9
DM
5221 } else {
5222 die "skip\n";
5223 }
3eec5767
DM
5224}
5225
a05cff86 5226sub vmconfig_update_disk {
d559309f 5227 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
a05cff86
DM
5228
5229 # fixme: do we need force?
5230
5231 my $drive = parse_drive($opt, $value);
5232
5233 if ($conf->{$opt}) {
5234
5235 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5236
5237 my $media = $drive->{media} || 'disk';
5238 my $oldmedia = $old_drive->{media} || 'disk';
5239 die "unable to change media type\n" if $media ne $oldmedia;
5240
5241 if (!drive_is_cdrom($old_drive)) {
5242
a1b7d579 5243 if ($drive->{file} ne $old_drive->{file}) {
a05cff86 5244
7196b757 5245 die "skip\n" if !$hotplug;
a05cff86
DM
5246
5247 # unplug and register as unused
5248 vm_deviceunplug($vmid, $conf, $opt);
5249 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
a1b7d579 5250
a05cff86
DM
5251 } else {
5252 # update existing disk
5253
5254 # skip non hotpluggable value
6ecfbb44 5255 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
22de899a 5256 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
6e11f143 5257 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
a05cff86
DM
5258 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5259 die "skip\n";
5260 }
5261
5262 # apply throttle
5263 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5264 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5265 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5266 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5267 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5268 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5269 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5270 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5271 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5272 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5273 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
9196a8ec
WB
5274 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5275 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5276 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5277 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5278 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5279 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5280 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
a1b7d579 5281
a05cff86
DM
5282 qemu_block_set_io_throttle($vmid,"drive-$opt",
5283 ($drive->{mbps} || 0)*1024*1024,
5284 ($drive->{mbps_rd} || 0)*1024*1024,
5285 ($drive->{mbps_wr} || 0)*1024*1024,
5286 $drive->{iops} || 0,
5287 $drive->{iops_rd} || 0,
5288 $drive->{iops_wr} || 0,
5289 ($drive->{mbps_max} || 0)*1024*1024,
5290 ($drive->{mbps_rd_max} || 0)*1024*1024,
5291 ($drive->{mbps_wr_max} || 0)*1024*1024,
5292 $drive->{iops_max} || 0,
5293 $drive->{iops_rd_max} || 0,
9196a8ec
WB
5294 $drive->{iops_wr_max} || 0,
5295 $drive->{bps_max_length} || 1,
5296 $drive->{bps_rd_max_length} || 1,
5297 $drive->{bps_wr_max_length} || 1,
5298 $drive->{iops_max_length} || 1,
5299 $drive->{iops_rd_max_length} || 1,
5300 $drive->{iops_wr_max_length} || 1);
a05cff86
DM
5301
5302 }
a1b7d579 5303
a05cff86
DM
5304 return 1;
5305 }
4de1bb25
DM
5306
5307 } else { # cdrom
a1b7d579 5308
4de1bb25
DM
5309 if ($drive->{file} eq 'none') {
5310 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
2d9ddec5
WB
5311 if (drive_is_cloudinit($old_drive)) {
5312 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5313 }
4de1bb25
DM
5314 } else {
5315 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5316 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5317 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5318 }
a1b7d579 5319
34758d66 5320 return 1;
a05cff86
DM
5321 }
5322 }
5323 }
5324
a1b7d579 5325 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4de1bb25 5326 # hotplug new disks
f7b4356f 5327 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
d559309f 5328 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
a05cff86
DM
5329}
5330
1e3baf05 5331sub vm_start {
ba9e1000 5332 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
2189246c 5333 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
1e3baf05 5334
ffda963f
FG
5335 PVE::QemuConfig->lock_config($vmid, sub {
5336 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
1e3baf05 5337
ffda963f 5338 die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
3dcb98d5 5339
7ceade4c
DC
5340 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5341
5342 PVE::QemuConfig->check_lock($conf)
5343 if !($skiplock || $is_suspended);
1e3baf05 5344
7e8dcf2c 5345 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
1e3baf05 5346
64457ed4
DC
5347 # clean up leftover reboot request files
5348 eval { clear_reboot_request($vmid); };
5349 warn $@ if $@;
5350
055d554d 5351 if (!$statefile && scalar(keys %{$conf->{pending}})) {
3a11fadb 5352 vmconfig_apply_pending($vmid, $conf, $storecfg);
ffda963f 5353 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
5354 }
5355
0c9a7596
AD
5356 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5357
6c47d546
DM
5358 my $defaults = load_defaults();
5359
5360 # set environment variable useful inside network script
5361 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5362
2189246c
AD
5363 my $local_volumes = {};
5364
3b4cf0f0 5365 if ($targetstorage) {
2189246c
AD
5366 foreach_drive($conf, sub {
5367 my ($ds, $drive) = @_;
5368
5369 return if drive_is_cdrom($drive);
5370
5371 my $volid = $drive->{file};
5372
5373 return if !$volid;
5374
5375 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5376
5377 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5378 return if $scfg->{shared};
5379 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5380 });
5381
5382 my $format = undef;
5383
5384 foreach my $opt (sort keys %$local_volumes) {
5385
5386 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5387 my $drive = parse_drive($opt, $conf->{$opt});
5388
5389 #if remote storage is specified, use default format
5390 if ($targetstorage && $targetstorage ne "1") {
5391 $storeid = $targetstorage;
5392 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5393 $format = $defFormat;
5394 } else {
5395 #else we use same format than original
5396 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5397 $format = qemu_img_format($scfg, $volid);
5398 }
5399
5400 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5401 my $newdrive = $drive;
5402 $newdrive->{format} = $format;
5403 $newdrive->{file} = $newvolid;
5404 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5405 $local_volumes->{$opt} = $drivestr;
5406 #pass drive to conf for command line
5407 $conf->{$opt} = $drivestr;
5408 }
5409 }
5410
9e784b11
DC
5411 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5412
7ceade4c
DC
5413 if ($is_suspended) {
5414 # enforce machine type on suspended vm to ensure HW compatibility
5415 $forcemachine = $conf->{runningmachine};
b0a9a385 5416 print "Resuming suspended VM\n";
7ceade4c
DC
5417 }
5418
67812f9c 5419 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
6c47d546 5420
5bc1e039 5421 my $migrate_uri;
1e3baf05
DM
5422 if ($statefile) {
5423 if ($statefile eq 'tcp') {
5bc1e039
SP
5424 my $localip = "localhost";
5425 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
af0eba7e 5426 my $nodename = PVE::INotify::nodename();
2de2d6f7 5427
b7a5a225
TL
5428 if (!defined($migration_type)) {
5429 if (defined($datacenterconf->{migration}->{type})) {
5430 $migration_type = $datacenterconf->{migration}->{type};
5431 } else {
5432 $migration_type = 'secure';
5433 }
5434 }
5435
2de2d6f7
TL
5436 if ($migration_type eq 'insecure') {
5437 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5438 if ($migrate_network_addr) {
5439 $localip = $migrate_network_addr;
5440 } else {
5bc1e039 5441 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
2de2d6f7
TL
5442 }
5443
5444 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5bc1e039 5445 }
2de2d6f7 5446
af0eba7e 5447 my $pfamily = PVE::Tools::get_host_address_family($nodename);
a447e92c 5448 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
407e0b8b 5449 $migrate_uri = "tcp:${localip}:${migrate_port}";
6c47d546
DM
5450 push @$cmd, '-incoming', $migrate_uri;
5451 push @$cmd, '-S';
1c9d54bf
TL
5452
5453 } elsif ($statefile eq 'unix') {
5454 # should be default for secure migrations as a ssh TCP forward
5455 # tunnel is not deterministic reliable ready and fails regurarly
5456 # to set up in time, so use UNIX socket forwards
54323eed
TL
5457 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5458 unlink $socket_addr;
5459
5460 $migrate_uri = "unix:$socket_addr";
1c9d54bf
TL
5461
5462 push @$cmd, '-incoming', $migrate_uri;
5463 push @$cmd, '-S';
5464
5c1d42b7 5465 } elsif (-e $statefile) {
6c47d546 5466 push @$cmd, '-loadstate', $statefile;
5c1d42b7
TL
5467 } else {
5468 my $statepath = PVE::Storage::path($storecfg, $statefile);
5469 push @$vollist, $statepath;
5470 push @$cmd, '-loadstate', $statepath;
1e3baf05 5471 }
91bd6c90
DM
5472 } elsif ($paused) {
5473 push @$cmd, '-S';
1e3baf05
DM
5474 }
5475
1e3baf05 5476 # host pci devices
040b06b7
DA
5477 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5478 my $d = parse_hostpci($conf->{"hostpci$i"});
5479 next if !$d;
b1f72af6
AD
5480 my $pcidevices = $d->{pciid};
5481 foreach my $pcidevice (@$pcidevices) {
2fd24788 5482 my $pciid = $pcidevice->{id};
000fc0a2 5483
b71351a7
DC
5484 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5485 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
b1f72af6 5486 die "no pci device info for device '$pciid'\n" if !$info;
6ab45bd7
DC
5487
5488 if ($d->{mdev}) {
5489 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5490 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5491 } else {
5492 die "can't unbind/bind pci group to vfio '$pciid'\n"
5493 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5494 die "can't reset pci device '$pciid'\n"
5495 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5496 }
b1f72af6 5497 }
040b06b7 5498 }
1e3baf05
DM
5499
5500 PVE::Storage::activate_volumes($storecfg, $vollist);
5501
4f8cfa19
WB
5502 eval {
5503 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5504 outfunc => sub {}, errfunc => sub {});
5505 };
5506 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5507 # timeout should be more than enough here...
5508 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
2b401189 5509
8e59d952
WB
5510 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5511 : $defaults->{cpuunits};
5512
ccb2e2ea 5513 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
f38de678 5514 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
7023f3ea
AD
5515
5516 my %properties = (
5517 Slice => 'qemu.slice',
5518 KillMode => 'none',
5519 CPUShares => $cpuunits
5520 );
5521
5522 if (my $cpulimit = $conf->{cpulimit}) {
5523 $properties{CPUQuota} = int($cpulimit * 100);
5524 }
5525 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5526
503308ed
WB
5527 my $run_qemu = sub {
5528 PVE::Tools::run_fork sub {
d04d6af1 5529 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
503308ed
WB
5530 run_command($cmd, %run_params);
5531 };
5532 };
5533
7023f3ea
AD
5534 if ($conf->{hugepages}) {
5535
5536 my $code = sub {
5537 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5538 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5539
5540 PVE::QemuServer::Memory::hugepages_mount();
5541 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5542
503308ed 5543 eval { $run_qemu->() };
7023f3ea
AD
5544 if (my $err = $@) {
5545 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5546 die $err;
5547 }
5548
5549 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5550 };
5551 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5552
5553 } else {
503308ed 5554 eval { $run_qemu->() };
7023f3ea 5555 }
77cde36b
DC
5556
5557 if (my $err = $@) {
5558 # deactivate volumes if start fails
5559 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5560 die "start failed: $err";
5561 }
1e3baf05 5562
5bc1e039 5563 print "migration listens on $migrate_uri\n" if $migrate_uri;
afdb31d5 5564
b37ecfe6 5565 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
95381ce0 5566 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
8c609afd 5567 warn $@ if $@;
62de2cbd
DM
5568 }
5569
2189246c
AD
5570 #start nbd server for storage migration
5571 if ($targetstorage) {
2189246c
AD
5572 my $nodename = PVE::INotify::nodename();
5573 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5574 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5575 my $pfamily = PVE::Tools::get_host_address_family($nodename);
a447e92c 5576 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
2189246c 5577
a447e92c 5578 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
2189246c
AD
5579
5580 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5581
5582 foreach my $opt (sort keys %$local_volumes) {
5583 my $volid = $local_volumes->{$opt};
5584 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
a447e92c 5585 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
2189246c
AD
5586 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5587 }
5588 }
5589
1d794448 5590 if ($migratedfrom) {
a89fded1 5591 eval {
8e90138a 5592 set_migration_caps($vmid);
a89fded1 5593 };
1d794448 5594 warn $@ if $@;
a89fded1 5595
1d794448
DM
5596 if ($spice_port) {
5597 print "spice listens on port $spice_port\n";
5598 if ($spice_ticket) {
8e90138a
DM
5599 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5600 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
95a4b4a9
AD
5601 }
5602 }
5603
1d794448 5604 } else {
51153f86
DC
5605 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5606 if !$statefile && $conf->{balloon};
25088687
DM
5607
5608 foreach my $opt (keys %$conf) {
5609 next if $opt !~ m/^net\d+$/;
5610 my $nicconf = parse_net($conf->{$opt});
5611 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5612 }
e18b0b99 5613 }
a1b7d579 5614
eb065317
AD
5615 vm_mon_cmd_nocheck($vmid, 'qom-set',
5616 path => "machine/peripheral/balloon0",
5617 property => "guest-stats-polling-interval",
5618 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5619
7ceade4c 5620 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
b0a9a385 5621 print "Resumed VM, removing state\n";
7ceade4c
DC
5622 delete $conf->@{qw(lock vmstate runningmachine)};
5623 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5624 PVE::Storage::vdisk_free($storecfg, $vmstate);
5625 PVE::QemuConfig->write_config($vmid, $conf);
5626 }
5627
9e784b11 5628 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
1e3baf05
DM
5629 });
5630}
5631
0eedc444
AD
5632sub vm_mon_cmd {
5633 my ($vmid, $execute, %params) = @_;
5634
26f11676
DM
5635 my $cmd = { execute => $execute, arguments => \%params };
5636 vm_qmp_command($vmid, $cmd);
0eedc444
AD
5637}
5638
5639sub vm_mon_cmd_nocheck {
5640 my ($vmid, $execute, %params) = @_;
5641
26f11676
DM
5642 my $cmd = { execute => $execute, arguments => \%params };
5643 vm_qmp_command($vmid, $cmd, 1);
0eedc444
AD
5644}
5645
c971c4f2 5646sub vm_qmp_command {
c5a07de5 5647 my ($vmid, $cmd, $nocheck) = @_;
97d62eb7 5648
c971c4f2 5649 my $res;
26f11676 5650
14db5366 5651 my $timeout;
1688362d
SR
5652 if ($cmd->{arguments}) {
5653 $timeout = delete $cmd->{arguments}->{timeout};
14db5366 5654 }
be190583 5655
c971c4f2
AD
5656 eval {
5657 die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
7a6c2150
DM
5658 my $sname = qmp_socket($vmid);
5659 if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
c5a07de5 5660 my $qmpclient = PVE::QMPClient->new();
dab36e1e 5661
14db5366 5662 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
dab36e1e
DM
5663 } else {
5664 die "unable to open monitor socket\n";
5665 }
c971c4f2 5666 };
26f11676 5667 if (my $err = $@) {
c971c4f2
AD
5668 syslog("err", "VM $vmid qmp command failed - $err");
5669 die $err;
5670 }
5671
5672 return $res;
5673}
5674
9df5cbcc
DM
5675sub vm_human_monitor_command {
5676 my ($vmid, $cmdline) = @_;
5677
f5eb281a 5678 my $cmd = {
9df5cbcc
DM
5679 execute => 'human-monitor-command',
5680 arguments => { 'command-line' => $cmdline},
5681 };
5682
5683 return vm_qmp_command($vmid, $cmd);
5684}
5685
1e3baf05 5686sub vm_commandline {
b14477e7 5687 my ($storecfg, $vmid, $snapname) = @_;
1e3baf05 5688
ffda963f 5689 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5690
b14477e7
RV
5691 if ($snapname) {
5692 my $snapshot = $conf->{snapshots}->{$snapname};
87d92707
TL
5693 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5694
5695 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
b14477e7 5696
b14477e7
RV
5697 $conf = $snapshot;
5698 }
5699
1e3baf05
DM
5700 my $defaults = load_defaults();
5701
6b64503e 5702 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
1e3baf05 5703
5930c1ff 5704 return PVE::Tools::cmd2string($cmd);
1e3baf05
DM
5705}
5706
5707sub vm_reset {
5708 my ($vmid, $skiplock) = @_;
5709
ffda963f 5710 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5711
ffda963f 5712 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5713
ffda963f 5714 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 5715
816e2c4a 5716 vm_mon_cmd($vmid, "system_reset");
ff1a2432
DM
5717 });
5718}
5719
5720sub get_vm_volumes {
5721 my ($conf) = @_;
1e3baf05 5722
ff1a2432 5723 my $vollist = [];
d5769dc2 5724 foreach_volid($conf, sub {
392f8b5d 5725 my ($volid, $attr) = @_;
ff1a2432 5726
d5769dc2 5727 return if $volid =~ m|^/|;
ff1a2432 5728
d5769dc2
DM
5729 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5730 return if !$sid;
ff1a2432
DM
5731
5732 push @$vollist, $volid;
1e3baf05 5733 });
ff1a2432
DM
5734
5735 return $vollist;
5736}
5737
5738sub vm_stop_cleanup {
70b04821 5739 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
ff1a2432 5740
745fed70 5741 eval {
ff1a2432 5742
254575e9
DM
5743 if (!$keepActive) {
5744 my $vollist = get_vm_volumes($conf);
5745 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5746 }
a1b7d579 5747
ab6a046f 5748 foreach my $ext (qw(mon qmp pid vnc qga)) {
961bfcb2
DM
5749 unlink "/var/run/qemu-server/${vmid}.$ext";
5750 }
a1b7d579 5751
6dbcb073
DC
5752 if ($conf->{ivshmem}) {
5753 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4c5a6a24
TL
5754 # just delete it for now, VMs which have this already open do not
5755 # are affected, but new VMs will get a separated one. If this
5756 # becomes an issue we either add some sort of ref-counting or just
5757 # add a "don't delete on stop" flag to the ivshmem format.
6dbcb073
DC
5758 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5759 }
5760
6ab45bd7
DC
5761 foreach my $key (keys %$conf) {
5762 next if $key !~ m/^hostpci(\d+)$/;
5763 my $hostpciindex = $1;
5764 my $d = parse_hostpci($conf->{$key});
5765 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5766
5767 foreach my $pci (@{$d->{pciid}}) {
2fd24788 5768 my $pciid = $pci->{id};
6ab45bd7
DC
5769 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5770 }
5771 }
5772
70b04821 5773 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
745fed70
DM
5774 };
5775 warn $@ if $@; # avoid errors - just warn
1e3baf05
DM
5776}
5777
575d19da
DC
5778# call only in locked context
5779sub _do_vm_stop {
5780 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
9269013a 5781
575d19da
DC
5782 my $pid = check_running($vmid, $nocheck);
5783 return if !$pid;
1e3baf05 5784
575d19da
DC
5785 my $conf;
5786 if (!$nocheck) {
5787 $conf = PVE::QemuConfig->load_config($vmid);
5788 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5789 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5790 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5791 $timeout = $opts->{down} if $opts->{down};
e6c3b671 5792 }
575d19da
DC
5793 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5794 }
19672434 5795
575d19da
DC
5796 eval {
5797 if ($shutdown) {
5798 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5799 vm_qmp_command($vmid, {
0eb21691
SR
5800 execute => "guest-shutdown",
5801 arguments => { timeout => $timeout }
5802 }, $nocheck);
9269013a 5803 } else {
575d19da 5804 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
1e3baf05
DM
5805 }
5806 } else {
575d19da 5807 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
1e3baf05 5808 }
575d19da
DC
5809 };
5810 my $err = $@;
1e3baf05 5811
575d19da
DC
5812 if (!$err) {
5813 $timeout = 60 if !defined($timeout);
1e3baf05
DM
5814
5815 my $count = 0;
e6c3b671 5816 while (($count < $timeout) && check_running($vmid, $nocheck)) {
1e3baf05
DM
5817 $count++;
5818 sleep 1;
5819 }
5820
5821 if ($count >= $timeout) {
575d19da
DC
5822 if ($force) {
5823 warn "VM still running - terminating now with SIGTERM\n";
5824 kill 15, $pid;
5825 } else {
5826 die "VM quit/powerdown failed - got timeout\n";
5827 }
5828 } else {
5829 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5830 return;
1e3baf05 5831 }
575d19da
DC
5832 } else {
5833 if ($force) {
5834 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5835 kill 15, $pid;
5836 } else {
5837 die "VM quit/powerdown failed\n";
5838 }
5839 }
5840
5841 # wait again
5842 $timeout = 10;
5843
5844 my $count = 0;
5845 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5846 $count++;
5847 sleep 1;
5848 }
5849
5850 if ($count >= $timeout) {
5851 warn "VM still running - terminating now with SIGKILL\n";
5852 kill 9, $pid;
5853 sleep 1;
5854 }
1e3baf05 5855
575d19da
DC
5856 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5857}
5858
5859# Note: use $nocheck to skip tests if VM configuration file exists.
5860# We need that when migration VMs to other nodes (files already moved)
5861# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5862sub vm_stop {
5863 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5864
5865 $force = 1 if !defined($force) && !$shutdown;
5866
5867 if ($migratedfrom){
5868 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5869 kill 15, $pid if $pid;
5870 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5871 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5872 return;
5873 }
5874
5875 PVE::QemuConfig->lock_config($vmid, sub {
5876 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
ff1a2432 5877 });
1e3baf05
DM
5878}
5879
165411f0
DC
5880sub vm_reboot {
5881 my ($vmid, $timeout) = @_;
5882
5883 PVE::QemuConfig->lock_config($vmid, sub {
5884
5885 # only reboot if running, as qmeventd starts it again on a stop event
5886 return if !check_running($vmid);
5887
5888 create_reboot_request($vmid);
5889
5890 my $storecfg = PVE::Storage::config();
5891 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5892
5893 });
5894}
5895
1e3baf05 5896sub vm_suspend {
48b4cdc2 5897 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
159719e5
DC
5898
5899 my $conf;
5900 my $path;
5901 my $storecfg;
5902 my $vmstate;
1e3baf05 5903
ffda963f 5904 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5905
159719e5 5906 $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5907
159719e5 5908 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
e79706d4 5909 PVE::QemuConfig->check_lock($conf)
159719e5
DC
5910 if !($skiplock || $is_backing_up);
5911
5912 die "cannot suspend to disk during backup\n"
5913 if $is_backing_up && $includestate;
bcb7c9cf 5914
159719e5
DC
5915 if ($includestate) {
5916 $conf->{lock} = 'suspending';
5917 my $date = strftime("%Y-%m-%d", localtime(time()));
5918 $storecfg = PVE::Storage::config();
48b4cdc2 5919 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
159719e5
DC
5920 $path = PVE::Storage::path($storecfg, $vmstate);
5921 PVE::QemuConfig->write_config($vmid, $conf);
5922 } else {
5923 vm_mon_cmd($vmid, "stop");
5924 }
1e3baf05 5925 });
159719e5
DC
5926
5927 if ($includestate) {
5928 # save vm state
5929 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5930
5931 eval {
5932 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5933 for(;;) {
5934 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5935 if (!$state->{status}) {
5936 die "savevm not active\n";
5937 } elsif ($state->{status} eq 'active') {
5938 sleep(1);
5939 next;
5940 } elsif ($state->{status} eq 'completed') {
b0a9a385 5941 print "State saved, quitting\n";
159719e5
DC
5942 last;
5943 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5944 die "query-savevm failed with error '$state->{error}'\n"
5945 } else {
5946 die "query-savevm returned status '$state->{status}'\n";
5947 }
5948 }
5949 };
5950 my $err = $@;
5951
5952 PVE::QemuConfig->lock_config($vmid, sub {
5953 $conf = PVE::QemuConfig->load_config($vmid);
5954 if ($err) {
5955 # cleanup, but leave suspending lock, to indicate something went wrong
5956 eval {
5957 vm_mon_cmd($vmid, "savevm-end");
5958 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5959 PVE::Storage::vdisk_free($storecfg, $vmstate);
5960 delete $conf->@{qw(vmstate runningmachine)};
5961 PVE::QemuConfig->write_config($vmid, $conf);
5962 };
5963 warn $@ if $@;
5964 die $err;
5965 }
5966
5967 die "lock changed unexpectedly\n"
5968 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5969
5970 vm_qmp_command($vmid, { execute => "quit" });
5971 $conf->{lock} = 'suspended';
5972 PVE::QemuConfig->write_config($vmid, $conf);
5973 });
5974 }
1e3baf05
DM
5975}
5976
5977sub vm_resume {
289e0b85 5978 my ($vmid, $skiplock, $nocheck) = @_;
1e3baf05 5979
ffda963f 5980 PVE::QemuConfig->lock_config($vmid, sub {
3e24733b
FG
5981 my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5982 my $res = $vm_mon_cmd->($vmid, 'query-status');
c2786bed
DC
5983 my $resume_cmd = 'cont';
5984
5985 if ($res->{status} && $res->{status} eq 'suspended') {
5986 $resume_cmd = 'system_wakeup';
5987 }
5988
289e0b85 5989 if (!$nocheck) {
1e3baf05 5990
ffda963f 5991 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5992
e79706d4
FG
5993 PVE::QemuConfig->check_lock($conf)
5994 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
289e0b85 5995 }
3e24733b
FG
5996
5997 $vm_mon_cmd->($vmid, $resume_cmd);
1e3baf05
DM
5998 });
5999}
6000
5fdbe4f0
DM
6001sub vm_sendkey {
6002 my ($vmid, $skiplock, $key) = @_;
1e3baf05 6003
ffda963f 6004 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 6005
ffda963f 6006 my $conf = PVE::QemuConfig->load_config($vmid);
f5eb281a 6007
7b7c6d1b 6008 # there is no qmp command, so we use the human monitor command
d30820d6
DC
6009 my $res = vm_human_monitor_command($vmid, "sendkey $key");
6010 die $res if $res ne '';
1e3baf05
DM
6011 });
6012}
6013
6014sub vm_destroy {
6015 my ($storecfg, $vmid, $skiplock) = @_;
6016
ffda963f 6017 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 6018
ffda963f 6019 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 6020
ff1a2432 6021 if (!check_running($vmid)) {
15cc8784 6022 destroy_vm($storecfg, $vmid, undef, $skiplock);
ff1a2432
DM
6023 } else {
6024 die "VM $vmid is running - destroy failed\n";
1e3baf05
DM
6025 }
6026 });
6027}
6028
3e16d5fc
DM
6029# vzdump restore implementaion
6030
ed221350 6031sub tar_archive_read_firstfile {
3e16d5fc 6032 my $archive = shift;
afdb31d5 6033
3e16d5fc
DM
6034 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6035
6036 # try to detect archive type first
387ba257 6037 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
3e16d5fc 6038 die "unable to open file '$archive'\n";
387ba257 6039 my $firstfile = <$fh>;
3e16d5fc 6040 kill 15, $pid;
387ba257 6041 close $fh;
3e16d5fc
DM
6042
6043 die "ERROR: archive contaions no data\n" if !$firstfile;
6044 chomp $firstfile;
6045
6046 return $firstfile;
6047}
6048
ed221350
DM
6049sub tar_restore_cleanup {
6050 my ($storecfg, $statfile) = @_;
3e16d5fc
DM
6051
6052 print STDERR "starting cleanup\n";
6053
6054 if (my $fd = IO::File->new($statfile, "r")) {
6055 while (defined(my $line = <$fd>)) {
6056 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6057 my $volid = $2;
6058 eval {
6059 if ($volid =~ m|^/|) {
6060 unlink $volid || die 'unlink failed\n';
6061 } else {
ed221350 6062 PVE::Storage::vdisk_free($storecfg, $volid);
3e16d5fc 6063 }
afdb31d5 6064 print STDERR "temporary volume '$volid' sucessfuly removed\n";
3e16d5fc
DM
6065 };
6066 print STDERR "unable to cleanup '$volid' - $@" if $@;
6067 } else {
6068 print STDERR "unable to parse line in statfile - $line";
afdb31d5 6069 }
3e16d5fc
DM
6070 }
6071 $fd->close();
6072 }
6073}
6074
6075sub restore_archive {
a0d1b1a2 6076 my ($archive, $vmid, $user, $opts) = @_;
3e16d5fc 6077
91bd6c90
DM
6078 my $format = $opts->{format};
6079 my $comp;
6080
6081 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6082 $format = 'tar' if !$format;
6083 $comp = 'gzip';
6084 } elsif ($archive =~ m/\.tar$/) {
6085 $format = 'tar' if !$format;
6086 } elsif ($archive =~ m/.tar.lzo$/) {
6087 $format = 'tar' if !$format;
6088 $comp = 'lzop';
6089 } elsif ($archive =~ m/\.vma$/) {
6090 $format = 'vma' if !$format;
6091 } elsif ($archive =~ m/\.vma\.gz$/) {
6092 $format = 'vma' if !$format;
6093 $comp = 'gzip';
6094 } elsif ($archive =~ m/\.vma\.lzo$/) {
6095 $format = 'vma' if !$format;
6096 $comp = 'lzop';
6097 } else {
6098 $format = 'vma' if !$format; # default
6099 }
6100
6101 # try to detect archive format
6102 if ($format eq 'tar') {
6103 return restore_tar_archive($archive, $vmid, $user, $opts);
6104 } else {
6105 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6106 }
6107}
6108
6109sub restore_update_config_line {
6110 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6111
6112 return if $line =~ m/^\#qmdump\#/;
6113 return if $line =~ m/^\#vzdump\#/;
6114 return if $line =~ m/^lock:/;
6115 return if $line =~ m/^unused\d+:/;
6116 return if $line =~ m/^parent:/;
6117
b5b99790 6118 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
91bd6c90
DM
6119 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6120 # try to convert old 1.X settings
6121 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6122 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6123 my ($model, $macaddr) = split(/\=/, $devconfig);
b5b99790 6124 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
91bd6c90
DM
6125 my $net = {
6126 model => $model,
6127 bridge => "vmbr$ind",
6128 macaddr => $macaddr,
6129 };
6130 my $netstr = print_net($net);
6131
6132 print $outfd "net$cookie->{netcount}: $netstr\n";
6133 $cookie->{netcount}++;
6134 }
6135 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6136 my ($id, $netstr) = ($1, $2);
6137 my $net = parse_net($netstr);
b5b99790 6138 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
91bd6c90
DM
6139 $netstr = print_net($net);
6140 print $outfd "$id: $netstr\n";
6470743f 6141 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
91bd6c90 6142 my $virtdev = $1;
907ea891 6143 my $value = $3;
d9faf790
WB
6144 my $di = parse_drive($virtdev, $value);
6145 if (defined($di->{backup}) && !$di->{backup}) {
91bd6c90 6146 print $outfd "#$line";
c0f7406e 6147 } elsif ($map->{$virtdev}) {
8fd57431 6148 delete $di->{format}; # format can change on restore
91bd6c90 6149 $di->{file} = $map->{$virtdev};
ed221350 6150 $value = print_drive($vmid, $di);
91bd6c90
DM
6151 print $outfd "$virtdev: $value\n";
6152 } else {
6153 print $outfd $line;
6154 }
1a0c2f03 6155 } elsif (($line =~ m/^vmgenid: (.*)/)) {
babecffe 6156 my $vmgenid = $1;
6ee499ff 6157 if ($vmgenid ne '0') {
1a0c2f03 6158 # always generate a new vmgenid if there was a valid one setup
6ee499ff
DC
6159 $vmgenid = generate_uuid();
6160 }
1a0c2f03 6161 print $outfd "vmgenid: $vmgenid\n";
19a5dd55
WL
6162 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6163 my ($uuid, $uuid_str);
6164 UUID::generate($uuid);
6165 UUID::unparse($uuid, $uuid_str);
6166 my $smbios1 = parse_smbios1($2);
6167 $smbios1->{uuid} = $uuid_str;
6168 print $outfd $1.print_smbios1($smbios1)."\n";
91bd6c90
DM
6169 } else {
6170 print $outfd $line;
6171 }
6172}
6173
6174sub scan_volids {
6175 my ($cfg, $vmid) = @_;
6176
6177 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6178
6179 my $volid_hash = {};
6180 foreach my $storeid (keys %$info) {
6181 foreach my $item (@{$info->{$storeid}}) {
6182 next if !($item->{volid} && $item->{size});
5996a936 6183 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
91bd6c90
DM
6184 $volid_hash->{$item->{volid}} = $item;
6185 }
6186 }
6187
6188 return $volid_hash;
6189}
6190
77019edf
WB
6191sub is_volume_in_use {
6192 my ($storecfg, $conf, $skip_drive, $volid) = @_;
a8e2f942 6193
77019edf 6194 my $path = PVE::Storage::path($storecfg, $volid);
a8e2f942
DM
6195
6196 my $scan_config = sub {
6197 my ($cref, $snapname) = @_;
6198
6199 foreach my $key (keys %$cref) {
6200 my $value = $cref->{$key};
74479ee9 6201 if (is_valid_drivename($key)) {
a8e2f942
DM
6202 next if $skip_drive && $key eq $skip_drive;
6203 my $drive = parse_drive($key, $value);
6204 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
77019edf 6205 return 1 if $volid eq $drive->{file};
a8e2f942 6206 if ($drive->{file} =~ m!^/!) {
77019edf 6207 return 1 if $drive->{file} eq $path;
a8e2f942
DM
6208 } else {
6209 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6210 next if !$storeid;
6211 my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6212 next if !$scfg;
77019edf 6213 return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
a8e2f942
DM
6214 }
6215 }
6216 }
77019edf
WB
6217
6218 return 0;
a8e2f942
DM
6219 };
6220
77019edf 6221 return 1 if &$scan_config($conf);
a8e2f942
DM
6222
6223 undef $skip_drive;
6224
77019edf
WB
6225 foreach my $snapname (keys %{$conf->{snapshots}}) {
6226 return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
a8e2f942
DM
6227 }
6228
77019edf 6229 return 0;
a8e2f942
DM
6230}
6231
91bd6c90
DM
6232sub update_disksize {
6233 my ($vmid, $conf, $volid_hash) = @_;
be190583 6234
91bd6c90 6235 my $changes;
53b81297 6236 my $prefix = "VM $vmid:";
91bd6c90 6237
c449137a
DC
6238 # used and unused disks
6239 my $referenced = {};
91bd6c90 6240
5996a936
DM
6241 # Note: it is allowed to define multiple storages with same path (alias), so
6242 # we need to check both 'volid' and real 'path' (two different volid can point
6243 # to the same path).
6244
c449137a 6245 my $referencedpath = {};
be190583 6246
91bd6c90
DM
6247 # update size info
6248 foreach my $opt (keys %$conf) {
74479ee9 6249 if (is_valid_drivename($opt)) {
ed221350 6250 my $drive = parse_drive($opt, $conf->{$opt});
91bd6c90
DM
6251 my $volid = $drive->{file};
6252 next if !$volid;
6253
c449137a 6254 $referenced->{$volid} = 1;
be190583 6255 if ($volid_hash->{$volid} &&
5996a936 6256 (my $path = $volid_hash->{$volid}->{path})) {
c449137a 6257 $referencedpath->{$path} = 1;
5996a936 6258 }
91bd6c90 6259
ed221350 6260 next if drive_is_cdrom($drive);
91bd6c90
DM
6261 next if !$volid_hash->{$volid};
6262
6263 $drive->{size} = $volid_hash->{$volid}->{size};
7a907ce6
DM
6264 my $new = print_drive($vmid, $drive);
6265 if ($new ne $conf->{$opt}) {
6266 $changes = 1;
6267 $conf->{$opt} = $new;
53b81297 6268 print "$prefix update disk '$opt' information.\n";
7a907ce6 6269 }
91bd6c90
DM
6270 }
6271 }
6272
5996a936
DM
6273 # remove 'unusedX' entry if volume is used
6274 foreach my $opt (keys %$conf) {
6275 next if $opt !~ m/^unused\d+$/;
6276 my $volid = $conf->{$opt};
6277 my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
c449137a 6278 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
53b81297 6279 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5996a936
DM
6280 $changes = 1;
6281 delete $conf->{$opt};
6282 }
c449137a
DC
6283
6284 $referenced->{$volid} = 1;
6285 $referencedpath->{$path} = 1 if $path;
5996a936
DM
6286 }
6287
91bd6c90
DM
6288 foreach my $volid (sort keys %$volid_hash) {
6289 next if $volid =~ m/vm-$vmid-state-/;
c449137a 6290 next if $referenced->{$volid};
5996a936
DM
6291 my $path = $volid_hash->{$volid}->{path};
6292 next if !$path; # just to be sure
c449137a 6293 next if $referencedpath->{$path};
91bd6c90 6294 $changes = 1;
53b81297
TL
6295 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6296 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
c449137a 6297 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
91bd6c90
DM
6298 }
6299
6300 return $changes;
6301}
6302
6303sub rescan {
9224dcee 6304 my ($vmid, $nolock, $dryrun) = @_;
91bd6c90 6305
20519efc 6306 my $cfg = PVE::Storage::config();
91bd6c90 6307
b9a1a3ab
TL
6308 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6309 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
4771526a
AA
6310 foreach my $stor (keys %{$cfg->{ids}}) {
6311 delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6312 }
6313
53b81297 6314 print "rescan volumes...\n";
91bd6c90
DM
6315 my $volid_hash = scan_volids($cfg, $vmid);
6316
6317 my $updatefn = sub {
6318 my ($vmid) = @_;
6319
ffda963f 6320 my $conf = PVE::QemuConfig->load_config($vmid);
be190583 6321
ffda963f 6322 PVE::QemuConfig->check_lock($conf);
91bd6c90 6323
03da3f0d
DM
6324 my $vm_volids = {};
6325 foreach my $volid (keys %$volid_hash) {
6326 my $info = $volid_hash->{$volid};
6327 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6328 }
6329
6330 my $changes = update_disksize($vmid, $conf, $vm_volids);
91bd6c90 6331
9224dcee 6332 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
91bd6c90
DM
6333 };
6334
6335 if (defined($vmid)) {
6336 if ($nolock) {
6337 &$updatefn($vmid);
6338 } else {
ffda963f 6339 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
91bd6c90
DM
6340 }
6341 } else {
6342 my $vmlist = config_list();
6343 foreach my $vmid (keys %$vmlist) {
6344 if ($nolock) {
6345 &$updatefn($vmid);
6346 } else {
ffda963f 6347 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
be190583 6348 }
91bd6c90
DM
6349 }
6350 }
6351}
6352
6353sub restore_vma_archive {
6354 my ($archive, $vmid, $user, $opts, $comp) = @_;
6355
91bd6c90
DM
6356 my $readfrom = $archive;
6357
7c536e11
WB
6358 my $cfg = PVE::Storage::config();
6359 my $commands = [];
6360 my $bwlimit = $opts->{bwlimit};
6361
6362 my $dbg_cmdstring = '';
6363 my $add_pipe = sub {
6364 my ($cmd) = @_;
6365 push @$commands, $cmd;
6366 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6367 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
91bd6c90 6368 $readfrom = '-';
7c536e11
WB
6369 };
6370
6371 my $input = undef;
6372 if ($archive eq '-') {
6373 $input = '<&STDIN';
6374 } else {
6375 # If we use a backup from a PVE defined storage we also consider that
6376 # storage's rate limit:
6377 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6378 if (defined($volid)) {
6379 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6380 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6381 if ($readlimit) {
6382 print STDERR "applying read rate limit: $readlimit\n";
9444c6e4 6383 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7c536e11
WB
6384 $add_pipe->($cstream);
6385 }
6386 }
6387 }
6388
6389 if ($comp) {
6390 my $cmd;
91bd6c90 6391 if ($comp eq 'gzip') {
7c536e11 6392 $cmd = ['zcat', $readfrom];
91bd6c90 6393 } elsif ($comp eq 'lzop') {
7c536e11 6394 $cmd = ['lzop', '-d', '-c', $readfrom];
91bd6c90
DM
6395 } else {
6396 die "unknown compression method '$comp'\n";
6397 }
7c536e11 6398 $add_pipe->($cmd);
91bd6c90
DM
6399 }
6400
6401 my $tmpdir = "/var/tmp/vzdumptmp$$";
6402 rmtree $tmpdir;
6403
6404 # disable interrupts (always do cleanups)
5b97ef24
TL
6405 local $SIG{INT} =
6406 local $SIG{TERM} =
6407 local $SIG{QUIT} =
6408 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
91bd6c90
DM
6409
6410 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6411 POSIX::mkfifo($mapfifo, 0600);
6412 my $fifofh;
6413
6414 my $openfifo = sub {
6415 open($fifofh, '>', $mapfifo) || die $!;
6416 };
6417
7c536e11 6418 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
91bd6c90
DM
6419
6420 my $oldtimeout;
6421 my $timeout = 5;
6422
6423 my $devinfo = {};
6424
6425 my $rpcenv = PVE::RPCEnvironment::get();
6426
ffda963f 6427 my $conffile = PVE::QemuConfig->config_file($vmid);
91bd6c90
DM
6428 my $tmpfn = "$conffile.$$.tmp";
6429
ed221350 6430 # Note: $oldconf is undef if VM does not exists
ffda963f
FG
6431 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6432 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
ed221350 6433
7c536e11
WB
6434 my %storage_limits;
6435
91bd6c90
DM
6436 my $print_devmap = sub {
6437 my $virtdev_hash = {};
6438
6439 my $cfgfn = "$tmpdir/qemu-server.conf";
6440
6441 # we can read the config - that is already extracted
6442 my $fh = IO::File->new($cfgfn, "r") ||
6443 "unable to read qemu-server.conf - $!\n";
6444
6738ab9c 6445 my $fwcfgfn = "$tmpdir/qemu-server.fw";
3457d090
WL
6446 if (-f $fwcfgfn) {
6447 my $pve_firewall_dir = '/etc/pve/firewall';
6448 mkdir $pve_firewall_dir; # make sure the dir exists
6449 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6450 }
6738ab9c 6451
91bd6c90
DM
6452 while (defined(my $line = <$fh>)) {
6453 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6454 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6455 die "archive does not contain data for drive '$virtdev'\n"
6456 if !$devinfo->{$devname};
6457 if (defined($opts->{storage})) {
6458 $storeid = $opts->{storage} || 'local';
6459 } elsif (!$storeid) {
6460 $storeid = 'local';
6461 }
6462 $format = 'raw' if !$format;
6463 $devinfo->{$devname}->{devname} = $devname;
6464 $devinfo->{$devname}->{virtdev} = $virtdev;
6465 $devinfo->{$devname}->{format} = $format;
6466 $devinfo->{$devname}->{storeid} = $storeid;
6467
be190583 6468 # check permission on storage
91bd6c90
DM
6469 my $pool = $opts->{pool}; # todo: do we need that?
6470 if ($user ne 'root@pam') {
6471 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6472 }
6473
7c536e11
WB
6474 $storage_limits{$storeid} = $bwlimit;
6475
91bd6c90 6476 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
c4ab3c55
ML
6477 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6478 my $virtdev = $1;
6479 my $drive = parse_drive($virtdev, $2);
6480 if (drive_is_cloudinit($drive)) {
6481 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6482 my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6483 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6484
6485 my $d = {
6486 format => $format,
6487 storeid => $opts->{storage} // $storeid,
6488 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6489 file => $drive->{file}, # to make drive_is_cloudinit check possible
6490 name => "vm-$vmid-cloudinit",
87056e18 6491 is_cloudinit => 1,
c4ab3c55
ML
6492 };
6493 $virtdev_hash->{$virtdev} = $d;
6494 }
91bd6c90
DM
6495 }
6496 }
6497
7c536e11
WB
6498 foreach my $key (keys %storage_limits) {
6499 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6500 next if !$limit;
6501 print STDERR "rate limit for storage $key: $limit KiB/s\n";
6502 $storage_limits{$key} = $limit * 1024;
6503 }
6504
91bd6c90 6505 foreach my $devname (keys %$devinfo) {
be190583
DM
6506 die "found no device mapping information for device '$devname'\n"
6507 if !$devinfo->{$devname}->{virtdev};
91bd6c90
DM
6508 }
6509
ed221350 6510 # create empty/temp config
be190583 6511 if ($oldconf) {
ed221350
DM
6512 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6513 foreach_drive($oldconf, sub {
6514 my ($ds, $drive) = @_;
6515
a82348eb 6516 return if drive_is_cdrom($drive, 1);
ed221350
DM
6517
6518 my $volid = $drive->{file};
ed221350
DM
6519 return if !$volid || $volid =~ m|^/|;
6520
6521 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6522 return if !$path || !$owner || ($owner != $vmid);
6523
6524 # Note: only delete disk we want to restore
6525 # other volumes will become unused
6526 if ($virtdev_hash->{$ds}) {
6b72854b
FG
6527 eval { PVE::Storage::vdisk_free($cfg, $volid); };
6528 if (my $err = $@) {
6529 warn $err;
6530 }
ed221350
DM
6531 }
6532 });
381b8fae 6533
2b2923ae 6534 # delete vmstate files, after the restore we have no snapshots anymore
381b8fae
DC
6535 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6536 my $snap = $oldconf->{snapshots}->{$snapname};
6537 if ($snap->{vmstate}) {
6538 eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6539 if (my $err = $@) {
6540 warn $err;
6541 }
6542 }
6543 }
ed221350
DM
6544 }
6545
6546 my $map = {};
91bd6c90
DM
6547 foreach my $virtdev (sort keys %$virtdev_hash) {
6548 my $d = $virtdev_hash->{$virtdev};
6549 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
7c536e11
WB
6550 my $storeid = $d->{storeid};
6551 my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6552
6553 my $map_opts = '';
6554 if (my $limit = $storage_limits{$storeid}) {
6555 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6556 }
8fd57431
DM
6557
6558 # test if requested format is supported
7c536e11 6559 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
8fd57431
DM
6560 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6561 $d->{format} = $defFormat if !$supported;
6562
87056e18
TL
6563 my $name;
6564 if ($d->{is_cloudinit}) {
6565 $name = $d->{name};
6566 $name .= ".$d->{format}" if $d->{format} ne 'raw';
c4ab3c55 6567 }
2b2923ae
TL
6568
6569 my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
91bd6c90
DM
6570 print STDERR "new volume ID is '$volid'\n";
6571 $d->{volid} = $volid;
91bd6c90 6572
2b2923ae 6573 PVE::Storage::activate_volumes($cfg, [$volid]);
5f96f4df 6574
91bd6c90 6575 my $write_zeros = 1;
88240a83 6576 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
91bd6c90
DM
6577 $write_zeros = 0;
6578 }
6579
87056e18
TL
6580 if (!$d->{is_cloudinit}) {
6581 my $path = PVE::Storage::path($cfg, $volid);
6582
c4ab3c55 6583 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
91bd6c90 6584
c4ab3c55
ML
6585 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6586 }
91bd6c90
DM
6587 $map->{$virtdev} = $volid;
6588 }
6589
6590 $fh->seek(0, 0) || die "seek failed - $!\n";
6591
6592 my $outfd = new IO::File ($tmpfn, "w") ||
6593 die "unable to write config for VM $vmid\n";
6594
6595 my $cookie = { netcount => 0 };
6596 while (defined(my $line = <$fh>)) {
be190583 6597 restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
91bd6c90
DM
6598 }
6599
6600 $fh->close();
6601 $outfd->close();
6602 };
6603
6604 eval {
6605 # enable interrupts
6cb0144a
EK
6606 local $SIG{INT} =
6607 local $SIG{TERM} =
6608 local $SIG{QUIT} =
6609 local $SIG{HUP} =
6610 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
91bd6c90
DM
6611 local $SIG{ALRM} = sub { die "got timeout\n"; };
6612
6613 $oldtimeout = alarm($timeout);
6614
6615 my $parser = sub {
6616 my $line = shift;
6617
6618 print "$line\n";
6619
6620 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6621 my ($dev_id, $size, $devname) = ($1, $2, $3);
6622 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6623 } elsif ($line =~ m/^CTIME: /) {
46f58b5f 6624 # we correctly received the vma config, so we can disable
3cf90d7a
DM
6625 # the timeout now for disk allocation (set to 10 minutes, so
6626 # that we always timeout if something goes wrong)
6627 alarm(600);
91bd6c90
DM
6628 &$print_devmap();
6629 print $fifofh "done\n";
6630 my $tmp = $oldtimeout || 0;
6631 $oldtimeout = undef;
6632 alarm($tmp);
6633 close($fifofh);
6634 }
6635 };
be190583 6636
7c536e11
WB
6637 print "restore vma archive: $dbg_cmdstring\n";
6638 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
91bd6c90
DM
6639 };
6640 my $err = $@;
6641
6642 alarm($oldtimeout) if $oldtimeout;
6643
5f96f4df
WL
6644 my $vollist = [];
6645 foreach my $devname (keys %$devinfo) {
6646 my $volid = $devinfo->{$devname}->{volid};
6647 push @$vollist, $volid if $volid;
6648 }
6649
5f96f4df
WL
6650 PVE::Storage::deactivate_volumes($cfg, $vollist);
6651
91bd6c90
DM
6652 unlink $mapfifo;
6653
6654 if ($err) {
6655 rmtree $tmpdir;
6656 unlink $tmpfn;
6657
91bd6c90
DM
6658 foreach my $devname (keys %$devinfo) {
6659 my $volid = $devinfo->{$devname}->{volid};
6660 next if !$volid;
6661 eval {
6662 if ($volid =~ m|^/|) {
6663 unlink $volid || die 'unlink failed\n';
6664 } else {
6665 PVE::Storage::vdisk_free($cfg, $volid);
6666 }
6667 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6668 };
6669 print STDERR "unable to cleanup '$volid' - $@" if $@;
6670 }
6671 die $err;
6672 }
6673
6674 rmtree $tmpdir;
ed221350
DM
6675
6676 rename($tmpfn, $conffile) ||
91bd6c90
DM
6677 die "unable to commit configuration file '$conffile'\n";
6678
ed221350
DM
6679 PVE::Cluster::cfs_update(); # make sure we read new file
6680
91bd6c90
DM
6681 eval { rescan($vmid, 1); };
6682 warn $@ if $@;
6683}
6684
6685sub restore_tar_archive {
6686 my ($archive, $vmid, $user, $opts) = @_;
6687
9c502e26 6688 if ($archive ne '-') {
ed221350 6689 my $firstfile = tar_archive_read_firstfile($archive);
9c502e26
DM
6690 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6691 if $firstfile ne 'qemu-server.conf';
6692 }
3e16d5fc 6693
20519efc 6694 my $storecfg = PVE::Storage::config();
ebb55558 6695
ed221350 6696 # destroy existing data - keep empty config
ffda963f 6697 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
ebb55558 6698 destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
ed221350 6699
3e16d5fc
DM
6700 my $tocmd = "/usr/lib/qemu-server/qmextract";
6701
2415a446 6702 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
a0d1b1a2 6703 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
3e16d5fc
DM
6704 $tocmd .= ' --prealloc' if $opts->{prealloc};
6705 $tocmd .= ' --info' if $opts->{info};
6706
a0d1b1a2 6707 # tar option "xf" does not autodetect compression when read from STDIN,
9c502e26 6708 # so we pipe to zcat
2415a446
DM
6709 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6710 PVE::Tools::shellquote("--to-command=$tocmd");
3e16d5fc
DM
6711
6712 my $tmpdir = "/var/tmp/vzdumptmp$$";
6713 mkpath $tmpdir;
6714
6715 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6716 local $ENV{VZDUMP_VMID} = $vmid;
a0d1b1a2 6717 local $ENV{VZDUMP_USER} = $user;
3e16d5fc 6718
ffda963f 6719 my $conffile = PVE::QemuConfig->config_file($vmid);
3e16d5fc
DM
6720 my $tmpfn = "$conffile.$$.tmp";
6721
6722 # disable interrupts (always do cleanups)
6cb0144a
EK
6723 local $SIG{INT} =
6724 local $SIG{TERM} =
6725 local $SIG{QUIT} =
6726 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
3e16d5fc 6727
afdb31d5 6728 eval {
3e16d5fc 6729 # enable interrupts
6cb0144a
EK
6730 local $SIG{INT} =
6731 local $SIG{TERM} =
6732 local $SIG{QUIT} =
6733 local $SIG{HUP} =
6734 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
3e16d5fc 6735
9c502e26
DM
6736 if ($archive eq '-') {
6737 print "extracting archive from STDIN\n";
6738 run_command($cmd, input => "<&STDIN");
6739 } else {
6740 print "extracting archive '$archive'\n";
6741 run_command($cmd);
6742 }
3e16d5fc
DM
6743
6744 return if $opts->{info};
6745
6746 # read new mapping
6747 my $map = {};
6748 my $statfile = "$tmpdir/qmrestore.stat";
6749 if (my $fd = IO::File->new($statfile, "r")) {
6750 while (defined (my $line = <$fd>)) {
6751 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6752 $map->{$1} = $2 if $1;
6753 } else {
6754 print STDERR "unable to parse line in statfile - $line\n";
6755 }
6756 }
6757 $fd->close();
6758 }
6759
6760 my $confsrc = "$tmpdir/qemu-server.conf";
6761
6762 my $srcfd = new IO::File($confsrc, "r") ||
6763 die "unable to open file '$confsrc'\n";
6764
6765 my $outfd = new IO::File ($tmpfn, "w") ||
6766 die "unable to write config for VM $vmid\n";
6767
91bd6c90 6768 my $cookie = { netcount => 0 };
3e16d5fc 6769 while (defined (my $line = <$srcfd>)) {
be190583 6770 restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
3e16d5fc
DM
6771 }
6772
6773 $srcfd->close();
6774 $outfd->close();
6775 };
6776 my $err = $@;
6777
afdb31d5 6778 if ($err) {
3e16d5fc
DM
6779
6780 unlink $tmpfn;
6781
ed221350 6782 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
afdb31d5 6783
3e16d5fc 6784 die $err;
afdb31d5 6785 }
3e16d5fc
DM
6786
6787 rmtree $tmpdir;
6788
6789 rename $tmpfn, $conffile ||
6790 die "unable to commit configuration file '$conffile'\n";
91bd6c90 6791
ed221350
DM
6792 PVE::Cluster::cfs_update(); # make sure we read new file
6793
91bd6c90
DM
6794 eval { rescan($vmid, 1); };
6795 warn $@ if $@;
3e16d5fc
DM
6796};
6797
65a5ce88 6798sub foreach_storage_used_by_vm {
18bfb361
DM
6799 my ($conf, $func) = @_;
6800
6801 my $sidhash = {};
6802
8ddbcf8b
FG
6803 foreach_drive($conf, sub {
6804 my ($ds, $drive) = @_;
6805 return if drive_is_cdrom($drive);
18bfb361
DM
6806
6807 my $volid = $drive->{file};
6808
6809 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
be190583 6810 $sidhash->{$sid} = $sid if $sid;
8ddbcf8b 6811 });
18bfb361
DM
6812
6813 foreach my $sid (sort keys %$sidhash) {
6814 &$func($sid);
6815 }
6816}
6817
e5eaa028
WL
6818sub do_snapshots_with_qemu {
6819 my ($storecfg, $volid) = @_;
6820
6821 my $storage_name = PVE::Storage::parse_volume_id($volid);
8aa2ed7c 6822 my $scfg = $storecfg->{ids}->{$storage_name};
e5eaa028 6823
8aa2ed7c 6824 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
e5eaa028
WL
6825 return 1;
6826 }
6827
6828 if ($volid =~ m/\.(qcow2|qed)$/){
6829 return 1;
6830 }
6831
6832 return undef;
6833}
6834
4dcc780c 6835sub qga_check_running {
a4938c72 6836 my ($vmid, $nowarn) = @_;
4dcc780c
WL
6837
6838 eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6839 if ($@) {
a4938c72 6840 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
4dcc780c
WL
6841 return 0;
6842 }
6843 return 1;
6844}
6845
04a69bb4
AD
6846sub template_create {
6847 my ($vmid, $conf, $disk) = @_;
6848
04a69bb4 6849 my $storecfg = PVE::Storage::config();
04a69bb4 6850
9cd07842
DM
6851 foreach_drive($conf, sub {
6852 my ($ds, $drive) = @_;
6853
6854 return if drive_is_cdrom($drive);
6855 return if $disk && $ds ne $disk;
6856
6857 my $volid = $drive->{file};
bbd56097 6858 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
9cd07842 6859
04a69bb4
AD
6860 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6861 $drive->{file} = $voliddst;
152fe752 6862 $conf->{$ds} = print_drive($vmid, $drive);
ffda963f 6863 PVE::QemuConfig->write_config($vmid, $conf);
04a69bb4 6864 });
04a69bb4
AD
6865}
6866
92bdc3f0
DC
6867sub convert_iscsi_path {
6868 my ($path) = @_;
6869
6870 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6871 my $portal = $1;
6872 my $target = $2;
6873 my $lun = $3;
6874
6875 my $initiator_name = get_initiator_name();
6876
6877 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6878 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6879 }
6880
6881 die "cannot convert iscsi path '$path', unkown format\n";
6882}
6883
5133de42 6884sub qemu_img_convert {
988e2714 6885 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5133de42
AD
6886
6887 my $storecfg = PVE::Storage::config();
6888 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6889 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6890
af1f1ec0 6891 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6bb91c17 6892
af1f1ec0
DC
6893 my $cachemode;
6894 my $src_path;
6895 my $src_is_iscsi = 0;
6896 my $src_format = 'raw';
6bb91c17 6897
af1f1ec0
DC
6898 if ($src_storeid) {
6899 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5133de42 6900 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
af1f1ec0
DC
6901 $src_format = qemu_img_format($src_scfg, $src_volname);
6902 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6903 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6904 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6905 } elsif (-f $src_volid) {
6906 $src_path = $src_volid;
6907 if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6908 $src_format = $1;
6909 }
6910 }
5133de42 6911
af1f1ec0 6912 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
5133de42 6913
af1f1ec0
DC
6914 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6915 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6916 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6917 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
5133de42 6918
af1f1ec0
DC
6919 my $cmd = [];
6920 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6921 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6922 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6923 push @$cmd, '-T', $cachemode if defined($cachemode);
6924
6925 if ($src_is_iscsi) {
6926 push @$cmd, '--image-opts';
6927 $src_path = convert_iscsi_path($src_path);
6928 } else {
6929 push @$cmd, '-f', $src_format;
6930 }
92bdc3f0 6931
af1f1ec0
DC
6932 if ($dst_is_iscsi) {
6933 push @$cmd, '--target-image-opts';
6934 $dst_path = convert_iscsi_path($dst_path);
6935 } else {
6936 push @$cmd, '-O', $dst_format;
6937 }
92bdc3f0 6938
af1f1ec0 6939 push @$cmd, $src_path;
92bdc3f0 6940
af1f1ec0
DC
6941 if (!$dst_is_iscsi && $is_zero_initialized) {
6942 push @$cmd, "zeroinit:$dst_path";
6943 } else {
6944 push @$cmd, $dst_path;
6945 }
92bdc3f0 6946
af1f1ec0
DC
6947 my $parser = sub {
6948 my $line = shift;
6949 if($line =~ m/\((\S+)\/100\%\)/){
6950 my $percent = $1;
6951 my $transferred = int($size * $percent / 100);
6952 my $remaining = $size - $transferred;
92bdc3f0 6953
af1f1ec0 6954 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
988e2714 6955 }
5133de42 6956
af1f1ec0 6957 };
5133de42 6958
af1f1ec0
DC
6959 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6960 my $err = $@;
6961 die "copy failed: $err" if $err;
5133de42
AD
6962}
6963
6964sub qemu_img_format {
6965 my ($scfg, $volname) = @_;
6966
9c52f5ed 6967 if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
5133de42 6968 return $1;
be190583 6969 } else {
5133de42 6970 return "raw";
5133de42
AD
6971 }
6972}
6973
cfad42af 6974sub qemu_drive_mirror {
9fa05d31 6975 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
cfad42af 6976
5a345967
AD
6977 $jobs = {} if !$jobs;
6978
6979 my $qemu_target;
6980 my $format;
35e4ab04 6981 $jobs->{"drive-$drive"} = {};
152fe752 6982
1e5143de 6983 if ($dst_volid =~ /^nbd:/) {
87955688 6984 $qemu_target = $dst_volid;
5a345967 6985 $format = "nbd";
5a345967 6986 } else {
5a345967
AD
6987 my $storecfg = PVE::Storage::config();
6988 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6989
6990 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
cfad42af 6991
5a345967 6992 $format = qemu_img_format($dst_scfg, $dst_volname);
21ccdb50 6993
5a345967 6994 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
21ccdb50 6995
5a345967
AD
6996 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6997 }
988e2714
WB
6998
6999 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
88383920
DM
7000 $opts->{format} = $format if $format;
7001
9fa05d31 7002 if (defined($bwlimit)) {
f6409f61
TL
7003 $opts->{speed} = $bwlimit * 1024;
7004 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
9fa05d31
SI
7005 } else {
7006 print "drive mirror is starting for drive-$drive\n";
7007 }
21ccdb50 7008
6dde5ea2
TL
7009 # if a job already runs for this device we get an error, catch it for cleanup
7010 eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
5a345967
AD
7011 if (my $err = $@) {
7012 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6dde5ea2
TL
7013 warn "$@\n" if $@;
7014 die "mirroring error: $err\n";
5a345967
AD
7015 }
7016
5619e74a 7017 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5a345967
AD
7018}
7019
7020sub qemu_drive_mirror_monitor {
5619e74a 7021 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
2e953867 7022
08ac653f 7023 eval {
5a345967
AD
7024 my $err_complete = 0;
7025
08ac653f 7026 while (1) {
5a345967
AD
7027 die "storage migration timed out\n" if $err_complete > 300;
7028
08ac653f 7029 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
08ac653f 7030
5a345967
AD
7031 my $running_mirror_jobs = {};
7032 foreach my $stat (@$stats) {
7033 next if $stat->{type} ne 'mirror';
7034 $running_mirror_jobs->{$stat->{device}} = $stat;
7035 }
08ac653f 7036
5a345967 7037 my $readycounter = 0;
67fb9de6 7038
5a345967
AD
7039 foreach my $job (keys %$jobs) {
7040
7041 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7042 print "$job : finished\n";
7043 delete $jobs->{$job};
7044 next;
7045 }
7046
bd2d5fe6 7047 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
f34ebd52 7048
5a345967
AD
7049 my $busy = $running_mirror_jobs->{$job}->{busy};
7050 my $ready = $running_mirror_jobs->{$job}->{ready};
7051 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7052 my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7053 my $remaining = $total - $transferred;
7054 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
08ac653f 7055
5a345967
AD
7056 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7057 }
f34ebd52 7058
d1782eba 7059 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
5a345967 7060 }
b467f79a 7061
5a345967
AD
7062 last if scalar(keys %$jobs) == 0;
7063
7064 if ($readycounter == scalar(keys %$jobs)) {
7065 print "all mirroring jobs are ready \n";
7066 last if $skipcomplete; #do the complete later
7067
7068 if ($vmiddst && $vmiddst != $vmid) {
1a988fd2
DC
7069 my $agent_running = $qga && qga_check_running($vmid);
7070 if ($agent_running) {
5619e74a
AD
7071 print "freeze filesystem\n";
7072 eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7073 } else {
7074 print "suspend vm\n";
7075 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7076 }
7077
5a345967
AD
7078 # if we clone a disk for a new target vm, we don't switch the disk
7079 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
5619e74a 7080
1a988fd2 7081 if ($agent_running) {
5619e74a
AD
7082 print "unfreeze filesystem\n";
7083 eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7084 } else {
7085 print "resume vm\n";
7086 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7087 }
7088
2e953867 7089 last;
5a345967
AD
7090 } else {
7091
7092 foreach my $job (keys %$jobs) {
7093 # try to switch the disk if source and destination are on the same guest
bd2d5fe6 7094 print "$job: Completing block job...\n";
5a345967
AD
7095
7096 eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7097 if ($@ =~ m/cannot be completed/) {
bd2d5fe6 7098 print "$job: Block job cannot be completed, try again.\n";
5a345967
AD
7099 $err_complete++;
7100 }else {
bd2d5fe6 7101 print "$job: Completed successfully.\n";
5a345967
AD
7102 $jobs->{$job}->{complete} = 1;
7103 }
7104 }
2e953867 7105 }
08ac653f 7106 }
08ac653f 7107 sleep 1;
cfad42af 7108 }
08ac653f 7109 };
88383920 7110 my $err = $@;
08ac653f 7111
88383920 7112 if ($err) {
5a345967 7113 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
88383920
DM
7114 die "mirroring error: $err";
7115 }
7116
5a345967
AD
7117}
7118
7119sub qemu_blockjobs_cancel {
7120 my ($vmid, $jobs) = @_;
7121
7122 foreach my $job (keys %$jobs) {
bd2d5fe6 7123 print "$job: Cancelling block job\n";
5a345967
AD
7124 eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7125 $jobs->{$job}->{cancel} = 1;
7126 }
7127
7128 while (1) {
7129 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7130
7131 my $running_jobs = {};
7132 foreach my $stat (@$stats) {
7133 $running_jobs->{$stat->{device}} = $stat;
7134 }
7135
7136 foreach my $job (keys %$jobs) {
7137
bd2d5fe6
WB
7138 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7139 print "$job: Done.\n";
5a345967
AD
7140 delete $jobs->{$job};
7141 }
7142 }
7143
7144 last if scalar(keys %$jobs) == 0;
7145
7146 sleep 1;
cfad42af
AD
7147 }
7148}
7149
152fe752 7150sub clone_disk {
be190583 7151 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7e303ef3 7152 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
152fe752
DM
7153
7154 my $newvolid;
7155
7156 if (!$full) {
7157 print "create linked clone of drive $drivename ($drive->{file})\n";
258e646c 7158 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
152fe752
DM
7159 push @$newvollist, $newvolid;
7160 } else {
5a345967 7161
152fe752
DM
7162 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7163 $storeid = $storage if $storage;
7164
44549149 7165 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
152fe752
DM
7166 my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7167
7168 print "create full clone of drive $drivename ($drive->{file})\n";
931432bd 7169 my $name = undef;
931432bd 7170 $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
152fe752
DM
7171 push @$newvollist, $newvolid;
7172
3999f370 7173 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
1dbd6d30 7174
988e2714 7175 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
152fe752 7176 if (!$running || $snapname) {
d189e590 7177 # TODO: handle bwlimits
988e2714 7178 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
152fe752 7179 } else {
2e541679
AD
7180
7181 my $kvmver = get_running_qemu_version ($vmid);
7182 if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
961af8a3
WB
7183 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7184 if $drive->{iothread};
2e541679 7185 }
2af848a2 7186
7e303ef3 7187 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
be190583 7188 }
152fe752
DM
7189 }
7190
7191 my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7192
7193 my $disk = $drive;
7194 $disk->{format} = undef;
7195 $disk->{file} = $newvolid;
7196 $disk->{size} = $size;
7197
7198 return $disk;
7199}
7200
ff556cf2
DM
7201# this only works if VM is running
7202sub get_current_qemu_machine {
7203 my ($vmid) = @_;
7204
7205 my $cmd = { execute => 'query-machines', arguments => {} };
8e90138a 7206 my $res = vm_qmp_command($vmid, $cmd);
ff556cf2
DM
7207
7208 my ($current, $default);
7209 foreach my $e (@$res) {
7210 $default = $e->{name} if $e->{'is-default'};
7211 $current = $e->{name} if $e->{'is-current'};
7212 }
7213
7214 # fallback to the default machine if current is not supported by qemu
7215 return $current || $default || 'pc';
7216}
7217
98cfd8b6
AD
7218sub get_running_qemu_version {
7219 my ($vmid) = @_;
7220 my $cmd = { execute => 'query-version', arguments => {} };
7221 my $res = vm_qmp_command($vmid, $cmd);
7222 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7223}
7224
23f73120
AD
7225sub qemu_machine_feature_enabled {
7226 my ($machine, $kvmver, $version_major, $version_minor) = @_;
7227
7228 my $current_major;
7229 my $current_minor;
7230
d731ecbe 7231 if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
23f73120
AD
7232
7233 $current_major = $3;
7234 $current_minor = $4;
7235
7236 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7237
7238 $current_major = $1;
7239 $current_minor = $2;
7240 }
7241
dd84e5ec
WB
7242 return 1 if $current_major > $version_major ||
7243 ($current_major == $version_major &&
7244 $current_minor >= $version_minor);
23f73120
AD
7245}
7246
42dbd2ee 7247sub qemu_machine_pxe {
8071149b 7248 my ($vmid, $conf) = @_;
42dbd2ee 7249
8071149b 7250 my $machine = PVE::QemuServer::get_current_qemu_machine($vmid);
42dbd2ee 7251
3807f3e4
DC
7252 if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7253 $machine .= '.pxe';
42dbd2ee
AD
7254 }
7255
d1363934 7256 return $machine;
42dbd2ee
AD
7257}
7258
249c4a6c
AD
7259sub qemu_use_old_bios_files {
7260 my ($machine_type) = @_;
7261
7262 return if !$machine_type;
7263
7264 my $use_old_bios_files = undef;
7265
7266 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7267 $machine_type = $1;
7268 $use_old_bios_files = 1;
7269 } else {
74cc511f 7270 my $kvmver = kvm_user_version();
249c4a6c
AD
7271 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7272 # load new efi bios files on migration. So this hack is required to allow
7273 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7274 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
74cc511f 7275 $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
249c4a6c
AD
7276 }
7277
7278 return ($use_old_bios_files, $machine_type);
7279}
7280
96ed3574
WB
7281sub create_efidisk($$$$$) {
7282 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
3e1f1122 7283
96ed3574
WB
7284 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7285 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
3e1f1122 7286
af1f1ec0
DC
7287 my $vars_size_b = -s $ovmf_vars;
7288 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
3e1f1122
TL
7289 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7290 PVE::Storage::activate_volumes($storecfg, [$volid]);
7291
af1f1ec0 7292 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
3e1f1122
TL
7293
7294 return ($volid, $vars_size);
7295}
7296
22de899a
AD
7297sub vm_iothreads_list {
7298 my ($vmid) = @_;
7299
7300 my $res = vm_mon_cmd($vmid, 'query-iothreads');
7301
7302 my $iothreads = {};
7303 foreach my $iothread (@$res) {
7304 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7305 }
7306
7307 return $iothreads;
7308}
7309
ee034f5c
AD
7310sub scsihw_infos {
7311 my ($conf, $drive) = @_;
7312
7313 my $maxdev = 0;
7314
7fe1b688 7315 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
ee034f5c 7316 $maxdev = 7;
a1511b3c 7317 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
ee034f5c
AD
7318 $maxdev = 1;
7319 } else {
7320 $maxdev = 256;
7321 }
7322
7323 my $controller = int($drive->{index} / $maxdev);
a1511b3c 7324 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
ee034f5c
AD
7325
7326 return ($maxdev, $controller, $controller_prefix);
7327}
a1511b3c 7328
075e8249 7329sub add_hyperv_enlightenments {
2894c247 7330 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
4317f69f 7331
4317f69f
AD
7332 return if $winversion < 6;
7333 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7334
2894c247
DC
7335 if ($gpu_passthrough || defined($hv_vendor_id)) {
7336 $hv_vendor_id //= 'proxmox';
7337 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7338 }
5aba3953 7339
4317f69f
AD
7340 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7341 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7342 push @$cpuFlags , 'hv_vapic';
7343 push @$cpuFlags , 'hv_time';
7344 } else {
7345 push @$cpuFlags , 'hv_spinlocks=0xffff';
7346 }
7347
7348 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7349 push @$cpuFlags , 'hv_reset';
7350 push @$cpuFlags , 'hv_vpindex';
7351 push @$cpuFlags , 'hv_runtime';
7352 }
7353
7354 if ($winversion >= 7) {
7355 push @$cpuFlags , 'hv_relaxed';
ebb346d6 7356
df648a6a 7357 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
ebb346d6
AD
7358 push @$cpuFlags , 'hv_synic';
7359 push @$cpuFlags , 'hv_stimer';
7360 }
392dfbf5 7361
392dfbf5
SR
7362 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7363 push @$cpuFlags , 'hv_ipi';
392dfbf5 7364 }
4317f69f
AD
7365 }
7366}
7367
7368sub windows_version {
7369 my ($ostype) = @_;
7370
7371 return 0 if !$ostype;
7372
7373 my $winversion = 0;
7374
7375 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7376 $winversion = 5;
7377 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7378 $winversion = 6;
7379 } elsif ($ostype =~ m/^win(\d+)$/) {
7380 $winversion = $1;
7381 }
7382
7383 return $winversion;
7384}
7385
44549149
EK
7386sub resolve_dst_disk_format {
7387 my ($storecfg, $storeid, $src_volname, $format) = @_;
7388 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7389
7390 if (!$format) {
7391 # if no target format is specified, use the source disk format as hint
7392 if ($src_volname) {
7393 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7394 $format = qemu_img_format($scfg, $src_volname);
7395 } else {
7396 return $defFormat;
7397 }
7398 }
7399
7400 # test if requested format is supported - else use default
7401 my $supported = grep { $_ eq $format } @$validFormats;
7402 $format = $defFormat if !$supported;
7403 return $format;
7404}
7405
ae2fcb3b
EK
7406sub resolve_first_disk {
7407 my $conf = shift;
7408 my @disks = PVE::QemuServer::valid_drive_names();
7409 my $firstdisk;
7410 foreach my $ds (reverse @disks) {
7411 next if !$conf->{$ds};
7412 my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7413 next if PVE::QemuServer::drive_is_cdrom($disk);
7414 $firstdisk = $ds;
7415 }
7416 return $firstdisk;
7417}
7418
6ee499ff 7419sub generate_uuid {
ae2fcb3b
EK
7420 my ($uuid, $uuid_str);
7421 UUID::generate($uuid);
7422 UUID::unparse($uuid, $uuid_str);
6ee499ff
DC
7423 return $uuid_str;
7424}
7425
7426sub generate_smbios1_uuid {
7427 return "uuid=".generate_uuid();
ae2fcb3b
EK
7428}
7429
9c152e87
TL
7430sub nbd_stop {
7431 my ($vmid) = @_;
7432
7433 vm_mon_cmd($vmid, 'nbd-server-stop');
7434}
7435
dae98db9
DC
7436sub create_reboot_request {
7437 my ($vmid) = @_;
7438 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7439 or die "failed to create reboot trigger file: $!\n";
7440 close($fh);
7441}
7442
7443sub clear_reboot_request {
7444 my ($vmid) = @_;
7445 my $path = "/run/qemu-server/$vmid.reboot";
7446 my $res = 0;
7447
7448 $res = unlink($path);
7449 die "could not remove reboot request for $vmid: $!"
7450 if !$res && $! != POSIX::ENOENT;
7451
7452 return $res;
7453}
7454
65e866e5
DM
7455# bash completion helper
7456
7457sub complete_backup_archives {
7458 my ($cmdname, $pname, $cvalue) = @_;
7459
7460 my $cfg = PVE::Storage::config();
7461
7462 my $storeid;
7463
7464 if ($cvalue =~ m/^([^:]+):/) {
7465 $storeid = $1;
7466 }
7467
7468 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7469
7470 my $res = [];
7471 foreach my $id (keys %$data) {
7472 foreach my $item (@{$data->{$id}}) {
7473 next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7474 push @$res, $item->{volid} if defined($item->{volid});
7475 }
7476 }
7477
7478 return $res;
7479}
7480
7481my $complete_vmid_full = sub {
7482 my ($running) = @_;
7483
7484 my $idlist = vmstatus();
7485
7486 my $res = [];
7487
7488 foreach my $id (keys %$idlist) {
7489 my $d = $idlist->{$id};
7490 if (defined($running)) {
7491 next if $d->{template};
7492 next if $running && $d->{status} ne 'running';
7493 next if !$running && $d->{status} eq 'running';
7494 }
7495 push @$res, $id;
7496
7497 }
7498 return $res;
7499};
7500
7501sub complete_vmid {
7502 return &$complete_vmid_full();
7503}
7504
7505sub complete_vmid_stopped {
7506 return &$complete_vmid_full(0);
7507}
7508
7509sub complete_vmid_running {
7510 return &$complete_vmid_full(1);
7511}
7512
335af808
DM
7513sub complete_storage {
7514
7515 my $cfg = PVE::Storage::config();
7516 my $ids = $cfg->{ids};
7517
7518 my $res = [];
7519 foreach my $sid (keys %$ids) {
7520 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
c4c844ef 7521 next if !$ids->{$sid}->{content}->{images};
335af808
DM
7522 push @$res, $sid;
7523 }
7524
7525 return $res;
7526}
7527
1e3baf05 75281;