]> git.proxmox.com Git - qemu-server.git/blame - PVE/QemuServer.pm
bump version to 5.0-45
[qemu-server.git] / PVE / QemuServer.pm
CommitLineData
1e3baf05
DM
1package PVE::QemuServer;
2
3use strict;
990fc5e2 4use warnings;
1e3baf05
DM
5use POSIX;
6use IO::Handle;
7use IO::Select;
8use IO::File;
9use IO::Dir;
10use IO::Socket::UNIX;
11use File::Basename;
12use File::Path;
13use File::stat;
14use Getopt::Long;
fc1ddcdc 15use Digest::SHA;
1e3baf05
DM
16use Fcntl ':flock';
17use Cwd 'abs_path';
18use IPC::Open3;
c971c4f2 19use JSON;
1e3baf05
DM
20use Fcntl;
21use PVE::SafeSyslog;
22use Storable qw(dclone);
23use PVE::Exception qw(raise raise_param_exc);
24use PVE::Storage;
0c9a7596 25use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
b7ba6b79 26use PVE::JSONSchema qw(get_standard_option);
1e3baf05
DM
27use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
28use PVE::INotify;
29use PVE::ProcFSTools;
ffda963f 30use PVE::QemuConfig;
26f11676 31use PVE::QMPClient;
91bd6c90 32use PVE::RPCEnvironment;
de9768f0 33use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
3f669af2 34use PVE::QemuServer::Memory;
d40e5e18 35use PVE::QemuServer::USB qw(parse_usb_device);
0c9a7596 36use PVE::QemuServer::Cloudinit;
b71351a7 37use PVE::SysFSTools;
d04d6af1 38use PVE::Systemd;
6b64503e 39use Time::HiRes qw(gettimeofday);
a783c78e 40use File::Copy qw(copy);
46630a5f 41use URI::Escape;
1e3baf05 42
102cf9d8 43my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
96ed3574
WB
44my $OVMF = {
45 x86_64 => [
46 "$EDK2_FW_BASE/OVMF_CODE.fd",
47 "$EDK2_FW_BASE/OVMF_VARS.fd"
48 ],
49 aarch64 => [
50 "$EDK2_FW_BASE/AAVMF_CODE.fd",
51 "$EDK2_FW_BASE/AAVMF_VARS.fd"
52 ],
53};
2ddc0a5c 54
e5eaa028
WL
55my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
56
7f0b5beb 57my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
1e3baf05 58
9c52f5ed
WB
59my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
60
19672434 61# Note about locking: we use flock on the config file protect
1e3baf05
DM
62# against concurent actions.
63# Aditionaly, we have a 'lock' setting in the config file. This
22c377f0 64# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
1e3baf05
DM
65# allowed when such lock is set. But you can ignore this kind of
66# lock with the --skiplock flag.
67
97d62eb7 68cfs_register_file('/qemu-server/',
1858638f
DM
69 \&parse_vm_config,
70 \&write_vm_config);
1e3baf05 71
3ea94c60
DM
72PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
73 description => "Some command save/restore state from this location.",
74 type => 'string',
75 maxLength => 128,
76 optional => 1,
77});
78
8abd398b
DM
79PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
80 description => "The name of the snapshot.",
81 type => 'string', format => 'pve-configid',
82 maxLength => 40,
83});
84
c7d2b650
DM
85PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
86 type => 'string',
87 enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
88 description => "The drive's backing file's data format.",
89 optional => 1,
90});
91
c6737ef1
DC
92PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
93 description => "Specifies the Qemu machine type.",
94 type => 'string',
d731ecbe 95 pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
c6737ef1
DC
96 maxLength => 40,
97 optional => 1,
98});
99
1e3baf05
DM
100#no warnings 'redefine';
101
c8effec3
AD
102sub cgroups_write {
103 my ($controller, $vmid, $option, $value) = @_;
104
3a515a88
DM
105 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
106 PVE::ProcFSTools::write_proc_entry($path, $value);
c8effec3
AD
107
108}
109
1e3baf05
DM
110my $nodename = PVE::INotify::nodename();
111
112mkdir "/etc/pve/nodes/$nodename";
113my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
114mkdir $confdir;
115
116my $var_run_tmpdir = "/var/run/qemu-server";
117mkdir $var_run_tmpdir;
118
119my $lock_dir = "/var/lock/qemu-server";
120mkdir $lock_dir;
121
8930da74
DM
122my $cpu_vendor_list = {
123 # Intel CPUs
124 486 => 'GenuineIntel',
125 pentium => 'GenuineIntel',
126 pentium2 => 'GenuineIntel',
127 pentium3 => 'GenuineIntel',
128 coreduo => 'GenuineIntel',
129 core2duo => 'GenuineIntel',
130 Conroe => 'GenuineIntel',
9052caba 131 Penryn => 'GenuineIntel',
8930da74 132 Nehalem => 'GenuineIntel',
9052caba 133 'Nehalem-IBRS' => 'GenuineIntel',
8930da74 134 Westmere => 'GenuineIntel',
9052caba 135 'Westmere-IBRS' => 'GenuineIntel',
8930da74 136 SandyBridge => 'GenuineIntel',
9052caba 137 'SandyBridge-IBRS' => 'GenuineIntel',
8930da74 138 IvyBridge => 'GenuineIntel',
9052caba 139 'IvyBridge-IBRS' => 'GenuineIntel',
8930da74 140 Haswell => 'GenuineIntel',
9052caba 141 'Haswell-IBRS' => 'GenuineIntel',
8930da74 142 'Haswell-noTSX' => 'GenuineIntel',
9052caba 143 'Haswell-noTSX-IBRS' => 'GenuineIntel',
8930da74 144 Broadwell => 'GenuineIntel',
9052caba 145 'Broadwell-IBRS' => 'GenuineIntel',
8930da74 146 'Broadwell-noTSX' => 'GenuineIntel',
9052caba 147 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
3db920fc 148 'Skylake-Client' => 'GenuineIntel',
9052caba 149 'Skylake-Client-IBRS' => 'GenuineIntel',
a446dbf4
FG
150 'Skylake-Server' => 'GenuineIntel',
151 'Skylake-Server-IBRS' => 'GenuineIntel',
9052caba 152
8930da74
DM
153 # AMD CPUs
154 athlon => 'AuthenticAMD',
155 phenom => 'AuthenticAMD',
156 Opteron_G1 => 'AuthenticAMD',
157 Opteron_G2 => 'AuthenticAMD',
158 Opteron_G3 => 'AuthenticAMD',
159 Opteron_G4 => 'AuthenticAMD',
160 Opteron_G5 => 'AuthenticAMD',
2a850ee8
WB
161 EPYC => 'AuthenticAMD',
162 'EPYC-IBPB' => 'AuthenticAMD',
8930da74
DM
163
164 # generic types, use vendor from host node
165 host => 'default',
166 kvm32 => 'default',
167 kvm64 => 'default',
168 qemu32 => 'default',
169 qemu64 => 'default',
a446dbf4 170 max => 'default',
8930da74
DM
171};
172
eabc189e 173my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
049fc9eb 174
ff6ffe20 175my $cpu_fmt = {
16a91d65
WB
176 cputype => {
177 description => "Emulated CPU type.",
178 type => 'string',
7f694a71 179 enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
16a91d65
WB
180 default => 'kvm64',
181 default_key => 1,
182 },
183 hidden => {
184 description => "Do not identify as a KVM virtual machine.",
185 type => 'boolean',
186 optional => 1,
187 default => 0
188 },
39fd79e2 189 flags => {
049fc9eb
FG
190 description => "List of additional CPU flags separated by ';'."
191 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
eabc189e 192 . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
049fc9eb 193 format_description => '+FLAG[;-FLAG...]',
39fd79e2 194 type => 'string',
049fc9eb 195 pattern => qr/$cpu_flag(;$cpu_flag)*/,
39fd79e2 196 optional => 1,
39fd79e2 197 },
16a91d65
WB
198};
199
ec3582b5
WB
200my $watchdog_fmt = {
201 model => {
202 default_key => 1,
203 type => 'string',
204 enum => [qw(i6300esb ib700)],
205 description => "Watchdog type to emulate.",
206 default => 'i6300esb',
207 optional => 1,
208 },
209 action => {
210 type => 'string',
211 enum => [qw(reset shutdown poweroff pause debug none)],
212 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
213 optional => 1,
214 },
215};
216PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
217
9d66b397
SI
218my $agent_fmt = {
219 enabled => {
220 description => "Enable/disable Qemu GuestAgent.",
221 type => 'boolean',
222 default => 0,
223 default_key => 1,
224 },
225 fstrim_cloned_disks => {
226 description => "Run fstrim after cloning/moving a disk.",
227 type => 'boolean',
228 optional => 1,
229 default => 0
230 },
231};
232
55655ebc
DC
233my $vga_fmt = {
234 type => {
235 description => "Select the VGA type.",
236 type => 'string',
237 default => 'std',
238 optional => 1,
239 default_key => 1,
7c954c42 240 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
55655ebc
DC
241 },
242 memory => {
243 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
244 type => 'integer',
245 optional => 1,
246 minimum => 4,
247 maximum => 512,
248 },
249};
250
1e3baf05
DM
251my $confdesc = {
252 onboot => {
253 optional => 1,
254 type => 'boolean',
255 description => "Specifies whether a VM will be started during system bootup.",
256 default => 0,
257 },
258 autostart => {
259 optional => 1,
260 type => 'boolean',
261 description => "Automatic restart after crash (currently ignored).",
262 default => 0,
263 },
2ff09f52
DA
264 hotplug => {
265 optional => 1,
b3c2bdd1
DM
266 type => 'string', format => 'pve-hotplug-features',
267 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
268 default => 'network,disk,usb',
2ff09f52 269 },
1e3baf05
DM
270 reboot => {
271 optional => 1,
272 type => 'boolean',
273 description => "Allow reboot. If set to '0' the VM exit on reboot.",
274 default => 1,
275 },
276 lock => {
277 optional => 1,
278 type => 'string',
279 description => "Lock/unlock the VM.",
89caf77b 280 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
1e3baf05
DM
281 },
282 cpulimit => {
283 optional => 1,
c6f773b8 284 type => 'number',
52261945
DM
285 description => "Limit of CPU usage.",
286 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
1e3baf05 287 minimum => 0,
c6f773b8 288 maximum => 128,
52261945 289 default => 0,
1e3baf05
DM
290 },
291 cpuunits => {
292 optional => 1,
293 type => 'integer',
52261945 294 description => "CPU weight for a VM.",
237239bf
PA
295 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
296 minimum => 2,
297 maximum => 262144,
613d76a1 298 default => 1024,
1e3baf05
DM
299 },
300 memory => {
301 optional => 1,
302 type => 'integer',
7878afeb 303 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
1e3baf05
DM
304 minimum => 16,
305 default => 512,
306 },
13a48620
DA
307 balloon => {
308 optional => 1,
309 type => 'integer',
8b1accf7
DM
310 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
311 minimum => 0,
312 },
313 shares => {
314 optional => 1,
315 type => 'integer',
82329cd5 316 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
8b1accf7
DM
317 minimum => 0,
318 maximum => 50000,
319 default => 1000,
13a48620 320 },
1e3baf05
DM
321 keyboard => {
322 optional => 1,
323 type => 'string',
f889aa0f 324 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
aea47dd6 325 "It should not be necessary to set it.",
e95fe75f 326 enum => PVE::Tools::kvmkeymaplist(),
aea47dd6 327 default => undef,
1e3baf05
DM
328 },
329 name => {
330 optional => 1,
7fabe17d 331 type => 'string', format => 'dns-name',
1e3baf05
DM
332 description => "Set a name for the VM. Only used on the configuration web interface.",
333 },
cdd20088
AD
334 scsihw => {
335 optional => 1,
336 type => 'string',
52261945 337 description => "SCSI controller model",
6731a4cf 338 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
cdd20088
AD
339 default => 'lsi',
340 },
1e3baf05
DM
341 description => {
342 optional => 1,
343 type => 'string',
0581fe4f 344 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
1e3baf05
DM
345 },
346 ostype => {
347 optional => 1,
348 type => 'string',
0cb9971e 349 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
52261945
DM
350 description => "Specify guest operating system.",
351 verbose_description => <<EODESC,
352Specify guest operating system. This is used to enable special
353optimization/features for specific operating systems:
354
355[horizontal]
356other;; unspecified OS
357wxp;; Microsoft Windows XP
358w2k;; Microsoft Windows 2000
359w2k3;; Microsoft Windows 2003
360w2k8;; Microsoft Windows 2008
361wvista;; Microsoft Windows Vista
362win7;; Microsoft Windows 7
44c2a647
TL
363win8;; Microsoft Windows 8/2012/2012r2
364win10;; Microsoft Windows 10/2016
52261945
DM
365l24;; Linux 2.4 Kernel
366l26;; Linux 2.6/3.X Kernel
367solaris;; Solaris/OpenSolaris/OpenIndiania kernel
1e3baf05
DM
368EODESC
369 },
370 boot => {
371 optional => 1,
372 type => 'string',
373 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
374 pattern => '[acdn]{1,4}',
32baffb4 375 default => 'cdn',
1e3baf05
DM
376 },
377 bootdisk => {
378 optional => 1,
379 type => 'string', format => 'pve-qm-bootdisk',
380 description => "Enable booting from specified disk.",
03e480fc 381 pattern => '(ide|sata|scsi|virtio)\d+',
1e3baf05
DM
382 },
383 smp => {
384 optional => 1,
385 type => 'integer',
386 description => "The number of CPUs. Please use option -sockets instead.",
387 minimum => 1,
388 default => 1,
389 },
390 sockets => {
391 optional => 1,
392 type => 'integer',
393 description => "The number of CPU sockets.",
394 minimum => 1,
395 default => 1,
396 },
397 cores => {
398 optional => 1,
399 type => 'integer',
400 description => "The number of cores per socket.",
401 minimum => 1,
402 default => 1,
403 },
8a010eae
AD
404 numa => {
405 optional => 1,
406 type => 'boolean',
1917695c 407 description => "Enable/disable NUMA.",
8a010eae
AD
408 default => 0,
409 },
7023f3ea
AD
410 hugepages => {
411 optional => 1,
412 type => 'string',
413 description => "Enable/disable hugepages memory.",
414 enum => [qw(any 2 1024)],
415 },
de9d1e55 416 vcpus => {
3bd18e48
AD
417 optional => 1,
418 type => 'integer',
de9d1e55 419 description => "Number of hotplugged vcpus.",
3bd18e48 420 minimum => 1,
de9d1e55 421 default => 0,
3bd18e48 422 },
1e3baf05
DM
423 acpi => {
424 optional => 1,
425 type => 'boolean',
426 description => "Enable/disable ACPI.",
427 default => 1,
428 },
bc84dcca 429 agent => {
ab6a046f 430 optional => 1,
9d66b397
SI
431 description => "Enable/disable Qemu GuestAgent and its properties.",
432 type => 'string',
433 format => $agent_fmt,
ab6a046f 434 },
1e3baf05
DM
435 kvm => {
436 optional => 1,
437 type => 'boolean',
438 description => "Enable/disable KVM hardware virtualization.",
439 default => 1,
440 },
441 tdf => {
442 optional => 1,
443 type => 'boolean',
8c559505
DM
444 description => "Enable/disable time drift fix.",
445 default => 0,
1e3baf05 446 },
19672434 447 localtime => {
1e3baf05
DM
448 optional => 1,
449 type => 'boolean',
450 description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
451 },
452 freeze => {
453 optional => 1,
454 type => 'boolean',
455 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
456 },
457 vga => {
458 optional => 1,
55655ebc
DC
459 type => 'string', format => $vga_fmt,
460 description => "Configure the VGA hardware.",
461 verbose_description => "Configure the VGA Hardware. If you want to use ".
462 "high resolution modes (>= 1280x1024x16) you may need to increase " .
463 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
464 "is 'std' for all OS types besides some Windows versions (XP and " .
465 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
466 "display server. For win* OS you can select how many independent " .
467 "displays you want, Linux guests can add displays them self.\n".
468 "You can also run without any graphic card, using a serial device as terminal.",
1e3baf05 469 },
0ea9541d
DM
470 watchdog => {
471 optional => 1,
472 type => 'string', format => 'pve-qm-watchdog',
52261945
DM
473 description => "Create a virtual hardware watchdog device.",
474 verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
1917695c
TL
475 " (by a guest action), the watchdog must be periodically polled " .
476 "by an agent inside the guest or else the watchdog will reset " .
477 "the guest (or execute the respective action specified)",
0ea9541d 478 },
1e3baf05
DM
479 startdate => {
480 optional => 1,
19672434 481 type => 'string',
1e3baf05
DM
482 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
483 description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
484 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
485 default => 'now',
486 },
43574f73 487 startup => get_standard_option('pve-startup-order'),
68eda3ab
AD
488 template => {
489 optional => 1,
490 type => 'boolean',
491 description => "Enable/disable Template.",
492 default => 0,
493 },
1e3baf05
DM
494 args => {
495 optional => 1,
496 type => 'string',
52261945
DM
497 description => "Arbitrary arguments passed to kvm.",
498 verbose_description => <<EODESCR,
c7a8aad6 499Arbitrary arguments passed to kvm, for example:
1e3baf05
DM
500
501args: -no-reboot -no-hpet
c7a8aad6
FG
502
503NOTE: this option is for experts only.
1e3baf05
DM
504EODESCR
505 },
506 tablet => {
507 optional => 1,
508 type => 'boolean',
509 default => 1,
52261945
DM
510 description => "Enable/disable the USB tablet device.",
511 verbose_description => "Enable/disable the USB tablet device. This device is " .
1917695c
TL
512 "usually needed to allow absolute mouse positioning with VNC. " .
513 "Else the mouse runs out of sync with normal VNC clients. " .
514 "If you're running lots of console-only guests on one host, " .
515 "you may consider disabling this to save some context switches. " .
516 "This is turned off by default if you use spice (-vga=qxl).",
1e3baf05
DM
517 },
518 migrate_speed => {
519 optional => 1,
520 type => 'integer',
521 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
522 minimum => 0,
523 default => 0,
524 },
525 migrate_downtime => {
526 optional => 1,
04432191 527 type => 'number',
1e3baf05
DM
528 description => "Set maximum tolerated downtime (in seconds) for migrations.",
529 minimum => 0,
04432191 530 default => 0.1,
1e3baf05
DM
531 },
532 cdrom => {
533 optional => 1,
b799312f 534 type => 'string', format => 'pve-qm-ide',
8485b9ba 535 typetext => '<volume>',
1e3baf05
DM
536 description => "This is an alias for option -ide2",
537 },
538 cpu => {
539 optional => 1,
540 description => "Emulated CPU type.",
541 type => 'string',
ff6ffe20 542 format => $cpu_fmt,
1e3baf05 543 },
b7ba6b79
DM
544 parent => get_standard_option('pve-snapshot-name', {
545 optional => 1,
546 description => "Parent snapshot name. This is used internally, and should not be modified.",
547 }),
982c7f12
DM
548 snaptime => {
549 optional => 1,
550 description => "Timestamp for snapshots.",
551 type => 'integer',
552 minimum => 0,
553 },
18bfb361
DM
554 vmstate => {
555 optional => 1,
556 type => 'string', format => 'pve-volume-id',
557 description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
558 },
253624c7
FG
559 vmstatestorage => get_standard_option('pve-storage-id', {
560 description => "Default storage for VM state volumes/files.",
561 optional => 1,
562 }),
c6737ef1
DC
563 runningmachine => get_standard_option('pve-qemu-machine', {
564 description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
565 }),
566 machine => get_standard_option('pve-qemu-machine'),
d731ecbe
WB
567 arch => {
568 description => "Virtual processor architecture. Defaults to the host.",
569 optional => 1,
570 type => 'string',
571 enum => [qw(x86_64 aarch64)],
572 },
2796e7d5
DM
573 smbios1 => {
574 description => "Specify SMBIOS type 1 fields.",
575 type => 'string', format => 'pve-qm-smbios1',
2796e7d5
DM
576 maxLength => 256,
577 optional => 1,
578 },
cb0e4540
AG
579 protection => {
580 optional => 1,
581 type => 'boolean',
52261945 582 description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
cb0e4540
AG
583 default => 0,
584 },
3edb45e7 585 bios => {
a783c78e 586 optional => 1,
3edb45e7
DM
587 type => 'string',
588 enum => [ qw(seabios ovmf) ],
589 description => "Select BIOS implementation.",
590 default => 'seabios',
a783c78e 591 },
6ee499ff
DC
592 vmgenid => {
593 type => 'string',
594 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
595 format_description => 'UUID',
f7ed64e7
TL
596 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
597 verbose_description => "The VM generation ID (vmgenid) device exposes a".
598 " 128-bit integer value identifier to the guest OS. This allows to".
599 " notify the guest operating system when the virtual machine is".
600 " executed with a different configuration (e.g. snapshot execution".
601 " or creation from a template). The guest operating system notices".
602 " the change, and is then able to react as appropriate by marking".
603 " its copies of distributed databases as dirty, re-initializing its".
604 " random number generator, etc.\n".
605 "Note that auto-creation only works when done throug API/CLI create".
606 " or update methods, but not when manually editing the config file.",
607 default => "1 (autogenerated)",
6ee499ff
DC
608 optional => 1,
609 },
9ed7a77c
WB
610};
611
612my $confdesc_cloudinit = {
41cd94a0
WB
613 citype => {
614 optional => 1,
615 type => 'string',
498cdc36 616 description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
41cd94a0
WB
617 enum => ['configdrive2', 'nocloud'],
618 },
7b42f951
WB
619 ciuser => {
620 optional => 1,
621 type => 'string',
622 description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
623 },
624 cipassword => {
625 optional => 1,
626 type => 'string',
1d1c4e1c 627 description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
7b42f951 628 },
0c9a7596
AD
629 searchdomain => {
630 optional => 1,
631 type => 'string',
632 description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
633 },
634 nameserver => {
635 optional => 1,
636 type => 'string', format => 'address-list',
637 description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
638 },
639 sshkeys => {
640 optional => 1,
641 type => 'string',
642 format => 'urlencoded',
1d1c4e1c 643 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
0c9a7596 644 },
1e3baf05
DM
645};
646
647# what about other qemu settings ?
648#cpu => 'string',
649#machine => 'string',
650#fda => 'file',
651#fdb => 'file',
652#mtdblock => 'file',
653#sd => 'file',
654#pflash => 'file',
655#snapshot => 'bool',
656#bootp => 'file',
657##tftp => 'dir',
658##smb => 'dir',
659#kernel => 'file',
660#append => 'string',
661#initrd => 'file',
662##soundhw => 'string',
663
664while (my ($k, $v) = each %$confdesc) {
665 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
666}
667
668my $MAX_IDE_DISKS = 4;
f62db2a4 669my $MAX_SCSI_DISKS = 14;
a2650619 670my $MAX_VIRTIO_DISKS = 16;
cdb0931f 671my $MAX_SATA_DISKS = 6;
1e3baf05 672my $MAX_USB_DEVICES = 5;
5bdcf937 673my $MAX_NETS = 32;
c9db2240 674my $MAX_UNUSED_DISKS = 256;
5cffb2d2 675my $MAX_HOSTPCI_DEVICES = 4;
bae179aa 676my $MAX_SERIAL_PORTS = 4;
1989a89c 677my $MAX_PARALLEL_PORTS = 3;
2ed5d572
AD
678my $MAX_NUMA = 8;
679
ffc0d8c7
WB
680my $numa_fmt = {
681 cpus => {
682 type => "string",
683 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
52261945 684 description => "CPUs accessing this NUMA node.",
ffc0d8c7
WB
685 format_description => "id[-id];...",
686 },
687 memory => {
688 type => "number",
52261945 689 description => "Amount of memory this NUMA node provides.",
ffc0d8c7
WB
690 optional => 1,
691 },
692 hostnodes => {
693 type => "string",
694 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
52261945 695 description => "Host NUMA nodes to use.",
ffc0d8c7
WB
696 format_description => "id[-id];...",
697 optional => 1,
698 },
699 policy => {
700 type => 'string',
701 enum => [qw(preferred bind interleave)],
52261945 702 description => "NUMA allocation policy.",
ffc0d8c7
WB
703 optional => 1,
704 },
705};
706PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
2ed5d572
AD
707my $numadesc = {
708 optional => 1,
ffc0d8c7 709 type => 'string', format => $numa_fmt,
52261945 710 description => "NUMA topology.",
2ed5d572
AD
711};
712PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
713
714for (my $i = 0; $i < $MAX_NUMA; $i++) {
715 $confdesc->{"numa$i"} = $numadesc;
716}
1e3baf05
DM
717
718my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
55034103
KT
719 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
720 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
6b64503e 721my $nic_model_list_txt = join(' ', sort @$nic_model_list);
1e3baf05 722
52261945
DM
723my $net_fmt_bridge_descr = <<__EOD__;
724Bridge to attach the network device to. The Proxmox VE standard bridge
725is called 'vmbr0'.
726
727If you do not specify a bridge, we create a kvm user (NATed) network
728device, which provides DHCP and DNS services. The following addresses
729are used:
730
731 10.0.2.2 Gateway
732 10.0.2.3 DNS Server
733 10.0.2.4 SMB Server
734
735The DHCP server assign addresses to the guest starting from 10.0.2.15.
736__EOD__
737
cd9c34d1
WB
738my $net_fmt = {
739 macaddr => {
740 type => 'string',
741 pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
52261945 742 description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
cd9c34d1
WB
743 format_description => "XX:XX:XX:XX:XX:XX",
744 optional => 1,
745 },
7f694a71
DM
746 model => {
747 type => 'string',
52261945 748 description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
7f694a71
DM
749 enum => $nic_model_list,
750 default_key => 1,
751 },
752 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
cd9c34d1
WB
753 bridge => {
754 type => 'string',
52261945 755 description => $net_fmt_bridge_descr,
cd9c34d1
WB
756 format_description => 'bridge',
757 optional => 1,
758 },
759 queues => {
760 type => 'integer',
761 minimum => 0, maximum => 16,
762 description => 'Number of packet queues to be used on the device.',
cd9c34d1
WB
763 optional => 1,
764 },
765 rate => {
766 type => 'number',
767 minimum => 0,
52261945 768 description => "Rate limit in mbps (megabytes per second) as floating point number.",
cd9c34d1
WB
769 optional => 1,
770 },
771 tag => {
772 type => 'integer',
9f41a659 773 minimum => 1, maximum => 4094,
cd9c34d1 774 description => 'VLAN tag to apply to packets on this interface.',
cd9c34d1
WB
775 optional => 1,
776 },
777 trunks => {
778 type => 'string',
779 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
780 description => 'VLAN trunks to pass through this interface.',
7f694a71 781 format_description => 'vlanid[;vlanid...]',
cd9c34d1
WB
782 optional => 1,
783 },
784 firewall => {
785 type => 'boolean',
786 description => 'Whether this interface should be protected by the firewall.',
cd9c34d1
WB
787 optional => 1,
788 },
789 link_down => {
790 type => 'boolean',
52261945 791 description => 'Whether this interface should be disconnected (like pulling the plug).',
cd9c34d1
WB
792 optional => 1,
793 },
794};
52261945 795
1e3baf05
DM
796my $netdesc = {
797 optional => 1,
7f694a71 798 type => 'string', format => $net_fmt,
52261945 799 description => "Specify network devices.",
1e3baf05 800};
52261945 801
1e3baf05
DM
802PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
803
0c9a7596
AD
804my $ipconfig_fmt = {
805 ip => {
806 type => 'string',
807 format => 'pve-ipv4-config',
808 format_description => 'IPv4Format/CIDR',
809 description => 'IPv4 address in CIDR format.',
810 optional => 1,
811 default => 'dhcp',
812 },
813 gw => {
814 type => 'string',
815 format => 'ipv4',
816 format_description => 'GatewayIPv4',
817 description => 'Default gateway for IPv4 traffic.',
818 optional => 1,
819 requires => 'ip',
820 },
821 ip6 => {
822 type => 'string',
823 format => 'pve-ipv6-config',
824 format_description => 'IPv6Format/CIDR',
825 description => 'IPv6 address in CIDR format.',
826 optional => 1,
827 default => 'dhcp',
828 },
829 gw6 => {
830 type => 'string',
831 format => 'ipv6',
832 format_description => 'GatewayIPv6',
833 description => 'Default gateway for IPv6 traffic.',
834 optional => 1,
835 requires => 'ip6',
836 },
837};
838PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
839my $ipconfigdesc = {
840 optional => 1,
841 type => 'string', format => 'pve-qm-ipconfig',
842 description => <<'EODESCR',
843cloud-init: Specify IP addresses and gateways for the corresponding interface.
844
845IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
846
847The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
848For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
849
850If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
851EODESCR
852};
853PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
854
1e3baf05
DM
855for (my $i = 0; $i < $MAX_NETS; $i++) {
856 $confdesc->{"net$i"} = $netdesc;
9ed7a77c
WB
857 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
858}
859
860foreach my $key (keys %$confdesc_cloudinit) {
861 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1e3baf05
DM
862}
863
ffa42b86
DC
864PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
865sub verify_volume_id_or_qm_path {
822c8a07
WB
866 my ($volid, $noerr) = @_;
867
ffa42b86
DC
868 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
869 return $volid;
870 }
871
872 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
822c8a07
WB
873 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
874 if ($@) {
875 return undef if $noerr;
876 die $@;
877 }
878 return $volid;
879}
880
1e3baf05 881my $drivename_hash;
19672434 882
0541eeb8
WB
883my %drivedesc_base = (
884 volume => { alias => 'file' },
885 file => {
93c0971c 886 type => 'string',
ffa42b86 887 format => 'pve-volume-id-or-qm-path',
0541eeb8
WB
888 default_key => 1,
889 format_description => 'volume',
890 description => "The drive's backing volume.",
891 },
892 media => {
893 type => 'string',
0541eeb8
WB
894 enum => [qw(cdrom disk)],
895 description => "The drive's media type.",
896 default => 'disk',
897 optional => 1
898 },
899 cyls => {
900 type => 'integer',
0541eeb8
WB
901 description => "Force the drive's physical geometry to have a specific cylinder count.",
902 optional => 1
903 },
904 heads => {
905 type => 'integer',
0541eeb8
WB
906 description => "Force the drive's physical geometry to have a specific head count.",
907 optional => 1
908 },
909 secs => {
910 type => 'integer',
0541eeb8
WB
911 description => "Force the drive's physical geometry to have a specific sector count.",
912 optional => 1
913 },
914 trans => {
915 type => 'string',
0541eeb8
WB
916 enum => [qw(none lba auto)],
917 description => "Force disk geometry bios translation mode.",
918 optional => 1,
919 },
920 snapshot => {
921 type => 'boolean',
bfb04cfc
WB
922 description => "Controls qemu's snapshot mode feature."
923 . " If activated, changes made to the disk are temporary and will"
924 . " be discarded when the VM is shutdown.",
0541eeb8
WB
925 optional => 1,
926 },
927 cache => {
928 type => 'string',
0541eeb8
WB
929 enum => [qw(none writethrough writeback unsafe directsync)],
930 description => "The drive's cache mode",
931 optional => 1,
932 },
c7d2b650 933 format => get_standard_option('pve-qm-image-format'),
0541eeb8 934 size => {
47c28a68
WB
935 type => 'string',
936 format => 'disk-size',
7f694a71 937 format_description => 'DiskSize',
0541eeb8
WB
938 description => "Disk size. This is purely informational and has no effect.",
939 optional => 1,
940 },
941 backup => {
942 type => 'boolean',
0541eeb8
WB
943 description => "Whether the drive should be included when making backups.",
944 optional => 1,
945 },
8557d01f 946 replicate => {
9edac22f 947 type => 'boolean',
3ab7663a 948 description => 'Whether the drive should considered for replication jobs.',
9edac22f
WL
949 optional => 1,
950 default => 1,
951 },
6e9d2550
AD
952 rerror => {
953 type => 'string',
954 enum => [qw(ignore report stop)],
955 description => 'Read error action.',
956 optional => 1,
957 },
0541eeb8
WB
958 werror => {
959 type => 'string',
0541eeb8
WB
960 enum => [qw(enospc ignore report stop)],
961 description => 'Write error action.',
962 optional => 1,
963 },
964 aio => {
965 type => 'string',
0541eeb8
WB
966 enum => [qw(native threads)],
967 description => 'AIO type to use.',
968 optional => 1,
969 },
970 discard => {
971 type => 'string',
0541eeb8
WB
972 enum => [qw(ignore on)],
973 description => 'Controls whether to pass discard/trim requests to the underlying storage.',
974 optional => 1,
975 },
976 detect_zeroes => {
977 type => 'boolean',
978 description => 'Controls whether to detect and try to optimize writes of zeroes.',
979 optional => 1,
980 },
981 serial => {
982 type => 'string',
46630a5f 983 format => 'urlencoded',
0541eeb8 984 format_description => 'serial',
ba8fc5d1
WB
985 maxLength => 20*3, # *3 since it's %xx url enoded
986 description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
0541eeb8 987 optional => 1,
ec82e3ee
CH
988 },
989 shared => {
990 type => 'boolean',
991 description => 'Mark this locally-managed volume as available on all nodes',
992 verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
993 optional => 1,
994 default => 0,
0541eeb8
WB
995 }
996);
997
0541eeb8
WB
998my %iothread_fmt = ( iothread => {
999 type => 'boolean',
0541eeb8
WB
1000 description => "Whether to use iothreads for this drive",
1001 optional => 1,
1002});
1003
1004my %model_fmt = (
1005 model => {
1006 type => 'string',
46630a5f 1007 format => 'urlencoded',
0541eeb8 1008 format_description => 'model',
ba8fc5d1
WB
1009 maxLength => 40*3, # *3 since it's %xx url enoded
1010 description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
0541eeb8
WB
1011 optional => 1,
1012 },
1013);
1014
1015my %queues_fmt = (
1016 queues => {
1017 type => 'integer',
0541eeb8
WB
1018 description => "Number of queues.",
1019 minimum => 2,
1020 optional => 1
1021 }
1022);
1023
8e3c33ab
FG
1024my %scsiblock_fmt = (
1025 scsiblock => {
1026 type => 'boolean',
1027 description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1028 optional => 1,
1029 default => 0,
1030 },
1031);
1032
6c875f9f
NC
1033my %ssd_fmt = (
1034 ssd => {
1035 type => 'boolean',
1036 description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1037 optional => 1,
1038 },
1039);
1040
0541eeb8 1041my $add_throttle_desc = sub {
9196a8ec
WB
1042 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1043 my $d = {
0541eeb8 1044 type => $type,
7f694a71 1045 format_description => $unit,
9196a8ec 1046 description => "Maximum $what in $longunit.",
0541eeb8
WB
1047 optional => 1,
1048 };
9196a8ec
WB
1049 $d->{minimum} = $minimum if defined($minimum);
1050 $drivedesc_base{$key} = $d;
0541eeb8
WB
1051};
1052# throughput: (leaky bucket)
d3f3f1b3
DM
1053$add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1054$add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1055$add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1056$add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1057$add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1058$add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1059$add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1060$add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1061$add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
0541eeb8
WB
1062
1063# pools: (pool of IO before throttling starts taking effect)
d3f3f1b3
DM
1064$add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1065$add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1066$add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1067$add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1068$add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1069$add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
9196a8ec
WB
1070
1071# burst lengths
fb8e95a2
WB
1072$add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1073$add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1074$add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1075$add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1076$add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1077$add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1078
1079# legacy support
1080$drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1081$drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1082$drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1083$drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
0541eeb8
WB
1084
1085my $ide_fmt = {
1086 %drivedesc_base,
0541eeb8 1087 %model_fmt,
6c875f9f 1088 %ssd_fmt,
0541eeb8 1089};
b799312f 1090PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
0541eeb8 1091
1e3baf05
DM
1092my $idedesc = {
1093 optional => 1,
0541eeb8 1094 type => 'string', format => $ide_fmt,
3c770faa 1095 description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1e3baf05
DM
1096};
1097PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1098
0541eeb8
WB
1099my $scsi_fmt = {
1100 %drivedesc_base,
1101 %iothread_fmt,
1102 %queues_fmt,
8e3c33ab 1103 %scsiblock_fmt,
6c875f9f 1104 %ssd_fmt,
0541eeb8 1105};
1e3baf05
DM
1106my $scsidesc = {
1107 optional => 1,
0541eeb8 1108 type => 'string', format => $scsi_fmt,
3c770faa 1109 description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1e3baf05
DM
1110};
1111PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1112
0541eeb8
WB
1113my $sata_fmt = {
1114 %drivedesc_base,
6c875f9f 1115 %ssd_fmt,
0541eeb8 1116};
cdb0931f
DA
1117my $satadesc = {
1118 optional => 1,
0541eeb8 1119 type => 'string', format => $sata_fmt,
3c770faa 1120 description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
cdb0931f
DA
1121};
1122PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1123
0541eeb8
WB
1124my $virtio_fmt = {
1125 %drivedesc_base,
1126 %iothread_fmt,
0541eeb8 1127};
1e3baf05
DM
1128my $virtiodesc = {
1129 optional => 1,
0541eeb8 1130 type => 'string', format => $virtio_fmt,
3c770faa 1131 description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1e3baf05
DM
1132};
1133PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1134
0541eeb8
WB
1135my $alldrive_fmt = {
1136 %drivedesc_base,
0541eeb8
WB
1137 %iothread_fmt,
1138 %model_fmt,
1139 %queues_fmt,
8e3c33ab 1140 %scsiblock_fmt,
6c875f9f 1141 %ssd_fmt,
0541eeb8
WB
1142};
1143
6470743f
DC
1144my $efidisk_fmt = {
1145 volume => { alias => 'file' },
1146 file => {
1147 type => 'string',
1148 format => 'pve-volume-id-or-qm-path',
1149 default_key => 1,
1150 format_description => 'volume',
1151 description => "The drive's backing volume.",
1152 },
c7d2b650 1153 format => get_standard_option('pve-qm-image-format'),
6470743f
DC
1154 size => {
1155 type => 'string',
1156 format => 'disk-size',
1157 format_description => 'DiskSize',
1158 description => "Disk size. This is purely informational and has no effect.",
1159 optional => 1,
1160 },
1161};
1162
1163my $efidisk_desc = {
1164 optional => 1,
1165 type => 'string', format => $efidisk_fmt,
1166 description => "Configure a Disk for storing EFI vars",
1167};
1168
1169PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1170
ff6ffe20 1171my $usb_fmt = {
a6b9aee4
DC
1172 host => {
1173 default_key => 1,
1174 type => 'string', format => 'pve-qm-usb-device',
1175 format_description => 'HOSTUSBDEVICE|spice',
52261945
DM
1176 description => <<EODESCR,
1177The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1178
1179 'bus-port(.port)*' (decimal numbers) or
1180 'vendor_id:product_id' (hexadeciaml numbers) or
1181 'spice'
1182
1183You can use the 'lsusb -t' command to list existing usb devices.
1184
1185NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1186
1187The value 'spice' can be used to add a usb redirection devices for spice.
1188EODESCR
a6b9aee4
DC
1189 },
1190 usb3 => {
1191 optional => 1,
1192 type => 'boolean',
52261945
DM
1193 description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1194 default => 0,
a6b9aee4
DC
1195 },
1196};
1197
1e3baf05
DM
1198my $usbdesc = {
1199 optional => 1,
ff6ffe20 1200 type => 'string', format => $usb_fmt,
52261945 1201 description => "Configure an USB device (n is 0 to 4).",
1e3baf05
DM
1202};
1203PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1204
2fd24788 1205my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1f4f447b
WB
1206my $hostpci_fmt = {
1207 host => {
1208 default_key => 1,
1209 type => 'string',
1210 pattern => qr/$PCIRE(;$PCIRE)*/,
1211 format_description => 'HOSTPCIID[;HOSTPCIID2...]',
52261945
DM
1212 description => <<EODESCR,
1213Host PCI device pass through. The PCI ID of a host's PCI device or a list
1214of PCI virtual functions of the host. HOSTPCIID syntax is:
1215
1216'bus:dev.func' (hexadecimal numbers)
1217
1218You can us the 'lspci' command to list existing PCI devices.
52261945 1219EODESCR
1f4f447b
WB
1220 },
1221 rombar => {
1222 type => 'boolean',
52261945 1223 description => "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1f4f447b
WB
1224 optional => 1,
1225 default => 1,
1226 },
456a6fec
AD
1227 romfile => {
1228 type => 'string',
1229 pattern => '[^,;]+',
1230 format_description => 'string',
1231 description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1232 optional => 1,
1233 },
1f4f447b
WB
1234 pcie => {
1235 type => 'boolean',
52261945 1236 description => "Choose the PCI-express bus (needs the 'q35' machine model).",
1f4f447b
WB
1237 optional => 1,
1238 default => 0,
1239 },
1240 'x-vga' => {
1241 type => 'boolean',
52261945 1242 description => "Enable vfio-vga device support.",
1f4f447b
WB
1243 optional => 1,
1244 default => 0,
1245 },
6ab45bd7
DC
1246 'mdev' => {
1247 type => 'string',
1248 format_description => 'string',
1249 pattern => '[^/\.:]+',
1250 optional => 1,
1251 description => <<EODESCR
1252The type of mediated device to use.
1253An instance of this type will be created on startup of the VM and
1254will be cleaned up when the VM stops.
1255EODESCR
1256 }
1f4f447b
WB
1257};
1258PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1259
040b06b7
DA
1260my $hostpcidesc = {
1261 optional => 1,
1262 type => 'string', format => 'pve-qm-hostpci',
52261945 1263 description => "Map host PCI devices into guest.",
faab5306
DM
1264 verbose_description => <<EODESCR,
1265Map host PCI devices into guest.
1266
1267NOTE: This option allows direct access to host hardware. So it is no longer
1268possible to migrate such machines - use with special care.
1269
1270CAUTION: Experimental! User reported problems with this option.
1271EODESCR
040b06b7
DA
1272};
1273PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1274
bae179aa
DA
1275my $serialdesc = {
1276 optional => 1,
ca0cef26 1277 type => 'string',
1b0b51ed 1278 pattern => '(/dev/.+|socket)',
52261945
DM
1279 description => "Create a serial device inside the VM (n is 0 to 3)",
1280 verbose_description => <<EODESCR,
52261945
DM
1281Create a serial device inside the VM (n is 0 to 3), and pass through a
1282host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1283host side (use 'qm terminal' to open a terminal connection).
bae179aa 1284
8a61e0fd 1285NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
bae179aa 1286
52261945 1287CAUTION: Experimental! User reported problems with this option.
bae179aa
DA
1288EODESCR
1289};
bae179aa 1290
1989a89c
DA
1291my $paralleldesc= {
1292 optional => 1,
ca0cef26 1293 type => 'string',
9ecc8431 1294 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
52261945
DM
1295 description => "Map host parallel devices (n is 0 to 2).",
1296 verbose_description => <<EODESCR,
19672434 1297Map host parallel devices (n is 0 to 2).
1989a89c 1298
8a61e0fd 1299NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1989a89c 1300
52261945 1301CAUTION: Experimental! User reported problems with this option.
1989a89c
DA
1302EODESCR
1303};
1989a89c
DA
1304
1305for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1306 $confdesc->{"parallel$i"} = $paralleldesc;
1307}
1308
bae179aa
DA
1309for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1310 $confdesc->{"serial$i"} = $serialdesc;
1311}
1312
040b06b7
DA
1313for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1314 $confdesc->{"hostpci$i"} = $hostpcidesc;
1315}
1e3baf05
DM
1316
1317for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1318 $drivename_hash->{"ide$i"} = 1;
1319 $confdesc->{"ide$i"} = $idedesc;
1320}
1321
cdb0931f
DA
1322for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1323 $drivename_hash->{"sata$i"} = 1;
1324 $confdesc->{"sata$i"} = $satadesc;
1325}
1326
1e3baf05
DM
1327for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1328 $drivename_hash->{"scsi$i"} = 1;
1329 $confdesc->{"scsi$i"} = $scsidesc ;
1330}
1331
1332for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1333 $drivename_hash->{"virtio$i"} = 1;
1334 $confdesc->{"virtio$i"} = $virtiodesc;
1335}
1336
6470743f
DC
1337$drivename_hash->{efidisk0} = 1;
1338$confdesc->{efidisk0} = $efidisk_desc;
1339
1e3baf05
DM
1340for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1341 $confdesc->{"usb$i"} = $usbdesc;
1342}
1343
1344my $unuseddesc = {
1345 optional => 1,
1346 type => 'string', format => 'pve-volume-id',
52261945 1347 description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1e3baf05
DM
1348};
1349
1350for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1351 $confdesc->{"unused$i"} = $unuseddesc;
1352}
1353
1354my $kvm_api_version = 0;
1355
1356sub kvm_version {
1e3baf05
DM
1357 return $kvm_api_version if $kvm_api_version;
1358
646f2df4
WB
1359 open my $fh, '<', '/dev/kvm'
1360 or return undef;
1e3baf05 1361
646f2df4
WB
1362 # 0xae00 => KVM_GET_API_VERSION
1363 $kvm_api_version = ioctl($fh, 0xae00, 0);
1e3baf05 1364
646f2df4 1365 return $kvm_api_version;
1e3baf05
DM
1366}
1367
1368my $kvm_user_version;
1369
1370sub kvm_user_version {
1371
1372 return $kvm_user_version if $kvm_user_version;
1373
1374 $kvm_user_version = 'unknown';
1375
09b11429
TL
1376 my $code = sub {
1377 my $line = shift;
1378 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1379 $kvm_user_version = $2;
1380 }
1381 };
19672434 1382
09b11429
TL
1383 eval { run_command("kvm -version", outfunc => $code); };
1384 warn $@ if $@;
1e3baf05
DM
1385
1386 return $kvm_user_version;
1387
1388}
1389
db70021b
TL
1390sub kernel_has_vhost_net {
1391 return -c '/dev/vhost-net';
1392}
1e3baf05 1393
74479ee9 1394sub valid_drive_names {
1e3baf05 1395 # order is important - used to autoselect boot disk
19672434 1396 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1e3baf05 1397 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
cdb0931f 1398 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
6470743f
DC
1399 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1400 'efidisk0');
1e3baf05
DM
1401}
1402
74479ee9 1403sub is_valid_drivename {
1e3baf05
DM
1404 my $dev = shift;
1405
6b64503e 1406 return defined($drivename_hash->{$dev});
1e3baf05
DM
1407}
1408
1409sub option_exists {
1410 my $key = shift;
1411 return defined($confdesc->{$key});
19672434 1412}
1e3baf05
DM
1413
1414sub nic_models {
1415 return $nic_model_list;
1416}
1417
1418sub os_list_description {
1419
1420 return {
1421 other => 'Other',
1422 wxp => 'Windows XP',
1423 w2k => 'Windows 2000',
1424 w2k3 =>, 'Windows 2003',
1425 w2k8 => 'Windows 2008',
1426 wvista => 'Windows Vista',
1427 win7 => 'Windows 7',
a70ebde3 1428 win8 => 'Windows 8/2012',
0cb9971e 1429 win10 => 'Windows 10/2016',
1e3baf05
DM
1430 l24 => 'Linux 2.4',
1431 l26 => 'Linux 2.6',
19672434 1432 };
1e3baf05
DM
1433}
1434
1e3baf05
DM
1435my $cdrom_path;
1436
1437sub get_cdrom_path {
1438
1439 return $cdrom_path if $cdrom_path;
1440
1441 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1442 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1443 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1444}
1445
1446sub get_iso_path {
1447 my ($storecfg, $vmid, $cdrom) = @_;
1448
1449 if ($cdrom eq 'cdrom') {
1450 return get_cdrom_path();
1451 } elsif ($cdrom eq 'none') {
1452 return '';
1453 } elsif ($cdrom =~ m|^/|) {
1454 return $cdrom;
1455 } else {
6b64503e 1456 return PVE::Storage::path($storecfg, $cdrom);
1e3baf05
DM
1457 }
1458}
1459
1460# try to convert old style file names to volume IDs
1461sub filename_to_volume_id {
1462 my ($vmid, $file, $media) = @_;
1463
0c9a7596 1464 if (!($file eq 'none' || $file eq 'cdrom' ||
1e3baf05 1465 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
19672434 1466
1e3baf05 1467 return undef if $file =~ m|/|;
19672434 1468
1e3baf05
DM
1469 if ($media && $media eq 'cdrom') {
1470 $file = "local:iso/$file";
1471 } else {
1472 $file = "local:$vmid/$file";
1473 }
1474 }
1475
1476 return $file;
1477}
1478
1479sub verify_media_type {
1480 my ($opt, $vtype, $media) = @_;
1481
1482 return if !$media;
1483
1484 my $etype;
1485 if ($media eq 'disk') {
a125592c 1486 $etype = 'images';
1e3baf05
DM
1487 } elsif ($media eq 'cdrom') {
1488 $etype = 'iso';
1489 } else {
1490 die "internal error";
1491 }
1492
1493 return if ($vtype eq $etype);
19672434 1494
1e3baf05
DM
1495 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1496}
1497
1498sub cleanup_drive_path {
1499 my ($opt, $storecfg, $drive) = @_;
1500
1501 # try to convert filesystem paths to volume IDs
1502
1503 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1504 ($drive->{file} !~ m|^/dev/.+|) &&
1505 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
19672434 1506 ($drive->{file} !~ m/^\d+$/)) {
1e3baf05
DM
1507 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1508 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1509 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1510 verify_media_type($opt, $vtype, $drive->{media});
1511 $drive->{file} = $volid;
1512 }
1513
1514 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1515}
1516
b3c2bdd1
DM
1517sub parse_hotplug_features {
1518 my ($data) = @_;
1519
1520 my $res = {};
1521
1522 return $res if $data eq '0';
a1b7d579 1523
b3c2bdd1
DM
1524 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1525
45827685 1526 foreach my $feature (PVE::Tools::split_list($data)) {
b3c2bdd1
DM
1527 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1528 $res->{$1} = 1;
1529 } else {
596a0a20 1530 die "invalid hotplug feature '$feature'\n";
b3c2bdd1
DM
1531 }
1532 }
1533 return $res;
1534}
1535
1536PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1537sub pve_verify_hotplug_features {
1538 my ($value, $noerr) = @_;
1539
1540 return $value if parse_hotplug_features($value);
1541
1542 return undef if $noerr;
1543
1544 die "unable to parse hotplug option\n";
1545}
1546
1e3baf05
DM
1547# ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1548# [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
036e0e2b 1549# [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
6e47c3b4
WB
1550# [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1551# [,iothread=on][,serial=serial][,model=model]
1e3baf05
DM
1552
1553sub parse_drive {
1554 my ($key, $data) = @_;
1555
0541eeb8 1556 my ($interface, $index);
19672434 1557
0541eeb8
WB
1558 if ($key =~ m/^([^\d]+)(\d+)$/) {
1559 $interface = $1;
1560 $index = $2;
1e3baf05
DM
1561 } else {
1562 return undef;
1563 }
1564
0541eeb8
WB
1565 my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1566 : $confdesc->{$key}->{format};
1567 if (!$desc) {
1568 warn "invalid drive key: $key\n";
1569 return undef;
1570 }
1571 my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1572 return undef if !$res;
1573 $res->{interface} = $interface;
1574 $res->{index} = $index;
1575
1576 my $error = 0;
1577 foreach my $opt (qw(bps bps_rd bps_wr)) {
1578 if (my $bps = defined(delete $res->{$opt})) {
1579 if (defined($res->{"m$opt"})) {
1580 warn "both $opt and m$opt specified\n";
1581 ++$error;
1582 next;
1e3baf05 1583 }
0541eeb8 1584 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1e3baf05
DM
1585 }
1586 }
9196a8ec
WB
1587
1588 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1589 for my $requirement (
fb8e95a2
WB
1590 [mbps_max => 'mbps'],
1591 [mbps_rd_max => 'mbps_rd'],
1592 [mbps_wr_max => 'mbps_wr'],
1593 [miops_max => 'miops'],
1594 [miops_rd_max => 'miops_rd'],
1595 [miops_wr_max => 'miops_wr'],
9196a8ec
WB
1596 [bps_max_length => 'mbps_max'],
1597 [bps_rd_max_length => 'mbps_rd_max'],
1598 [bps_wr_max_length => 'mbps_wr_max'],
1599 [iops_max_length => 'iops_max'],
1600 [iops_rd_max_length => 'iops_rd_max'],
1601 [iops_wr_max_length => 'iops_wr_max']) {
1602 my ($option, $requires) = @$requirement;
1603 if ($res->{$option} && !$res->{$requires}) {
1604 warn "$option requires $requires\n";
1605 ++$error;
1606 }
1607 }
1608
0541eeb8 1609 return undef if $error;
be190583 1610
9bf371a6
DM
1611 return undef if $res->{mbps_rd} && $res->{mbps};
1612 return undef if $res->{mbps_wr} && $res->{mbps};
affd2f88
AD
1613 return undef if $res->{iops_rd} && $res->{iops};
1614 return undef if $res->{iops_wr} && $res->{iops};
74edd76b 1615
1e3baf05
DM
1616 if ($res->{media} && ($res->{media} eq 'cdrom')) {
1617 return undef if $res->{snapshot} || $res->{trans} || $res->{format};
19672434 1618 return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1e3baf05
DM
1619 return undef if $res->{interface} eq 'virtio';
1620 }
1621
0541eeb8
WB
1622 if (my $size = $res->{size}) {
1623 return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1e3baf05
DM
1624 }
1625
1626 return $res;
1627}
1628
1e3baf05
DM
1629sub print_drive {
1630 my ($vmid, $drive) = @_;
0541eeb8
WB
1631 my $data = { %$drive };
1632 delete $data->{$_} for qw(index interface);
1633 return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1e3baf05
DM
1634}
1635
28ef82d3
DM
1636sub scsi_inquiry {
1637 my($fh, $noerr) = @_;
1638
1639 my $SG_IO = 0x2285;
1640 my $SG_GET_VERSION_NUM = 0x2282;
1641
1642 my $versionbuf = "\x00" x 8;
1643 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1644 if (!$ret) {
1645 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1646 return undef;
1647 }
97d62eb7 1648 my $version = unpack("I", $versionbuf);
28ef82d3
DM
1649 if ($version < 30000) {
1650 die "scsi generic interface too old\n" if !$noerr;
1651 return undef;
1652 }
97d62eb7 1653
28ef82d3
DM
1654 my $buf = "\x00" x 36;
1655 my $sensebuf = "\x00" x 8;
f334aa3e 1656 my $cmd = pack("C x3 C x1", 0x12, 36);
97d62eb7 1657
28ef82d3
DM
1658 # see /usr/include/scsi/sg.h
1659 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1660
97d62eb7
DM
1661 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1662 length($sensebuf), 0, length($buf), $buf,
28ef82d3
DM
1663 $cmd, $sensebuf, 6000);
1664
1665 $ret = ioctl($fh, $SG_IO, $packet);
1666 if (!$ret) {
1667 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1668 return undef;
1669 }
97d62eb7 1670
28ef82d3
DM
1671 my @res = unpack($sg_io_hdr_t, $packet);
1672 if ($res[17] || $res[18]) {
1673 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1674 return undef;
1675 }
1676
1677 my $res = {};
09984754 1678 (my $byte0, my $byte1, $res->{vendor},
28ef82d3
DM
1679 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1680
09984754
DM
1681 $res->{removable} = $byte1 & 128 ? 1 : 0;
1682 $res->{type} = $byte0 & 31;
1683
28ef82d3
DM
1684 return $res;
1685}
1686
1687sub path_is_scsi {
1688 my ($path) = @_;
1689
1690 my $fh = IO::File->new("+<$path") || return undef;
1691 my $res = scsi_inquiry($fh, 1);
1692 close($fh);
1693
1694 return $res;
1695}
1696
db656e5f
DM
1697sub machine_type_is_q35 {
1698 my ($conf) = @_;
b467f79a 1699
db656e5f
DM
1700 return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1701}
1702
1703sub print_tabletdevice_full {
d559309f 1704 my ($conf, $arch) = @_;
b467f79a 1705
db656e5f
DM
1706 my $q35 = machine_type_is_q35($conf);
1707
1708 # we use uhci for old VMs because tablet driver was buggy in older qemu
d559309f
WB
1709 my $usbbus;
1710 if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1711 $usbbus = 'ehci';
1712 } else {
1713 $usbbus = 'uhci';
1714 }
b467f79a 1715
db656e5f
DM
1716 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1717}
1718
d559309f
WB
1719sub print_keyboarddevice_full {
1720 my ($conf, $arch, $machine) = @_;
1721
1722 return undef if $arch ne 'aarch64';
1723
1724 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1725}
1726
ca916ecc 1727sub print_drivedevice_full {
d559309f 1728 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
ca916ecc
DA
1729
1730 my $device = '';
1731 my $maxdev = 0;
19672434 1732
ca916ecc 1733 if ($drive->{interface} eq 'virtio') {
d559309f 1734 my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
2ed36a41 1735 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
51f492cd 1736 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
2ed36a41 1737 } elsif ($drive->{interface} eq 'scsi') {
6731a4cf 1738
ee034f5c 1739 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
2ed36a41
DM
1740 my $unit = $drive->{index} % $maxdev;
1741 my $devicetype = 'hd';
69bcf246
WB
1742 my $path = '';
1743 if (drive_is_cdrom($drive)) {
1744 $devicetype = 'cd';
29b19529 1745 } else {
69bcf246
WB
1746 if ($drive->{file} =~ m|^/|) {
1747 $path = $drive->{file};
1748 if (my $info = path_is_scsi($path)) {
8e3c33ab 1749 if ($info->{type} == 0 && $drive->{scsiblock}) {
69bcf246
WB
1750 $devicetype = 'block';
1751 } elsif ($info->{type} == 1) { # tape
1752 $devicetype = 'generic';
1753 }
1754 }
1755 } else {
1756 $path = PVE::Storage::path($storecfg, $drive->{file});
1757 }
1758
1759 if($path =~ m/^iscsi\:\/\//){
1760 $devicetype = 'generic';
1761 }
1762 }
1763
1764 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1765 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1766 } else {
1767 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1768 }
cdd20088 1769
6c875f9f
NC
1770 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1771 $device .= ",rotation_rate=1";
1772 }
1773
1774 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1775 my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
2ed36a41
DM
1776 my $controller = int($drive->{index} / $maxdev);
1777 my $unit = $drive->{index} % $maxdev;
1778 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1779
6c875f9f
NC
1780 $device = "ide-$devicetype";
1781 if ($drive->{interface} eq 'ide') {
1782 $device .= ",bus=ide.$controller,unit=$unit";
1783 } else {
1784 $device .= ",bus=ahci$controller.$unit";
1785 }
1786 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1787
1788 if ($devicetype eq 'hd') {
1789 if (my $model = $drive->{model}) {
1790 $model = URI::Escape::uri_unescape($model);
1791 $device .= ",model=$model";
1792 }
1793 if ($drive->{ssd}) {
1794 $device .= ",rotation_rate=1";
1795 }
0f2812c2 1796 }
2ed36a41
DM
1797 } elsif ($drive->{interface} eq 'usb') {
1798 die "implement me";
1799 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1800 } else {
1801 die "unsupported interface type";
ca916ecc
DA
1802 }
1803
3b408e82
DM
1804 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1805
a70e7e6c
TL
1806 if (my $serial = $drive->{serial}) {
1807 $serial = URI::Escape::uri_unescape($serial);
1808 $device .= ",serial=$serial";
1809 }
1810
1811
ca916ecc
DA
1812 return $device;
1813}
1814
15b21acc 1815sub get_initiator_name {
46f58b5f 1816 my $initiator;
15b21acc 1817
46f58b5f
DM
1818 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1819 while (defined(my $line = <$fh>)) {
1820 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
15b21acc
MR
1821 $initiator = $1;
1822 last;
1823 }
46f58b5f
DM
1824 $fh->close();
1825
15b21acc
MR
1826 return $initiator;
1827}
1828
1e3baf05
DM
1829sub print_drive_full {
1830 my ($storecfg, $vmid, $drive) = @_;
1831
d81f0f09
DM
1832 my $path;
1833 my $volid = $drive->{file};
1834 my $format;
1835
1836 if (drive_is_cdrom($drive)) {
1837 $path = get_iso_path($storecfg, $vmid, $volid);
1838 } else {
1839 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1840 if ($storeid) {
1841 $path = PVE::Storage::path($storecfg, $volid);
1842 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1843 $format = qemu_img_format($scfg, $volname);
1844 } else {
1845 $path = $volid;
5b61bff2 1846 $format = "raw";
d81f0f09
DM
1847 }
1848 }
1849
1e3baf05 1850 my $opts = '';
8a267708 1851 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1e3baf05 1852 foreach my $o (@qemu_drive_options) {
5fc74861 1853 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
19672434 1854 }
8a267708
DC
1855
1856 # snapshot only accepts on|off
1857 if (defined($drive->{snapshot})) {
1858 my $v = $drive->{snapshot} ? 'on' : 'off';
1859 $opts .= ",snapshot=$v";
1860 }
1861
fb8e95a2
WB
1862 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1863 my ($dir, $qmpname) = @$type;
1864 if (my $v = $drive->{"mbps$dir"}) {
1865 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1866 }
1867 if (my $v = $drive->{"mbps${dir}_max"}) {
1868 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1869 }
1870 if (my $v = $drive->{"bps${dir}_max_length"}) {
1871 $opts .= ",throttling.bps$qmpname-max-length=$v";
1872 }
1873 if (my $v = $drive->{"iops${dir}"}) {
1874 $opts .= ",throttling.iops$qmpname=$v";
1875 }
1876 if (my $v = $drive->{"iops${dir}_max"}) {
8aca1654 1877 $opts .= ",throttling.iops$qmpname-max=$v";
fb8e95a2
WB
1878 }
1879 if (my $v = $drive->{"iops${dir}_max_length"}) {
8aca1654 1880 $opts .= ",throttling.iops$qmpname-max-length=$v";
fb8e95a2
WB
1881 }
1882 }
1883
d81f0f09
DM
1884 $opts .= ",format=$format" if $format && !$drive->{format};
1885
b2ee900e
WB
1886 my $cache_direct = 0;
1887
1888 if (my $cache = $drive->{cache}) {
1889 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1890 } elsif (!drive_is_cdrom($drive)) {
1891 $opts .= ",cache=none";
1892 $cache_direct = 1;
1893 }
1894
1895 # aio native works only with O_DIRECT
1896 if (!$drive->{aio}) {
1897 if($cache_direct) {
1898 $opts .= ",aio=native";
1899 } else {
1900 $opts .= ",aio=threads";
1901 }
1902 }
11490cf2 1903
6e47c3b4
WB
1904 if (!drive_is_cdrom($drive)) {
1905 my $detectzeroes;
7d4e30f3 1906 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
6e47c3b4
WB
1907 $detectzeroes = 'off';
1908 } elsif ($drive->{discard}) {
1909 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1910 } else {
1911 # This used to be our default with discard not being specified:
1912 $detectzeroes = 'on';
1913 }
1914 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1915 }
f1e05305 1916
1e3baf05
DM
1917 my $pathinfo = $path ? "file=$path," : '';
1918
3ebfcc86 1919 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1e3baf05
DM
1920}
1921
cc4d6182 1922sub print_netdevice_full {
d559309f 1923 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
cc4d6182
DA
1924
1925 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1926
1927 my $device = $net->{model};
1928 if ($net->{model} eq 'virtio') {
1929 $device = 'virtio-net-pci';
1930 };
1931
d559309f 1932 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
5e2068d2 1933 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
a9410357
AD
1934 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1935 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1936 my $vectors = $net->{queues} * 2 + 2;
1937 $tmpstr .= ",vectors=$vectors,mq=on";
1938 }
cc4d6182 1939 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
ba9e1000
DM
1940
1941 if ($use_old_bios_files) {
1942 my $romfile;
1943 if ($device eq 'virtio-net-pci') {
1944 $romfile = 'pxe-virtio.rom';
1945 } elsif ($device eq 'e1000') {
1946 $romfile = 'pxe-e1000.rom';
1947 } elsif ($device eq 'ne2k') {
1948 $romfile = 'pxe-ne2k_pci.rom';
1949 } elsif ($device eq 'pcnet') {
1950 $romfile = 'pxe-pcnet.rom';
1951 } elsif ($device eq 'rtl8139') {
1952 $romfile = 'pxe-rtl8139.rom';
1953 }
1954 $tmpstr .= ",romfile=$romfile" if $romfile;
1955 }
1956
cc4d6182
DA
1957 return $tmpstr;
1958}
1959
1960sub print_netdev_full {
d559309f 1961 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
cc4d6182
DA
1962
1963 my $i = '';
1964 if ($netid =~ m/^net(\d+)$/) {
1965 $i = int($1);
1966 }
1967
1968 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1969
1970 my $ifname = "tap${vmid}i$i";
1971
1972 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1973 die "interface name '$ifname' is too long (max 15 character)\n"
1974 if length($ifname) >= 16;
1975
1976 my $vhostparam = '';
6f0cb675 1977 if (is_native($arch)) {
db70021b 1978 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
6f0cb675 1979 }
cc4d6182
DA
1980
1981 my $vmname = $conf->{name} || "vm$vmid";
1982
a9410357 1983 my $netdev = "";
208ba94e 1984 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
a9410357 1985
cc4d6182 1986 if ($net->{bridge}) {
208ba94e 1987 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
cc4d6182 1988 } else {
a9410357 1989 $netdev = "type=user,id=$netid,hostname=$vmname";
cc4d6182 1990 }
a9410357
AD
1991
1992 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1993
1994 return $netdev;
cc4d6182 1995}
1e3baf05 1996
0efb537e
AD
1997
1998sub print_cpu_device {
1999 my ($conf, $id) = @_;
2000
74c02ef7
PA
2001 my $kvm = $conf->{kvm} // 1;
2002 my $cpu = $kvm ? "kvm64" : "qemu64";
0efb537e
AD
2003 if (my $cputype = $conf->{cpu}) {
2004 my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2005 or die "Cannot parse cpu description: $cputype\n";
2006 $cpu = $cpuconf->{cputype};
2007 }
2008
0efb537e
AD
2009 my $cores = $conf->{cores} || 1;
2010
2011 my $current_core = ($id - 1) % $cores;
7032e08c 2012 my $current_socket = int(($id - 1 - $current_core)/$cores);
0efb537e
AD
2013
2014 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2015}
2016
55655ebc
DC
2017my $vga_map = {
2018 'cirrus' => 'cirrus-vga',
2019 'std' => 'VGA',
2020 'vmware' => 'vmware-svga',
2021 'virtio' => 'virtio-vga',
2022};
2023
2024sub print_vga_device {
d559309f 2025 my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
55655ebc
DC
2026
2027 my $type = $vga_map->{$vga->{type}};
86c9fafe 2028 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
d559309f
WB
2029 $type = 'virtio-gpu';
2030 }
55655ebc
DC
2031 my $vgamem_mb = $vga->{memory};
2032 if ($qxlnum) {
2033 $type = $id ? 'qxl' : 'qxl-vga';
2034 }
2035 die "no devicetype for $vga->{type}\n" if !$type;
2036
2037 my $memory = "";
2038 if ($vgamem_mb) {
2039 if ($vga->{type} eq 'virtio') {
2040 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2041 $memory = ",max_hostmem=$bytes";
2042 } elsif ($qxlnum) {
2043 # from https://www.spice-space.org/multiple-monitors.html
2044 $memory = ",vgamem_mb=$vga->{memory}";
2045 my $ram = $vgamem_mb * 4;
2046 my $vram = $vgamem_mb * 2;
2047 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2048 } else {
2049 $memory = ",vgamem_mb=$vga->{memory}";
2050 }
2051 } elsif ($qxlnum && $id) {
2052 $memory = ",ram_size=67108864,vram_size=33554432";
2053 }
2054
2055 my $q35 = machine_type_is_q35($conf);
2056 my $vgaid = "vga" . ($id // '');
2057 my $pciaddr;
daadd5a4 2058
55655ebc 2059 if ($q35 && $vgaid eq 'vga') {
daadd5a4 2060 # the first display uses pcie.0 bus on q35 machines
d559309f 2061 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
55655ebc 2062 } else {
d559309f 2063 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
55655ebc
DC
2064 }
2065
2066 return "$type,id=${vgaid}${memory}${pciaddr}";
2067}
2068
9ed7a77c
WB
2069sub drive_is_cloudinit {
2070 my ($drive) = @_;
2071 return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2072}
2073
1e3baf05 2074sub drive_is_cdrom {
9c52f5ed
WB
2075 my ($drive, $exclude_cloudinit) = @_;
2076
9ed7a77c 2077 return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1e3baf05
DM
2078
2079 return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2080
2081}
2082
ffc0d8c7
WB
2083sub parse_number_sets {
2084 my ($set) = @_;
2085 my $res = [];
2086 foreach my $part (split(/;/, $set)) {
2087 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2088 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2089 push @$res, [ $1, $2 ];
2ed5d572 2090 } else {
ffc0d8c7 2091 die "invalid range: $part\n";
2ed5d572
AD
2092 }
2093 }
ffc0d8c7
WB
2094 return $res;
2095}
2ed5d572 2096
ffc0d8c7
WB
2097sub parse_numa {
2098 my ($data) = @_;
2099
2100 my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2101 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2102 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2ed5d572
AD
2103 return $res;
2104}
2105
040b06b7
DA
2106sub parse_hostpci {
2107 my ($value) = @_;
2108
2109 return undef if !$value;
2110
1f4f447b 2111 my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
0cea6a01 2112
1f4f447b
WB
2113 my @idlist = split(/;/, $res->{host});
2114 delete $res->{host};
2115 foreach my $id (@idlist) {
2fd24788
DC
2116 if ($id =~ m/\./) { # full id 00:00.1
2117 push @{$res->{pciid}}, {
2118 id => $id,
2119 };
2120 } else { # partial id 00:00
2121 $res->{pciid} = PVE::SysFSTools::lspci($id);
0cea6a01 2122 }
040b06b7 2123 }
040b06b7
DA
2124 return $res;
2125}
2126
1e3baf05
DM
2127# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2128sub parse_net {
2129 my ($data) = @_;
2130
cd9c34d1
WB
2131 my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2132 if ($@) {
2133 warn $@;
2134 return undef;
1e3baf05 2135 }
b5b99790
WB
2136 if (!defined($res->{macaddr})) {
2137 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2138 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2139 }
0c9a7596
AD
2140 return $res;
2141}
2142
2143# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2144sub parse_ipconfig {
2145 my ($data) = @_;
2146
2147 my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2148 if ($@) {
2149 warn $@;
2150 return undef;
2151 }
2152
2153 if ($res->{gw} && !$res->{ip}) {
2154 warn 'gateway specified without specifying an IP address';
2155 return undef;
2156 }
2157 if ($res->{gw6} && !$res->{ip6}) {
2158 warn 'IPv6 gateway specified without specifying an IPv6 address';
2159 return undef;
2160 }
2161 if ($res->{gw} && $res->{ip} eq 'dhcp') {
2162 warn 'gateway specified together with DHCP';
2163 return undef;
2164 }
2165 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2166 # gw6 + auto/dhcp
2167 warn "IPv6 gateway specified together with $res->{ip6} address";
2168 return undef;
2169 }
2170
2171 if (!$res->{ip} && !$res->{ip6}) {
2172 return { ip => 'dhcp', ip6 => 'dhcp' };
2173 }
2174
1e3baf05
DM
2175 return $res;
2176}
2177
2178sub print_net {
2179 my $net = shift;
2180
cd9c34d1 2181 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1e3baf05
DM
2182}
2183
2184sub add_random_macs {
2185 my ($settings) = @_;
2186
2187 foreach my $opt (keys %$settings) {
2188 next if $opt !~ m/^net(\d+)$/;
2189 my $net = parse_net($settings->{$opt});
2190 next if !$net;
2191 $settings->{$opt} = print_net($net);
2192 }
2193}
2194
055d554d
DM
2195sub vm_is_volid_owner {
2196 my ($storecfg, $vmid, $volid) = @_;
2197
2198 if ($volid !~ m|^/|) {
2199 my ($path, $owner);
2200 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2201 if ($owner && ($owner == $vmid)) {
2202 return 1;
2203 }
2204 }
2205
2206 return undef;
2207}
2208
3dc38fbb
WB
2209sub split_flagged_list {
2210 my $text = shift || '';
2211 $text =~ s/[,;]/ /g;
2212 $text =~ s/^\s+//;
2213 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2214}
2215
2216sub join_flagged_list {
2217 my ($how, $lst) = @_;
2218 join $how, map { $lst->{$_} . $_ } keys %$lst;
2219}
2220
055d554d 2221sub vmconfig_delete_pending_option {
3dc38fbb 2222 my ($conf, $key, $force) = @_;
055d554d
DM
2223
2224 delete $conf->{pending}->{$key};
3dc38fbb
WB
2225 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2226 $pending_delete_hash->{$key} = $force ? '!' : '';
2227 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
055d554d
DM
2228}
2229
2230sub vmconfig_undelete_pending_option {
2231 my ($conf, $key) = @_;
2232
3dc38fbb 2233 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
055d554d
DM
2234 delete $pending_delete_hash->{$key};
2235
3dc38fbb
WB
2236 if (%$pending_delete_hash) {
2237 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
055d554d
DM
2238 } else {
2239 delete $conf->{pending}->{delete};
2240 }
2241}
2242
2243sub vmconfig_register_unused_drive {
2244 my ($storecfg, $vmid, $conf, $drive) = @_;
2245
2d9ddec5
WB
2246 if (drive_is_cloudinit($drive)) {
2247 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2248 warn $@ if $@;
2249 } elsif (!drive_is_cdrom($drive)) {
055d554d
DM
2250 my $volid = $drive->{file};
2251 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
8793d495 2252 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
055d554d
DM
2253 }
2254 }
2255}
2256
c750e90a
DM
2257sub vmconfig_cleanup_pending {
2258 my ($conf) = @_;
2259
2260 # remove pending changes when nothing changed
2261 my $changes;
2262 foreach my $opt (keys %{$conf->{pending}}) {
2263 if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq $conf->{$opt})) {
2264 $changes = 1;
2265 delete $conf->{pending}->{$opt};
2266 }
2267 }
2268
3dc38fbb 2269 my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
c750e90a 2270 my $pending_delete_hash = {};
3dc38fbb 2271 while (my ($opt, $force) = each %$current_delete_hash) {
c750e90a 2272 if (defined($conf->{$opt})) {
3dc38fbb 2273 $pending_delete_hash->{$opt} = $force;
c750e90a
DM
2274 } else {
2275 $changes = 1;
2276 }
2277 }
2278
3dc38fbb
WB
2279 if (%$pending_delete_hash) {
2280 $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
c750e90a
DM
2281 } else {
2282 delete $conf->{pending}->{delete};
2283 }
2284
2285 return $changes;
2286}
2287
bd27e851 2288# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
ff6ffe20 2289my $smbios1_fmt = {
bd27e851
WB
2290 uuid => {
2291 type => 'string',
2292 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2293 format_description => 'UUID',
52261945 2294 description => "Set SMBIOS1 UUID.",
bd27e851
WB
2295 optional => 1,
2296 },
2297 version => {
2298 type => 'string',
2299 pattern => '\S+',
7f694a71 2300 format_description => 'string',
52261945 2301 description => "Set SMBIOS1 version.",
bd27e851
WB
2302 optional => 1,
2303 },
2304 serial => {
2305 type => 'string',
2306 pattern => '\S+',
7f694a71 2307 format_description => 'string',
52261945 2308 description => "Set SMBIOS1 serial number.",
bd27e851
WB
2309 optional => 1,
2310 },
2311 manufacturer => {
2312 type => 'string',
2313 pattern => '\S+',
7f694a71 2314 format_description => 'string',
52261945 2315 description => "Set SMBIOS1 manufacturer.",
bd27e851
WB
2316 optional => 1,
2317 },
2318 product => {
2319 type => 'string',
2320 pattern => '\S+',
7f694a71 2321 format_description => 'string',
52261945 2322 description => "Set SMBIOS1 product ID.",
bd27e851
WB
2323 optional => 1,
2324 },
2325 sku => {
2326 type => 'string',
2327 pattern => '\S+',
7f694a71 2328 format_description => 'string',
52261945 2329 description => "Set SMBIOS1 SKU string.",
bd27e851
WB
2330 optional => 1,
2331 },
2332 family => {
2333 type => 'string',
2334 pattern => '\S+',
7f694a71 2335 format_description => 'string',
52261945 2336 description => "Set SMBIOS1 family string.",
bd27e851
WB
2337 optional => 1,
2338 },
2796e7d5
DM
2339};
2340
2796e7d5
DM
2341sub parse_smbios1 {
2342 my ($data) = @_;
2343
ff6ffe20 2344 my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
bd27e851 2345 warn $@ if $@;
2796e7d5
DM
2346 return $res;
2347}
2348
cd11416f
DM
2349sub print_smbios1 {
2350 my ($smbios1) = @_;
ff6ffe20 2351 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
cd11416f
DM
2352}
2353
ff6ffe20 2354PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2796e7d5 2355
1e3baf05
DM
2356PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2357sub verify_bootdisk {
2358 my ($value, $noerr) = @_;
2359
74479ee9 2360 return $value if is_valid_drivename($value);
1e3baf05
DM
2361
2362 return undef if $noerr;
2363
2364 die "invalid boot disk '$value'\n";
2365}
2366
0ea9541d
DM
2367sub parse_watchdog {
2368 my ($value) = @_;
2369
2370 return undef if !$value;
2371
ec3582b5
WB
2372 my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2373 warn $@ if $@;
0ea9541d
DM
2374 return $res;
2375}
2376
9d66b397
SI
2377sub parse_guest_agent {
2378 my ($value) = @_;
2379
2380 return {} if !defined($value->{agent});
2381
2382 my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2383 warn $@ if $@;
2384
2385 # if the agent is disabled ignore the other potentially set properties
2386 return {} if !$res->{enabled};
2387 return $res;
2388}
2389
55655ebc
DC
2390sub parse_vga {
2391 my ($value) = @_;
2392
2393 return {} if !$value;
2394 my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2395 warn $@ if $@;
2396 return $res;
2397}
2398
1e3baf05
DM
2399PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2400sub verify_usb_device {
2401 my ($value, $noerr) = @_;
2402
2403 return $value if parse_usb_device($value);
2404
2405 return undef if $noerr;
19672434 2406
1e3baf05
DM
2407 die "unable to parse usb device\n";
2408}
2409
1e3baf05
DM
2410# add JSON properties for create and set function
2411sub json_config_properties {
2412 my $prop = shift;
2413
2414 foreach my $opt (keys %$confdesc) {
c6737ef1 2415 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1e3baf05
DM
2416 $prop->{$opt} = $confdesc->{$opt};
2417 }
2418
2419 return $prop;
2420}
2421
d41121fd
DM
2422# return copy of $confdesc_cloudinit to generate documentation
2423sub cloudinit_config_properties {
2424
2425 return dclone($confdesc_cloudinit);
2426}
2427
1e3baf05
DM
2428sub check_type {
2429 my ($key, $value) = @_;
2430
2431 die "unknown setting '$key'\n" if !$confdesc->{$key};
2432
2433 my $type = $confdesc->{$key}->{type};
2434
6b64503e 2435 if (!defined($value)) {
1e3baf05
DM
2436 die "got undefined value\n";
2437 }
2438
2439 if ($value =~ m/[\n\r]/) {
2440 die "property contains a line feed\n";
2441 }
2442
2443 if ($type eq 'boolean') {
19672434
DM
2444 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2445 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2446 die "type check ('boolean') failed - got '$value'\n";
1e3baf05
DM
2447 } elsif ($type eq 'integer') {
2448 return int($1) if $value =~ m/^(\d+)$/;
2449 die "type check ('integer') failed - got '$value'\n";
04432191
AD
2450 } elsif ($type eq 'number') {
2451 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2452 die "type check ('number') failed - got '$value'\n";
1e3baf05
DM
2453 } elsif ($type eq 'string') {
2454 if (my $fmt = $confdesc->{$key}->{format}) {
1e3baf05 2455 PVE::JSONSchema::check_format($fmt, $value);
19672434
DM
2456 return $value;
2457 }
1e3baf05 2458 $value =~ s/^\"(.*)\"$/$1/;
19672434 2459 return $value;
1e3baf05
DM
2460 } else {
2461 die "internal error"
2462 }
2463}
2464
1e3baf05
DM
2465sub touch_config {
2466 my ($vmid) = @_;
2467
ffda963f 2468 my $conf = PVE::QemuConfig->config_file($vmid);
1e3baf05
DM
2469 utime undef, undef, $conf;
2470}
2471
1e3baf05 2472sub destroy_vm {
15cc8784 2473 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1e3baf05 2474
ffda963f 2475 my $conffile = PVE::QemuConfig->config_file($vmid);
1e3baf05 2476
ffda963f 2477 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 2478
ffda963f 2479 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 2480
5e67a2d2
DC
2481 if ($conf->{template}) {
2482 # check if any base image is still used by a linked clone
2483 foreach_drive($conf, sub {
2484 my ($ds, $drive) = @_;
2485
2486 return if drive_is_cdrom($drive);
2487
2488 my $volid = $drive->{file};
2489
2490 return if !$volid || $volid =~ m|^/|;
2491
2492 die "base volume '$volid' is still in use by linked cloned\n"
2493 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2494
2495 });
2496 }
2497
19672434 2498 # only remove disks owned by this VM
1e3baf05
DM
2499 foreach_drive($conf, sub {
2500 my ($ds, $drive) = @_;
2501
9c52f5ed 2502 return if drive_is_cdrom($drive, 1);
1e3baf05
DM
2503
2504 my $volid = $drive->{file};
ed221350 2505
ff1a2432 2506 return if !$volid || $volid =~ m|^/|;
1e3baf05 2507
6b64503e 2508 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
ff1a2432 2509 return if !$path || !$owner || ($owner != $vmid);
1e3baf05 2510
31b52247
FG
2511 eval {
2512 PVE::Storage::vdisk_free($storecfg, $volid);
2513 };
2514 warn "Could not remove disk '$volid', check manually: $@" if $@;
2515
1e3baf05 2516 });
19672434 2517
a6af7b3e 2518 if ($keep_empty_config) {
9c502e26 2519 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
a6af7b3e
DM
2520 } else {
2521 unlink $conffile;
2522 }
1e3baf05
DM
2523
2524 # also remove unused disk
2525 eval {
6b64503e 2526 my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1e3baf05
DM
2527
2528 eval {
6b64503e 2529 PVE::Storage::foreach_volid($dl, sub {
1e3baf05 2530 my ($volid, $sid, $volname, $d) = @_;
6b64503e 2531 PVE::Storage::vdisk_free($storecfg, $volid);
1e3baf05
DM
2532 });
2533 };
2534 warn $@ if $@;
2535
2536 };
2537 warn $@ if $@;
2538}
2539
1e3baf05
DM
2540sub parse_vm_config {
2541 my ($filename, $raw) = @_;
2542
2543 return undef if !defined($raw);
2544
554ac7e7 2545 my $res = {
fc1ddcdc 2546 digest => Digest::SHA::sha1_hex($raw),
0d18dcfc 2547 snapshots => {},
0d732d16 2548 pending => {},
554ac7e7 2549 };
1e3baf05 2550
19672434 2551 $filename =~ m|/qemu-server/(\d+)\.conf$|
1e3baf05
DM
2552 || die "got strange filename '$filename'";
2553
2554 my $vmid = $1;
2555
0d18dcfc 2556 my $conf = $res;
b0ec896e 2557 my $descr;
e297c490 2558 my $section = '';
0581fe4f 2559
0d18dcfc
DM
2560 my @lines = split(/\n/, $raw);
2561 foreach my $line (@lines) {
1e3baf05 2562 next if $line =~ m/^\s*$/;
be190583 2563
eab09f4e 2564 if ($line =~ m/^\[PENDING\]\s*$/i) {
e297c490 2565 $section = 'pending';
b0ec896e
DM
2566 if (defined($descr)) {
2567 $descr =~ s/\s+$//;
2568 $conf->{description} = $descr;
2569 }
2570 $descr = undef;
e297c490 2571 $conf = $res->{$section} = {};
eab09f4e
AD
2572 next;
2573
0d732d16 2574 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
e297c490 2575 $section = $1;
b0ec896e
DM
2576 if (defined($descr)) {
2577 $descr =~ s/\s+$//;
2578 $conf->{description} = $descr;
2579 }
2580 $descr = undef;
e297c490 2581 $conf = $res->{snapshots}->{$section} = {};
0d18dcfc
DM
2582 next;
2583 }
1e3baf05 2584
0581fe4f 2585 if ($line =~ m/^\#(.*)\s*$/) {
b0ec896e 2586 $descr = '' if !defined($descr);
0581fe4f
DM
2587 $descr .= PVE::Tools::decode_text($1) . "\n";
2588 next;
2589 }
2590
1e3baf05 2591 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
b0ec896e 2592 $descr = '' if !defined($descr);
0581fe4f 2593 $descr .= PVE::Tools::decode_text($2);
0d18dcfc
DM
2594 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2595 $conf->{snapstate} = $1;
1e3baf05
DM
2596 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2597 my $key = $1;
2598 my $value = $2;
0d18dcfc 2599 $conf->{$key} = $value;
ef824322 2600 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
e297c490 2601 my $value = $1;
ef824322
DM
2602 if ($section eq 'pending') {
2603 $conf->{delete} = $value; # we parse this later
2604 } else {
2605 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
eab09f4e 2606 }
15cf7698 2607 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
1e3baf05
DM
2608 my $key = $1;
2609 my $value = $2;
2610 eval { $value = check_type($key, $value); };
2611 if ($@) {
2612 warn "vm $vmid - unable to parse value of '$key' - $@";
2613 } else {
b799312f 2614 $key = 'ide2' if $key eq 'cdrom';
1e3baf05 2615 my $fmt = $confdesc->{$key}->{format};
b799312f 2616 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
1e3baf05
DM
2617 my $v = parse_drive($key, $value);
2618 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2619 $v->{file} = $volid;
6b64503e 2620 $value = print_drive($vmid, $v);
1e3baf05
DM
2621 } else {
2622 warn "vm $vmid - unable to parse value of '$key'\n";
2623 next;
2624 }
2625 }
2626
b799312f 2627 $conf->{$key} = $value;
1e3baf05
DM
2628 }
2629 }
2630 }
2631
b0ec896e
DM
2632 if (defined($descr)) {
2633 $descr =~ s/\s+$//;
2634 $conf->{description} = $descr;
2635 }
0d18dcfc 2636 delete $res->{snapstate}; # just to be sure
1e3baf05
DM
2637
2638 return $res;
2639}
2640
1858638f
DM
2641sub write_vm_config {
2642 my ($filename, $conf) = @_;
1e3baf05 2643
0d18dcfc
DM
2644 delete $conf->{snapstate}; # just to be sure
2645
1858638f
DM
2646 if ($conf->{cdrom}) {
2647 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2648 $conf->{ide2} = $conf->{cdrom};
2649 delete $conf->{cdrom};
2650 }
1e3baf05
DM
2651
2652 # we do not use 'smp' any longer
1858638f
DM
2653 if ($conf->{sockets}) {
2654 delete $conf->{smp};
2655 } elsif ($conf->{smp}) {
2656 $conf->{sockets} = $conf->{smp};
2657 delete $conf->{cores};
2658 delete $conf->{smp};
1e3baf05
DM
2659 }
2660
ee2f90b1 2661 my $used_volids = {};
0d18dcfc 2662
ee2f90b1 2663 my $cleanup_config = sub {
ef824322 2664 my ($cref, $pending, $snapname) = @_;
1858638f 2665
ee2f90b1
DM
2666 foreach my $key (keys %$cref) {
2667 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
ef824322 2668 $key eq 'snapstate' || $key eq 'pending';
ee2f90b1 2669 my $value = $cref->{$key};
ef824322
DM
2670 if ($key eq 'delete') {
2671 die "propertry 'delete' is only allowed in [PENDING]\n"
2672 if !$pending;
2673 # fixme: check syntax?
2674 next;
2675 }
ee2f90b1
DM
2676 eval { $value = check_type($key, $value); };
2677 die "unable to parse value of '$key' - $@" if $@;
1858638f 2678
ee2f90b1
DM
2679 $cref->{$key} = $value;
2680
74479ee9 2681 if (!$snapname && is_valid_drivename($key)) {
ed221350 2682 my $drive = parse_drive($key, $value);
ee2f90b1
DM
2683 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2684 }
1e3baf05 2685 }
ee2f90b1
DM
2686 };
2687
2688 &$cleanup_config($conf);
ef824322
DM
2689
2690 &$cleanup_config($conf->{pending}, 1);
2691
ee2f90b1 2692 foreach my $snapname (keys %{$conf->{snapshots}}) {
ef824322
DM
2693 die "internal error" if $snapname eq 'pending';
2694 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
1e3baf05
DM
2695 }
2696
1858638f
DM
2697 # remove 'unusedX' settings if we re-add a volume
2698 foreach my $key (keys %$conf) {
2699 my $value = $conf->{$key};
ee2f90b1 2700 if ($key =~ m/^unused/ && $used_volids->{$value}) {
1858638f 2701 delete $conf->{$key};
1e3baf05 2702 }
1858638f 2703 }
be190583 2704
0d18dcfc 2705 my $generate_raw_config = sub {
b0ec896e 2706 my ($conf, $pending) = @_;
0581fe4f 2707
0d18dcfc
DM
2708 my $raw = '';
2709
2710 # add description as comment to top of file
b0ec896e
DM
2711 if (defined(my $descr = $conf->{description})) {
2712 if ($descr) {
2713 foreach my $cl (split(/\n/, $descr)) {
2714 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2715 }
2716 } else {
2717 $raw .= "#\n" if $pending;
2718 }
0d18dcfc
DM
2719 }
2720
2721 foreach my $key (sort keys %$conf) {
ef824322 2722 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
0d18dcfc
DM
2723 $raw .= "$key: $conf->{$key}\n";
2724 }
2725 return $raw;
2726 };
0581fe4f 2727
0d18dcfc 2728 my $raw = &$generate_raw_config($conf);
ef824322
DM
2729
2730 if (scalar(keys %{$conf->{pending}})){
2731 $raw .= "\n[PENDING]\n";
b0ec896e 2732 $raw .= &$generate_raw_config($conf->{pending}, 1);
ef824322
DM
2733 }
2734
0d18dcfc
DM
2735 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2736 $raw .= "\n[$snapname]\n";
2737 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1858638f 2738 }
1e3baf05 2739
1858638f
DM
2740 return $raw;
2741}
1e3baf05 2742
19672434 2743sub load_defaults {
1e3baf05
DM
2744
2745 my $res = {};
2746
2747 # we use static defaults from our JSON schema configuration
2748 foreach my $key (keys %$confdesc) {
2749 if (defined(my $default = $confdesc->{$key}->{default})) {
2750 $res->{$key} = $default;
2751 }
2752 }
19672434 2753
1e3baf05
DM
2754 return $res;
2755}
2756
2757sub config_list {
2758 my $vmlist = PVE::Cluster::get_vmlist();
2759 my $res = {};
2760 return $res if !$vmlist || !$vmlist->{ids};
2761 my $ids = $vmlist->{ids};
2762
1e3baf05
DM
2763 foreach my $vmid (keys %$ids) {
2764 my $d = $ids->{$vmid};
2765 next if !$d->{node} || $d->{node} ne $nodename;
5ee957cc 2766 next if !$d->{type} || $d->{type} ne 'qemu';
1e3baf05
DM
2767 $res->{$vmid}->{exists} = 1;
2768 }
2769 return $res;
2770}
2771
64e13401
DM
2772# test if VM uses local resources (to prevent migration)
2773sub check_local_resources {
2774 my ($conf, $noerr) = @_;
2775
2776 my $loc_res = 0;
19672434 2777
e0ab7331
DM
2778 $loc_res = 1 if $conf->{hostusb}; # old syntax
2779 $loc_res = 1 if $conf->{hostpci}; # old syntax
64e13401 2780
0d29ab3b 2781 foreach my $k (keys %$conf) {
49ca581d 2782 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
d44712fc
EK
2783 # sockets are safe: they will recreated be on the target side post-migrate
2784 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2fe1a152 2785 $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
64e13401
DM
2786 }
2787
2788 die "VM uses local resources\n" if $loc_res && !$noerr;
2789
2790 return $loc_res;
2791}
2792
719893a9 2793# check if used storages are available on all nodes (use by migrate)
47152e2e
DM
2794sub check_storage_availability {
2795 my ($storecfg, $conf, $node) = @_;
2796
2797 foreach_drive($conf, sub {
2798 my ($ds, $drive) = @_;
2799
2800 my $volid = $drive->{file};
2801 return if !$volid;
2802
2803 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2804 return if !$sid;
2805
2806 # check if storage is available on both nodes
2807 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2808 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2809 });
2810}
2811
719893a9
DM
2812# list nodes where all VM images are available (used by has_feature API)
2813sub shared_nodes {
2814 my ($conf, $storecfg) = @_;
2815
2816 my $nodelist = PVE::Cluster::get_nodelist();
2817 my $nodehash = { map { $_ => 1 } @$nodelist };
2818 my $nodename = PVE::INotify::nodename();
be190583 2819
719893a9
DM
2820 foreach_drive($conf, sub {
2821 my ($ds, $drive) = @_;
2822
2823 my $volid = $drive->{file};
2824 return if !$volid;
2825
2826 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2827 if ($storeid) {
2828 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2829 if ($scfg->{disable}) {
2830 $nodehash = {};
2831 } elsif (my $avail = $scfg->{nodes}) {
2832 foreach my $node (keys %$nodehash) {
2833 delete $nodehash->{$node} if !$avail->{$node};
2834 }
2835 } elsif (!$scfg->{shared}) {
2836 foreach my $node (keys %$nodehash) {
2837 delete $nodehash->{$node} if $node ne $nodename
2838 }
2839 }
2840 }
2841 });
2842
2843 return $nodehash
2844}
2845
1e3baf05
DM
2846sub check_cmdline {
2847 my ($pidfile, $pid) = @_;
2848
6b64503e
DM
2849 my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2850 if (defined($fh)) {
1e3baf05
DM
2851 my $line = <$fh>;
2852 $fh->close;
2853 return undef if !$line;
6b64503e 2854 my @param = split(/\0/, $line);
1e3baf05
DM
2855
2856 my $cmd = $param[0];
6908fd9b 2857 return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
1e3baf05
DM
2858
2859 for (my $i = 0; $i < scalar (@param); $i++) {
2860 my $p = $param[$i];
2861 next if !$p;
2862 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2863 my $p = $param[$i+1];
2864 return 1 if $p && ($p eq $pidfile);
2865 return undef;
2866 }
2867 }
2868 }
2869 return undef;
2870}
2871
2872sub check_running {
7e8dcf2c 2873 my ($vmid, $nocheck, $node) = @_;
1e3baf05 2874
ffda963f 2875 my $filename = PVE::QemuConfig->config_file($vmid, $node);
1e3baf05
DM
2876
2877 die "unable to find configuration file for VM $vmid - no such machine\n"
e6c3b671 2878 if !$nocheck && ! -f $filename;
1e3baf05 2879
e6c3b671 2880 my $pidfile = pidfile_name($vmid);
1e3baf05 2881
e6c3b671
DM
2882 if (my $fd = IO::File->new("<$pidfile")) {
2883 my $st = stat($fd);
1e3baf05 2884 my $line = <$fd>;
6b64503e 2885 close($fd);
1e3baf05
DM
2886
2887 my $mtime = $st->mtime;
2888 if ($mtime > time()) {
2889 warn "file '$filename' modified in future\n";
2890 }
2891
2892 if ($line =~ m/^(\d+)$/) {
2893 my $pid = $1;
e6c3b671
DM
2894 if (check_cmdline($pidfile, $pid)) {
2895 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2896 return $pid;
2897 }
2898 }
1e3baf05
DM
2899 }
2900 }
2901
2902 return undef;
2903}
2904
2905sub vzlist {
19672434 2906
1e3baf05
DM
2907 my $vzlist = config_list();
2908
6b64503e 2909 my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
1e3baf05 2910
19672434 2911 while (defined(my $de = $fd->read)) {
1e3baf05
DM
2912 next if $de !~ m/^(\d+)\.pid$/;
2913 my $vmid = $1;
6b64503e
DM
2914 next if !defined($vzlist->{$vmid});
2915 if (my $pid = check_running($vmid)) {
1e3baf05
DM
2916 $vzlist->{$vmid}->{pid} = $pid;
2917 }
2918 }
2919
2920 return $vzlist;
2921}
2922
1e3baf05
DM
2923sub disksize {
2924 my ($storecfg, $conf) = @_;
2925
2926 my $bootdisk = $conf->{bootdisk};
2927 return undef if !$bootdisk;
74479ee9 2928 return undef if !is_valid_drivename($bootdisk);
1e3baf05
DM
2929
2930 return undef if !$conf->{$bootdisk};
2931
2932 my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2933 return undef if !defined($drive);
2934
2935 return undef if drive_is_cdrom($drive);
2936
2937 my $volid = $drive->{file};
2938 return undef if !$volid;
2939
24afaca0 2940 return $drive->{size};
1e3baf05
DM
2941}
2942
b1a70cab
DM
2943our $vmstatus_return_properties = {
2944 vmid => get_standard_option('pve-vmid'),
2945 status => {
2946 description => "Qemu process status.",
2947 type => 'string',
2948 enum => ['stopped', 'running'],
2949 },
2950 maxmem => {
2951 description => "Maximum memory in bytes.",
2952 type => 'integer',
2953 optional => 1,
2954 renderer => 'bytes',
2955 },
2956 maxdisk => {
2957 description => "Root disk size in bytes.",
2958 type => 'integer',
2959 optional => 1,
2960 renderer => 'bytes',
2961 },
2962 name => {
2963 description => "VM name.",
2964 type => 'string',
2965 optional => 1,
2966 },
2967 qmpstatus => {
2968 description => "Qemu QMP agent status.",
2969 type => 'string',
2970 optional => 1,
2971 },
2972 pid => {
2973 description => "PID of running qemu process.",
2974 type => 'integer',
2975 optional => 1,
2976 },
2977 uptime => {
2978 description => "Uptime.",
2979 type => 'integer',
2980 optional => 1,
2981 renderer => 'duration',
2982 },
2983 cpus => {
2984 description => "Maximum usable CPUs.",
2985 type => 'number',
2986 optional => 1,
2987 },
2988};
2989
1e3baf05
DM
2990my $last_proc_pid_stat;
2991
03a33f30
DM
2992# get VM status information
2993# This must be fast and should not block ($full == false)
2994# We only query KVM using QMP if $full == true (this can be slow)
1e3baf05 2995sub vmstatus {
03a33f30 2996 my ($opt_vmid, $full) = @_;
1e3baf05
DM
2997
2998 my $res = {};
2999
19672434 3000 my $storecfg = PVE::Storage::config();
1e3baf05
DM
3001
3002 my $list = vzlist();
3618ee99
EK
3003 my $defaults = load_defaults();
3004
694fcad4 3005 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
1e3baf05 3006
ae4915a2
DM
3007 my $cpucount = $cpuinfo->{cpus} || 1;
3008
1e3baf05
DM
3009 foreach my $vmid (keys %$list) {
3010 next if $opt_vmid && ($vmid ne $opt_vmid);
3011
ffda963f 3012 my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
1e3baf05
DM
3013 my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3014
b1a70cab 3015 my $d = { vmid => $vmid };
1e3baf05
DM
3016 $d->{pid} = $list->{$vmid}->{pid};
3017
3018 # fixme: better status?
3019 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3020
af990afe
DM
3021 my $size = disksize($storecfg, $conf);
3022 if (defined($size)) {
3023 $d->{disk} = 0; # no info available
1e3baf05
DM
3024 $d->{maxdisk} = $size;
3025 } else {
3026 $d->{disk} = 0;
3027 $d->{maxdisk} = 0;
3028 }
3029
3618ee99
EK
3030 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3031 * ($conf->{cores} || $defaults->{cores});
ae4915a2 3032 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
d7c8364b 3033 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
ae4915a2 3034
1e3baf05 3035 $d->{name} = $conf->{name} || "VM $vmid";
3618ee99
EK
3036 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3037 : $defaults->{memory}*(1024*1024);
1e3baf05 3038
8b1accf7 3039 if ($conf->{balloon}) {
4bdb0514 3040 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3618ee99
EK
3041 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3042 : $defaults->{shares};
8b1accf7
DM
3043 }
3044
1e3baf05
DM
3045 $d->{uptime} = 0;
3046 $d->{cpu} = 0;
1e3baf05
DM
3047 $d->{mem} = 0;
3048
3049 $d->{netout} = 0;
3050 $d->{netin} = 0;
3051
3052 $d->{diskread} = 0;
3053 $d->{diskwrite} = 0;
3054
ffda963f 3055 $d->{template} = PVE::QemuConfig->is_template($conf);
4d8c851b 3056
8107b378
DC
3057 $d->{serial} = 1 if conf_has_serial($conf);
3058
1e3baf05
DM
3059 $res->{$vmid} = $d;
3060 }
3061
3062 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3063 foreach my $dev (keys %$netdev) {
3064 next if $dev !~ m/^tap([1-9]\d*)i/;
3065 my $vmid = $1;
3066 my $d = $res->{$vmid};
3067 next if !$d;
19672434 3068
1e3baf05
DM
3069 $d->{netout} += $netdev->{$dev}->{receive};
3070 $d->{netin} += $netdev->{$dev}->{transmit};
604ea644
AD
3071
3072 if ($full) {
3073 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3074 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3075 }
3076
1e3baf05
DM
3077 }
3078
1e3baf05
DM
3079 my $ctime = gettimeofday;
3080
3081 foreach my $vmid (keys %$list) {
3082
3083 my $d = $res->{$vmid};
3084 my $pid = $d->{pid};
3085 next if !$pid;
3086
694fcad4
DM
3087 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3088 next if !$pstat; # not running
19672434 3089
694fcad4 3090 my $used = $pstat->{utime} + $pstat->{stime};
1e3baf05 3091
694fcad4 3092 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
1e3baf05 3093
694fcad4 3094 if ($pstat->{vsize}) {
6b64503e 3095 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
1e3baf05
DM
3096 }
3097
3098 my $old = $last_proc_pid_stat->{$pid};
3099 if (!$old) {
19672434
DM
3100 $last_proc_pid_stat->{$pid} = {
3101 time => $ctime,
1e3baf05
DM
3102 used => $used,
3103 cpu => 0,
1e3baf05
DM
3104 };
3105 next;
3106 }
3107
7f0b5beb 3108 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
1e3baf05
DM
3109
3110 if ($dtime > 1000) {
3111 my $dutime = $used - $old->{used};
3112
ae4915a2 3113 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
1e3baf05 3114 $last_proc_pid_stat->{$pid} = {
19672434 3115 time => $ctime,
1e3baf05
DM
3116 used => $used,
3117 cpu => $d->{cpu},
1e3baf05
DM
3118 };
3119 } else {
3120 $d->{cpu} = $old->{cpu};
1e3baf05
DM
3121 }
3122 }
3123
f5eb281a 3124 return $res if !$full;
03a33f30
DM
3125
3126 my $qmpclient = PVE::QMPClient->new();
3127
64e7fcf2
DM
3128 my $ballooncb = sub {
3129 my ($vmid, $resp) = @_;
3130
3131 my $info = $resp->{'return'};
38babf81
DM
3132 return if !$info->{max_mem};
3133
64e7fcf2
DM
3134 my $d = $res->{$vmid};
3135
38babf81
DM
3136 # use memory assigned to VM
3137 $d->{maxmem} = $info->{max_mem};
3138 $d->{balloon} = $info->{actual};
3139
3140 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3141 $d->{mem} = $info->{total_mem} - $info->{free_mem};
3142 $d->{freemem} = $info->{free_mem};
64e7fcf2
DM
3143 }
3144
604ea644 3145 $d->{ballooninfo} = $info;
64e7fcf2
DM
3146 };
3147
03a33f30
DM
3148 my $blockstatscb = sub {
3149 my ($vmid, $resp) = @_;
3150 my $data = $resp->{'return'} || [];
3151 my $totalrdbytes = 0;
3152 my $totalwrbytes = 0;
604ea644 3153
03a33f30
DM
3154 for my $blockstat (@$data) {
3155 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3156 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
604ea644
AD
3157
3158 $blockstat->{device} =~ s/drive-//;
3159 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
03a33f30
DM
3160 }
3161 $res->{$vmid}->{diskread} = $totalrdbytes;
3162 $res->{$vmid}->{diskwrite} = $totalwrbytes;
3163 };
3164
3165 my $statuscb = sub {
3166 my ($vmid, $resp) = @_;
64e7fcf2 3167
03a33f30 3168 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
64e7fcf2
DM
3169 # this fails if ballon driver is not loaded, so this must be
3170 # the last commnand (following command are aborted if this fails).
38babf81 3171 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
03a33f30
DM
3172
3173 my $status = 'unknown';
3174 if (!defined($status = $resp->{'return'}->{status})) {
3175 warn "unable to get VM status\n";
3176 return;
3177 }
3178
3179 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3180 };
3181
3182 foreach my $vmid (keys %$list) {
3183 next if $opt_vmid && ($vmid ne $opt_vmid);
3184 next if !$res->{$vmid}->{pid}; # not running
3185 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3186 }
3187
b017fbda 3188 $qmpclient->queue_execute(undef, 2);
03a33f30
DM
3189
3190 foreach my $vmid (keys %$list) {
3191 next if $opt_vmid && ($vmid ne $opt_vmid);
3192 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3193 }
3194
1e3baf05
DM
3195 return $res;
3196}
3197
3198sub foreach_drive {
b74ff047 3199 my ($conf, $func, @param) = @_;
1e3baf05 3200
74479ee9
FG
3201 foreach my $ds (valid_drive_names()) {
3202 next if !defined($conf->{$ds});
1e3baf05 3203
6b64503e 3204 my $drive = parse_drive($ds, $conf->{$ds});
1e3baf05
DM
3205 next if !$drive;
3206
b74ff047 3207 &$func($ds, $drive, @param);
1e3baf05
DM
3208 }
3209}
3210
d5769dc2 3211sub foreach_volid {
b6adff33 3212 my ($conf, $func, @param) = @_;
be190583 3213
d5769dc2
DM
3214 my $volhash = {};
3215
3216 my $test_volid = sub {
ec82e3ee 3217 my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
d5769dc2
DM
3218
3219 return if !$volid;
be190583 3220
392f8b5d
DM
3221 $volhash->{$volid}->{cdrom} //= 1;
3222 $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3223
3224 $volhash->{$volid}->{replicate} //= 0;
3225 $volhash->{$volid}->{replicate} = 1 if $replicate;
39019f75 3226
ec82e3ee
CH
3227 $volhash->{$volid}->{shared} //= 0;
3228 $volhash->{$volid}->{shared} = 1 if $shared;
3229
39019f75
DM
3230 $volhash->{$volid}->{referenced_in_config} //= 0;
3231 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3232
3233 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3234 if defined($snapname);
d5769dc2
DM
3235 };
3236
ed221350 3237 foreach_drive($conf, sub {
d5769dc2 3238 my ($ds, $drive) = @_;
ec82e3ee 3239 $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
d5769dc2
DM
3240 });
3241
3242 foreach my $snapname (keys %{$conf->{snapshots}}) {
3243 my $snap = $conf->{snapshots}->{$snapname};
39019f75 3244 $test_volid->($snap->{vmstate}, 0, 1, $snapname);
ed221350 3245 foreach_drive($snap, sub {
d5769dc2 3246 my ($ds, $drive) = @_;
ec82e3ee 3247 $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
d5769dc2
DM
3248 });
3249 }
3250
3251 foreach my $volid (keys %$volhash) {
b6adff33 3252 &$func($volid, $volhash->{$volid}, @param);
d5769dc2
DM
3253 }
3254}
3255
8107b378
DC
3256sub conf_has_serial {
3257 my ($conf) = @_;
3258
3259 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3260 if ($conf->{"serial$i"}) {
3261 return 1;
3262 }
3263 }
3264
3265 return 0;
3266}
3267
86b8228b
DM
3268sub vga_conf_has_spice {
3269 my ($vga) = @_;
3270
55655ebc
DC
3271 my $vgaconf = parse_vga($vga);
3272 my $vgatype = $vgaconf->{type};
3273 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
590e698c
DM
3274
3275 return $1 || 1;
86b8228b
DM
3276}
3277
d731ecbe
WB
3278my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3279sub get_host_arch() {
3280 $host_arch = (POSIX::uname())[4] if !$host_arch;
3281 return $host_arch;
3282}
3283
3284sub is_native($) {
3285 my ($arch) = @_;
3286 return get_host_arch() eq $arch;
3287}
3288
3289my $default_machines = {
3290 x86_64 => 'pc',
3291 aarch64 => 'virt',
3292};
3293
3294sub get_basic_machine_info {
3295 my ($conf, $forcemachine) = @_;
3296
3297 my $arch = $conf->{arch} // get_host_arch();
3298 my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3299 return ($arch, $machine);
3300}
3301
96ed3574
WB
3302sub get_ovmf_files($) {
3303 my ($arch) = @_;
3304
3305 my $ovmf = $OVMF->{$arch}
3306 or die "no OVMF images known for architecture '$arch'\n";
3307
3308 return @$ovmf;
3309}
3310
6908fd9b
WB
3311my $Arch2Qemu = {
3312 aarch64 => '/usr/bin/qemu-system-aarch64',
3313 x86_64 => '/usr/bin/qemu-system-x86_64',
3314};
3315sub get_command_for_arch($) {
3316 my ($arch) = @_;
3317 return '/usr/bin/kvm' if is_native($arch);
3318
3319 my $cmd = $Arch2Qemu->{$arch}
3320 or die "don't know how to emulate architecture '$arch'\n";
3321 return $cmd;
3322}
3323
4fc262bd
WB
3324sub get_cpu_options {
3325 my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3326
3327 my $cpuFlags = [];
3328 my $ostype = $conf->{ostype};
3329
3330 my $cpu = $kvm ? "kvm64" : "qemu64";
0f27a91d
WB
3331 if ($arch eq 'aarch64') {
3332 $cpu = 'cortex-a57';
3333 }
4fc262bd
WB
3334 if (my $cputype = $conf->{cpu}) {
3335 my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3336 or die "Cannot parse cpu description: $cputype\n";
3337 $cpu = $cpuconf->{cputype};
3338 $kvm_off = 1 if $cpuconf->{hidden};
3339
3340 if (defined(my $flags = $cpuconf->{flags})) {
3341 push @$cpuFlags, split(";", $flags);
3342 }
3343 }
3344
1ea63c15 3345 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
4fc262bd
WB
3346
3347 push @$cpuFlags , '-x2apic'
3348 if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3349
3350 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3351
3352 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3353
1ea63c15 3354 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
4fc262bd
WB
3355
3356 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3357 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3358 }
3359
3360 add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3361
1ea63c15 3362 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
4fc262bd
WB
3363
3364 push @$cpuFlags, 'kvm=off' if $kvm_off;
3365
3366 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3367 push @$cpuFlags, "vendor=${cpu_vendor}"
3368 if $cpu_vendor ne 'default';
3369 } elsif ($arch ne 'aarch64') {
3370 die "internal error"; # should not happen
3371 }
3372
3373 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3374
3375 return ('-cpu', $cpu);
3376}
3377
1e3baf05 3378sub config_to_command {
67812f9c 3379 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
1e3baf05
DM
3380
3381 my $cmd = [];
8c559505
DM
3382 my $globalFlags = [];
3383 my $machineFlags = [];
3384 my $rtcFlags = [];
5bdcf937 3385 my $devices = [];
b78ebef7 3386 my $pciaddr = '';
5bdcf937 3387 my $bridges = {};
1e3baf05
DM
3388 my $kvmver = kvm_user_version();
3389 my $vernum = 0; # unknown
b42d3cf9 3390 my $ostype = $conf->{ostype};
4317f69f 3391 my $winversion = windows_version($ostype);
d731ecbe
WB
3392 my $kvm = $conf->{kvm};
3393
3394 my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3395 $kvm //= 1 if is_native($arch);
4317f69f 3396
d731ecbe
WB
3397 if ($kvm) {
3398 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3399 if !defined kvm_version();
3400 }
bfcd9b7e 3401
a3c52213
DM
3402 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3403 $vernum = $1*1000000+$2*1000;
3404 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
1e3baf05
DM
3405 $vernum = $1*1000000+$2*1000+$3;
3406 }
3407
a3c52213 3408 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
1e3baf05
DM
3409
3410 my $have_ovz = -f '/proc/vz/vestat';
3411
db656e5f 3412 my $q35 = machine_type_is_q35($conf);
4d3f29ed 3413 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
249c4a6c
AD
3414 my $use_old_bios_files = undef;
3415 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
db656e5f 3416
f08e17c7
AD
3417 my $cpuunits = defined($conf->{cpuunits}) ?
3418 $conf->{cpuunits} : $defaults->{cpuunits};
3419
6908fd9b 3420 push @$cmd, get_command_for_arch($arch);
1e3baf05
DM
3421
3422 push @$cmd, '-id', $vmid;
3423
e4d4cda1
HR
3424 my $vmname = $conf->{name} || "vm$vmid";
3425
3426 push @$cmd, '-name', $vmname;
3427
1e3baf05
DM
3428 my $use_virtio = 0;
3429
c971c4f2
AD
3430 my $qmpsocket = qmp_socket($vmid);
3431 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3432 push @$cmd, '-mon', "chardev=qmp,mode=control";
3433
71bd73b5 3434 if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
b4496b9e 3435 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
71bd73b5
DC
3436 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3437 }
1e3baf05 3438
6b64503e 3439 push @$cmd, '-pidfile' , pidfile_name($vmid);
19672434 3440
1e3baf05
DM
3441 push @$cmd, '-daemonize';
3442
2796e7d5
DM
3443 if ($conf->{smbios1}) {
3444 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3445 }
3446
6ee499ff
DC
3447 if ($conf->{vmgenid}) {
3448 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3449 }
3450
96ed3574 3451 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3edb45e7 3452 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
96ed3574 3453 die "uefi base image not found\n" if ! -f $ovmf_code;
2ddc0a5c 3454
4dcce9ee 3455 my $path;
13bca7b4 3456 my $format;
4dcce9ee
TL
3457 if (my $efidisk = $conf->{efidisk0}) {
3458 my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2ddc0a5c 3459 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
13bca7b4 3460 $format = $d->{format};
2ddc0a5c
DC
3461 if ($storeid) {
3462 $path = PVE::Storage::path($storecfg, $d->{file});
13bca7b4
WB
3463 if (!defined($format)) {
3464 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3465 $format = qemu_img_format($scfg, $volname);
3466 }
2ddc0a5c
DC
3467 } else {
3468 $path = $d->{file};
13bca7b4
WB
3469 die "efidisk format must be specified\n"
3470 if !defined($format);
2ddc0a5c 3471 }
2ddc0a5c 3472 } else {
4dcce9ee
TL
3473 warn "no efidisk configured! Using temporary efivars disk.\n";
3474 $path = "/tmp/$vmid-ovmf.fd";
96ed3574 3475 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
13bca7b4 3476 $format = 'raw';
2ddc0a5c 3477 }
4dcce9ee 3478
96ed3574 3479 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
2bfbee03 3480 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
a783c78e
AD
3481 }
3482
da8b4189 3483
d40e5e18 3484 # add usb controllers
d559309f 3485 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
d40e5e18 3486 push @$devices, @usbcontrollers if @usbcontrollers;
55655ebc 3487 my $vga = parse_vga($conf->{vga});
2fa3151e 3488
55655ebc
DC
3489 my $qxlnum = vga_conf_has_spice($conf->{vga});
3490 $vga->{type} = 'qxl' if $qxlnum;
2fa3151e 3491
55655ebc 3492 if (!$vga->{type}) {
869ad4a7
WB
3493 if ($arch eq 'aarch64') {
3494 $vga->{type} = 'virtio';
3495 } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
55655ebc 3496 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
a2a5cd64 3497 } else {
55655ebc 3498 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
a2a5cd64 3499 }
5acbfe9e
DM
3500 }
3501
1e3baf05 3502 # enable absolute mouse coordinates (needed by vnc)
5acbfe9e
DM
3503 my $tablet;
3504 if (defined($conf->{tablet})) {
3505 $tablet = $conf->{tablet};
3506 } else {
3507 $tablet = $defaults->{tablet};
590e698c 3508 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
55655ebc 3509 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
5acbfe9e
DM
3510 }
3511
d559309f
WB
3512 if ($tablet) {
3513 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3514 my $kbd = print_keyboarddevice_full($conf, $arch);
3515 push @$devices, '-device', $kbd if defined($kbd);
3516 }
b467f79a 3517
16a91d65 3518 my $kvm_off = 0;
4317f69f
AD
3519 my $gpu_passthrough;
3520
1e3baf05 3521 # host pci devices
040b06b7 3522 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
2e3b7e2a
AD
3523 my $d = parse_hostpci($conf->{"hostpci$i"});
3524 next if !$d;
3525
3526 my $pcie = $d->{pcie};
3527 if($pcie){
3528 die "q35 machine model is not enabled" if !$q35;
739ba340
DC
3529 # win7 wants to have the pcie devices directly on the pcie bus
3530 # instead of in the root port
3531 if ($winversion == 7) {
3532 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3533 } else {
3534 $pciaddr = print_pcie_addr("hostpci$i");
3535 }
2e3b7e2a 3536 }else{
d559309f 3537 $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
2e3b7e2a
AD
3538 }
3539
1f4f447b 3540 my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
9009150b 3541 my $romfile = $d->{romfile};
456a6fec 3542
1f4f447b
WB
3543 my $xvga = '';
3544 if ($d->{'x-vga'}) {
3545 $xvga = ',x-vga=on';
16a91d65 3546 $kvm_off = 1;
55655ebc 3547 $vga->{type} = 'none';
4317f69f
AD
3548 $gpu_passthrough = 1;
3549
230a4382
AD
3550 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3551 $xvga = "";
3552 }
137483c0 3553 }
4543ecf0
AD
3554 my $pcidevices = $d->{pciid};
3555 my $multifunction = 1 if @$pcidevices > 1;
6ab45bd7
DC
3556 my $sysfspath;
3557 if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3558 my $id = $pcidevices->[0]->{id};
6ab45bd7 3559 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
2fd24788 3560 $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
6ab45bd7
DC
3561 } elsif ($d->{mdev}) {
3562 warn "ignoring mediated device with multifunction device\n";
3563 }
2e3b7e2a 3564
4543ecf0
AD
3565 my $j=0;
3566 foreach my $pcidevice (@$pcidevices) {
2e3b7e2a 3567
4543ecf0
AD
3568 my $id = "hostpci$i";
3569 $id .= ".$j" if $multifunction;
3570 my $addr = $pciaddr;
3571 $addr .= ".$j" if $multifunction;
6ab45bd7
DC
3572 my $devicestr = "vfio-pci";
3573 if ($sysfspath) {
3574 $devicestr .= ",sysfsdev=$sysfspath";
3575 } else {
2fd24788 3576 $devicestr .= ",host=$pcidevice->{id}";
6ab45bd7
DC
3577 }
3578 $devicestr .= ",id=$id$addr";
4543ecf0
AD
3579
3580 if($j == 0){
3581 $devicestr .= "$rombar$xvga";
3582 $devicestr .= ",multifunction=on" if $multifunction;
456a6fec 3583 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
4543ecf0
AD
3584 }
3585
3586 push @$devices, '-device', $devicestr;
3587 $j++;
3588 }
1e3baf05
DM
3589 }
3590
3591 # usb devices
d40e5e18
DC
3592 my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3593 push @$devices, @usbdevices if @usbdevices;
1e3baf05 3594 # serial devices
bae179aa 3595 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
34978be3 3596 if (my $path = $conf->{"serial$i"}) {
9f9d2fb2
DM
3597 if ($path eq 'socket') {
3598 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3599 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
91b01bbb
WB
3600 # On aarch64, serial0 is the UART device. Qemu only allows
3601 # connecting UART devices via the '-serial' command line, as
3602 # the device has a fixed slot on the hardware...
3603 if ($arch eq 'aarch64' && $i == 0) {
3604 push @$devices, '-serial', "chardev:serial$i";
3605 } else {
3606 push @$devices, '-device', "isa-serial,chardev=serial$i";
3607 }
9f9d2fb2
DM
3608 } else {
3609 die "no such serial device\n" if ! -c $path;
3610 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3611 push @$devices, '-device', "isa-serial,chardev=serial$i";
3612 }
34978be3 3613 }
1e3baf05
DM
3614 }
3615
3616 # parallel devices
1989a89c 3617 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
34978be3 3618 if (my $path = $conf->{"parallel$i"}) {
19672434 3619 die "no such parallel device\n" if ! -c $path;
32e69805 3620 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
4c5dbaf6 3621 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
5bdcf937 3622 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
34978be3 3623 }
1e3baf05
DM
3624 }
3625
19672434 3626
1e3baf05
DM
3627 my $sockets = 1;
3628 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3629 $sockets = $conf->{sockets} if $conf->{sockets};
3630
3631 my $cores = $conf->{cores} || 1;
3bd18e48 3632
de9d1e55 3633 my $maxcpus = $sockets * $cores;
76267728 3634
de9d1e55 3635 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
76267728 3636
de9d1e55
AD
3637 my $allowed_vcpus = $cpuinfo->{cpus};
3638
6965d5d1 3639 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
de9d1e55
AD
3640 if ($allowed_vcpus < $maxcpus);
3641
69c81430 3642 if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
1e3baf05 3643
69c81430
AD
3644 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3645 for (my $i = 2; $i <= $vcpus; $i++) {
3646 my $cpustr = print_cpu_device($conf,$i);
3647 push @$cmd, '-device', $cpustr;
3648 }
3649
3650 } else {
3651
3652 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3653 }
1e3baf05
DM
3654 push @$cmd, '-nodefaults';
3655
32baffb4 3656 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3b408e82 3657
0888fdce
DM
3658 my $bootindex_hash = {};
3659 my $i = 1;
3660 foreach my $o (split(//, $bootorder)) {
3661 $bootindex_hash->{$o} = $i*100;
3662 $i++;
afdb31d5 3663 }
3b408e82 3664
dbea4415 3665 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
1e3baf05 3666
6b64503e 3667 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
1e3baf05 3668
6b64503e 3669 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
1e3baf05 3670
84902837 3671 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
d559309f 3672 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
b7be4ba9
AD
3673 my $socket = vnc_socket($vmid);
3674 push @$cmd, '-vnc', "unix:$socket,x509,password";
3675 } else {
55655ebc 3676 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
b7be4ba9
AD
3677 push @$cmd, '-nographic';
3678 }
3679
1e3baf05 3680 # time drift fix
6b64503e 3681 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
1e3baf05 3682
8c559505 3683 my $useLocaltime = $conf->{localtime};
1e3baf05 3684
4317f69f
AD
3685 if ($winversion >= 5) { # windows
3686 $useLocaltime = 1 if !defined($conf->{localtime});
7a131888 3687
4317f69f
AD
3688 # use time drift fix when acpi is enabled
3689 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3690 $tdf = 1 if !defined($conf->{tdf});
462e8d19 3691 }
4317f69f 3692 }
462e8d19 3693
4317f69f
AD
3694 if ($winversion >= 6) {
3695 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3696 push @$cmd, '-no-hpet';
1e3baf05
DM
3697 }
3698
8c559505
DM
3699 push @$rtcFlags, 'driftfix=slew' if $tdf;
3700
74c02ef7 3701 if (!$kvm) {
8c559505 3702 push @$machineFlags, 'accel=tcg';
7f0b5beb 3703 }
1e3baf05 3704
952958bc
DM
3705 if ($machine_type) {
3706 push @$machineFlags, "type=${machine_type}";
3bafc510
DM
3707 }
3708
8c559505
DM
3709 if ($conf->{startdate}) {
3710 push @$rtcFlags, "base=$conf->{startdate}";
3711 } elsif ($useLocaltime) {
3712 push @$rtcFlags, 'base=localtime';
3713 }
1e3baf05 3714
4fc262bd 3715 push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
519ed28c 3716
0567a4d5
AD
3717 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3718
1e3baf05
DM
3719 push @$cmd, '-S' if $conf->{freeze};
3720
b20df606 3721 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
1e3baf05
DM
3722
3723 # enable sound
3724 #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3725 #push @$cmd, '-soundhw', 'es1370';
3726 #push @$cmd, '-soundhw', $soundhw if $soundhw;
ab6a046f 3727
9d66b397 3728 if (parse_guest_agent($conf)->{enabled}) {
7a6c2150 3729 my $qgasocket = qmp_socket($vmid, 1);
d559309f 3730 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
ab6a046f
AD
3731 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3732 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3733 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3734 }
3735
1d794448 3736 my $spice_port;
2fa3151e 3737
590e698c
DM
3738 if ($qxlnum) {
3739 if ($qxlnum > 1) {
ac087616 3740 if ($winversion){
590e698c 3741 for(my $i = 1; $i < $qxlnum; $i++){
d559309f 3742 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
590e698c
DM
3743 }
3744 } else {
3745 # assume other OS works like Linux
55655ebc
DC
3746 my ($ram, $vram) = ("134217728", "67108864");
3747 if ($vga->{memory}) {
3748 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3749 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3750 }
3751 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3752 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
2fa3151e
AD
3753 }
3754 }
3755
d559309f 3756 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
95a4b4a9 3757
af0eba7e
WB
3758 my $nodename = PVE::INotify::nodename();
3759 my $pfamily = PVE::Tools::get_host_address_family($nodename);
91152441
WB
3760 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3761 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3762 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3763 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
943340a6 3764
91152441 3765 push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
1011b570 3766
d2da6d9b
AD
3767 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3768 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3769 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
1011b570
DM
3770 }
3771
8d9ae0d2
DM
3772 # enable balloon by default, unless explicitly disabled
3773 if (!defined($conf->{balloon}) || $conf->{balloon}) {
d559309f 3774 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
8d9ae0d2
DM
3775 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3776 }
1e3baf05 3777
0ea9541d
DM
3778 if ($conf->{watchdog}) {
3779 my $wdopts = parse_watchdog($conf->{watchdog});
d559309f 3780 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
0a40e8ea 3781 my $watchdog = $wdopts->{model} || 'i6300esb';
5bdcf937
AD
3782 push @$devices, '-device', "$watchdog$pciaddr";
3783 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
0ea9541d
DM
3784 }
3785
1e3baf05 3786 my $vollist = [];
941e0c42 3787 my $scsicontroller = {};
26ee04b6 3788 my $ahcicontroller = {};
cdd20088 3789 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
1e3baf05 3790
5881b913
DM
3791 # Add iscsi initiator name if available
3792 if (my $initiator = get_initiator_name()) {
3793 push @$devices, '-iscsi', "initiator-name=$initiator";
3794 }
3795
1e3baf05
DM
3796 foreach_drive($conf, sub {
3797 my ($ds, $drive) = @_;
3798
ff1a2432 3799 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
1e3baf05 3800 push @$vollist, $drive->{file};
ff1a2432 3801 }
afdb31d5 3802
4dcce9ee
TL
3803 # ignore efidisk here, already added in bios/fw handling code above
3804 return if $drive->{interface} eq 'efidisk';
3805
1e3baf05 3806 $use_virtio = 1 if $ds =~ m/^virtio/;
3b408e82
DM
3807
3808 if (drive_is_cdrom ($drive)) {
3809 if ($bootindex_hash->{d}) {
3810 $drive->{bootindex} = $bootindex_hash->{d};
3811 $bootindex_hash->{d} += 1;
3812 }
3813 } else {
3814 if ($bootindex_hash->{c}) {
3815 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3816 $bootindex_hash->{c} += 1;
3817 }
3818 }
3819
51f492cd
AD
3820 if($drive->{interface} eq 'virtio'){
3821 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3822 }
3823
941e0c42 3824 if ($drive->{interface} eq 'scsi') {
cdd20088 3825
ee034f5c 3826 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
6731a4cf 3827
d559309f 3828 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
a1b7d579 3829 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
fc8b40fd
AD
3830
3831 my $iothread = '';
3832 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3833 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3834 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
e7a5104d
DC
3835 } elsif ($drive->{iothread}) {
3836 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
fc8b40fd
AD
3837 }
3838
6e11f143
AD
3839 my $queues = '';
3840 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3841 $queues = ",num_queues=$drive->{queues}";
3842 }
3843
3844 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
cdd20088 3845 $scsicontroller->{$controller}=1;
941e0c42 3846 }
3b408e82 3847
26ee04b6
DA
3848 if ($drive->{interface} eq 'sata') {
3849 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
d559309f 3850 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
5bdcf937 3851 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
26ee04b6
DA
3852 $ahcicontroller->{$controller}=1;
3853 }
46f58b5f 3854
15b21acc
MR
3855 my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3856 push @$devices, '-drive',$drive_cmd;
d559309f 3857 push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
1e3baf05
DM
3858 });
3859
cc4d6182 3860 for (my $i = 0; $i < $MAX_NETS; $i++) {
5f0c4c32 3861 next if !$conf->{"net$i"};
cc4d6182
DA
3862 my $d = parse_net($conf->{"net$i"});
3863 next if !$d;
1e3baf05 3864
cc4d6182 3865 $use_virtio = 1 if $d->{model} eq 'virtio';
1e3baf05 3866
cc4d6182
DA
3867 if ($bootindex_hash->{n}) {
3868 $d->{bootindex} = $bootindex_hash->{n};
3869 $bootindex_hash->{n} += 1;
3870 }
1e3baf05 3871
d559309f 3872 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
5bdcf937
AD
3873 push @$devices, '-netdev', $netdevfull;
3874
d559309f 3875 my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
5bdcf937
AD
3876 push @$devices, '-device', $netdevicefull;
3877 }
1e3baf05 3878
db656e5f
DM
3879 if (!$q35) {
3880 # add pci bridges
fc79e813
AD
3881 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3882 $bridges->{1} = 1;
3883 $bridges->{2} = 1;
3884 }
3885
6731a4cf
AD
3886 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3887
f8e83f05 3888 while (my ($k, $v) = each %$bridges) {
d559309f 3889 $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
f8e83f05
AD
3890 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3891 }
19672434
DM
3892 }
3893
5bdcf937 3894 push @$cmd, @$devices;
be190583 3895 push @$cmd, '-rtc', join(',', @$rtcFlags)
8c559505 3896 if scalar(@$rtcFlags);
be190583 3897 push @$cmd, '-machine', join(',', @$machineFlags)
8c559505
DM
3898 if scalar(@$machineFlags);
3899 push @$cmd, '-global', join(',', @$globalFlags)
3900 if scalar(@$globalFlags);
3901
76350670
DC
3902 # add custom args
3903 if ($conf->{args}) {
3904 my $aa = PVE::Tools::split_args($conf->{args});
3905 push @$cmd, @$aa;
3906 }
3907
1d794448 3908 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
1e3baf05 3909}
19672434 3910
1e3baf05
DM
3911sub vnc_socket {
3912 my ($vmid) = @_;
3913 return "${var_run_tmpdir}/$vmid.vnc";
3914}
3915
943340a6 3916sub spice_port {
1011b570 3917 my ($vmid) = @_;
943340a6 3918
1d794448 3919 my $res = vm_mon_cmd($vmid, 'query-spice');
943340a6
DM
3920
3921 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
1011b570
DM
3922}
3923
c971c4f2 3924sub qmp_socket {
71bd73b5 3925 my ($vmid, $qga, $name) = @_;
693d12a2 3926 my $sockettype = $qga ? 'qga' : 'qmp';
71bd73b5
DC
3927 my $ext = $name ? '-'.$name : '';
3928 return "${var_run_tmpdir}/$vmid$ext.$sockettype";
c971c4f2
AD
3929}
3930
1e3baf05
DM
3931sub pidfile_name {
3932 my ($vmid) = @_;
3933 return "${var_run_tmpdir}/$vmid.pid";
3934}
3935
86fdcfb2
DA
3936sub vm_devices_list {
3937 my ($vmid) = @_;
3938
ceea9078 3939 my $res = vm_mon_cmd($vmid, 'query-pci');
f721624b 3940 my $devices_to_check = [];
ceea9078
DM
3941 my $devices = {};
3942 foreach my $pcibus (@$res) {
f721624b
DC
3943 push @$devices_to_check, @{$pcibus->{devices}},
3944 }
3945
3946 while (@$devices_to_check) {
3947 my $to_check = [];
3948 for my $d (@$devices_to_check) {
3949 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3950 next if !$d->{'pci_bridge'};
3951
3952 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3953 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
f78cc802 3954 }
f721624b 3955 $devices_to_check = $to_check;
f78cc802
AD
3956 }
3957
3958 my $resblock = vm_mon_cmd($vmid, 'query-block');
3959 foreach my $block (@$resblock) {
3960 if($block->{device} =~ m/^drive-(\S+)/){
3961 $devices->{$1} = 1;
1dc4f496
DM
3962 }
3963 }
86fdcfb2 3964
3d7389fe
DM
3965 my $resmice = vm_mon_cmd($vmid, 'query-mice');
3966 foreach my $mice (@$resmice) {
3967 if ($mice->{name} eq 'QEMU HID Tablet') {
3968 $devices->{tablet} = 1;
3969 last;
3970 }
3971 }
3972
deb091c5
DC
3973 # for usb devices there is no query-usb
3974 # but we can iterate over the entries in
3975 # qom-list path=/machine/peripheral
3976 my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3977 foreach my $per (@$resperipheral) {
3978 if ($per->{name} =~ m/^usb\d+$/) {
3979 $devices->{$per->{name}} = 1;
3980 }
3981 }
3982
1dc4f496 3983 return $devices;
86fdcfb2
DA
3984}
3985
ec21aa11 3986sub vm_deviceplug {
d559309f 3987 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
ae57f6b3 3988
db656e5f
DM
3989 my $q35 = machine_type_is_q35($conf);
3990
95d6343b
DA
3991 my $devices_list = vm_devices_list($vmid);
3992 return 1 if defined($devices_list->{$deviceid});
3993
d559309f 3994 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
fee46675 3995
3d7389fe 3996 if ($deviceid eq 'tablet') {
fee46675 3997
d559309f
WB
3998 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3999
4000 } elsif ($deviceid eq 'keyboard') {
4001
4002 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3d7389fe 4003
4eb68604
DC
4004 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4005
f745762b
DC
4006 die "usb hotplug currently not reliable\n";
4007 # since we can't reliably hot unplug all added usb devices
4008 # and usb passthrough disables live migration
4009 # we disable usb hotplugging for now
4eb68604
DC
4010 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4011
fee46675 4012 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
40f28a9f 4013
22de899a
AD
4014 qemu_iothread_add($vmid, $deviceid, $device);
4015
fee46675 4016 qemu_driveadd($storecfg, $vmid, $device);
d559309f 4017 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
fee46675 4018
5e5dcb73 4019 qemu_deviceadd($vmid, $devicefull);
fee46675
DM
4020 eval { qemu_deviceaddverify($vmid, $deviceid); };
4021 if (my $err = $@) {
63c2da2f
DM
4022 eval { qemu_drivedel($vmid, $deviceid); };
4023 warn $@ if $@;
fee46675 4024 die $err;
5e5dcb73 4025 }
cfc817c7 4026
2733141c 4027 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
fee46675 4028
fc8b40fd 4029
cdd20088 4030 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
d559309f 4031 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
a1b7d579 4032 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
2733141c
AD
4033
4034 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
fee46675 4035
fc8b40fd
AD
4036 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4037 qemu_iothread_add($vmid, $deviceid, $device);
4038 $devicefull .= ",iothread=iothread-$deviceid";
4039 }
4040
6e11f143
AD
4041 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4042 $devicefull .= ",num_queues=$device->{queues}";
4043 }
4044
cfc817c7 4045 qemu_deviceadd($vmid, $devicefull);
fee46675 4046 qemu_deviceaddverify($vmid, $deviceid);
cfc817c7 4047
fee46675
DM
4048 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4049
d559309f 4050 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
fee46675 4051 qemu_driveadd($storecfg, $vmid, $device);
a1b7d579 4052
d559309f 4053 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
fee46675
DM
4054 eval { qemu_deviceadd($vmid, $devicefull); };
4055 if (my $err = $@) {
63c2da2f
DM
4056 eval { qemu_drivedel($vmid, $deviceid); };
4057 warn $@ if $@;
fee46675 4058 die $err;
a4f091a0 4059 }
a4f091a0 4060
fee46675
DM
4061 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4062
d559309f 4063 return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
8718099c
AD
4064
4065 my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4066 my $use_old_bios_files = undef;
4067 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4068
d559309f 4069 my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
2630d2a9 4070 qemu_deviceadd($vmid, $netdevicefull);
fee46675
DM
4071 eval { qemu_deviceaddverify($vmid, $deviceid); };
4072 if (my $err = $@) {
4073 eval { qemu_netdevdel($vmid, $deviceid); };
4074 warn $@ if $@;
4075 die $err;
2630d2a9 4076 }
2630d2a9 4077
fee46675 4078 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
b467f79a 4079
40f28a9f 4080 my $bridgeid = $2;
d559309f 4081 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
40f28a9f 4082 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
a1b7d579 4083
40f28a9f 4084 qemu_deviceadd($vmid, $devicefull);
fee46675
DM
4085 qemu_deviceaddverify($vmid, $deviceid);
4086
4087 } else {
a1b7d579 4088 die "can't hotplug device '$deviceid'\n";
40f28a9f
AD
4089 }
4090
5e5dcb73 4091 return 1;
a4dea331
DA
4092}
4093
3eec5767 4094# fixme: this should raise exceptions on error!
ec21aa11 4095sub vm_deviceunplug {
f19d1c47 4096 my ($vmid, $conf, $deviceid) = @_;
873c2d69 4097
95d6343b
DA
4098 my $devices_list = vm_devices_list($vmid);
4099 return 1 if !defined($devices_list->{$deviceid});
4100
63c2da2f
DM
4101 die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4102
d559309f 4103 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
63c2da2f 4104
3d7389fe 4105 qemu_devicedel($vmid, $deviceid);
3d7389fe 4106
4eb68604
DC
4107 } elsif ($deviceid =~ m/^usb\d+$/) {
4108
f745762b
DC
4109 die "usb hotplug currently not reliable\n";
4110 # when unplugging usb devices this way,
4111 # there may be remaining usb controllers/hubs
4112 # so we disable it for now
4eb68604
DC
4113 qemu_devicedel($vmid, $deviceid);
4114 qemu_devicedelverify($vmid, $deviceid);
4115
63c2da2f 4116 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
f19d1c47 4117
5e5dcb73 4118 qemu_devicedel($vmid, $deviceid);
63c2da2f
DM
4119 qemu_devicedelverify($vmid, $deviceid);
4120 qemu_drivedel($vmid, $deviceid);
22de899a
AD
4121 qemu_iothread_del($conf, $vmid, $deviceid);
4122
2733141c 4123 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
a1b7d579 4124
63c2da2f 4125 qemu_devicedel($vmid, $deviceid);
8ce30dde 4126 qemu_devicedelverify($vmid, $deviceid);
fc8b40fd 4127 qemu_iothread_del($conf, $vmid, $deviceid);
a1b7d579 4128
63c2da2f 4129 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
cfc817c7 4130
63c2da2f
DM
4131 qemu_devicedel($vmid, $deviceid);
4132 qemu_drivedel($vmid, $deviceid);
a1b7d579 4133 qemu_deletescsihw($conf, $vmid, $deviceid);
8ce30dde 4134
63c2da2f 4135 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
a4f091a0 4136
2630d2a9 4137 qemu_devicedel($vmid, $deviceid);
63c2da2f
DM
4138 qemu_devicedelverify($vmid, $deviceid);
4139 qemu_netdevdel($vmid, $deviceid);
4140
4141 } else {
4142 die "can't unplug device '$deviceid'\n";
2630d2a9
DA
4143 }
4144
5e5dcb73
DA
4145 return 1;
4146}
4147
4148sub qemu_deviceadd {
4149 my ($vmid, $devicefull) = @_;
873c2d69 4150
d695b5b7
AD
4151 $devicefull = "driver=".$devicefull;
4152 my %options = split(/[=,]/, $devicefull);
f19d1c47 4153
d695b5b7 4154 vm_mon_cmd($vmid, "device_add" , %options);
5e5dcb73 4155}
afdb31d5 4156
5e5dcb73 4157sub qemu_devicedel {
fee46675 4158 my ($vmid, $deviceid) = @_;
63c2da2f 4159
5a77d8c1 4160 my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
5e5dcb73
DA
4161}
4162
22de899a
AD
4163sub qemu_iothread_add {
4164 my($vmid, $deviceid, $device) = @_;
4165
4166 if ($device->{iothread}) {
4167 my $iothreads = vm_iothreads_list($vmid);
4168 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4169 }
4170}
4171
4172sub qemu_iothread_del {
4173 my($conf, $vmid, $deviceid) = @_;
4174
4175 my $device = parse_drive($deviceid, $conf->{$deviceid});
4176 if ($device->{iothread}) {
4177 my $iothreads = vm_iothreads_list($vmid);
4178 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4179 }
4180}
4181
4d3f29ed
AD
4182sub qemu_objectadd {
4183 my($vmid, $objectid, $qomtype) = @_;
4184
4185 vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4186
4187 return 1;
4188}
4189
4190sub qemu_objectdel {
4191 my($vmid, $objectid) = @_;
4192
4193 vm_mon_cmd($vmid, "object-del", id => $objectid);
4194
4195 return 1;
4196}
4197
5e5dcb73 4198sub qemu_driveadd {
fee46675 4199 my ($storecfg, $vmid, $device) = @_;
5e5dcb73
DA
4200
4201 my $drive = print_drive_full($storecfg, $vmid, $device);
7a69fc3c 4202 $drive =~ s/\\/\\\\/g;
8ead5ec7 4203 my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
fee46675 4204
5e5dcb73 4205 # If the command succeeds qemu prints: "OK"
fee46675
DM
4206 return 1 if $ret =~ m/OK/s;
4207
4208 die "adding drive failed: $ret\n";
5e5dcb73 4209}
afdb31d5 4210
5e5dcb73
DA
4211sub qemu_drivedel {
4212 my($vmid, $deviceid) = @_;
873c2d69 4213
7b7c6d1b 4214 my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
5e5dcb73 4215 $ret =~ s/^\s+//;
a1b7d579 4216
63c2da2f 4217 return 1 if $ret eq "";
a1b7d579 4218
63c2da2f 4219 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
a1b7d579
DM
4220 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4221
63c2da2f 4222 die "deleting drive $deviceid failed : $ret\n";
5e5dcb73 4223}
f19d1c47 4224
5e5dcb73 4225sub qemu_deviceaddverify {
fee46675 4226 my ($vmid, $deviceid) = @_;
873c2d69 4227
5e5dcb73
DA
4228 for (my $i = 0; $i <= 5; $i++) {
4229 my $devices_list = vm_devices_list($vmid);
4230 return 1 if defined($devices_list->{$deviceid});
4231 sleep 1;
afdb31d5 4232 }
fee46675
DM
4233
4234 die "error on hotplug device '$deviceid'\n";
5e5dcb73 4235}
afdb31d5 4236
5e5dcb73
DA
4237
4238sub qemu_devicedelverify {
63c2da2f
DM
4239 my ($vmid, $deviceid) = @_;
4240
a1b7d579 4241 # need to verify that the device is correctly removed as device_del
63c2da2f 4242 # is async and empty return is not reliable
5e5dcb73 4243
5e5dcb73
DA
4244 for (my $i = 0; $i <= 5; $i++) {
4245 my $devices_list = vm_devices_list($vmid);
4246 return 1 if !defined($devices_list->{$deviceid});
4247 sleep 1;
afdb31d5 4248 }
63c2da2f
DM
4249
4250 die "error on hot-unplugging device '$deviceid'\n";
873c2d69
DA
4251}
4252
cdd20088 4253sub qemu_findorcreatescsihw {
d559309f 4254 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
cfc817c7 4255
ee034f5c 4256 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
2733141c
AD
4257
4258 my $scsihwid="$controller_prefix$controller";
cfc817c7
DA
4259 my $devices_list = vm_devices_list($vmid);
4260
cdd20088 4261 if(!defined($devices_list->{$scsihwid})) {
d559309f 4262 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
cfc817c7 4263 }
fee46675 4264
cfc817c7
DA
4265 return 1;
4266}
4267
8ce30dde
AD
4268sub qemu_deletescsihw {
4269 my ($conf, $vmid, $opt) = @_;
4270
4271 my $device = parse_drive($opt, $conf->{$opt});
4272
a1511b3c 4273 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
2733141c
AD
4274 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4275 return 1;
4276 }
4277
ee034f5c 4278 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
8ce30dde
AD
4279
4280 my $devices_list = vm_devices_list($vmid);
4281 foreach my $opt (keys %{$devices_list}) {
74479ee9 4282 if (PVE::QemuServer::is_valid_drivename($opt)) {
8ce30dde
AD
4283 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4284 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4285 return 1;
4286 }
4287 }
4288 }
4289
4290 my $scsihwid="scsihw$controller";
4291
4292 vm_deviceunplug($vmid, $conf, $scsihwid);
4293
4294 return 1;
4295}
4296
281fedb3 4297sub qemu_add_pci_bridge {
d559309f 4298 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
40f28a9f
AD
4299
4300 my $bridges = {};
281fedb3
DM
4301
4302 my $bridgeid;
4303
d559309f 4304 print_pci_addr($device, $bridges, $arch, $machine_type);
40f28a9f
AD
4305
4306 while (my ($k, $v) = each %$bridges) {
4307 $bridgeid = $k;
4308 }
fee46675 4309 return 1 if !defined($bridgeid) || $bridgeid < 1;
281fedb3 4310
40f28a9f
AD
4311 my $bridge = "pci.$bridgeid";
4312 my $devices_list = vm_devices_list($vmid);
4313
281fedb3 4314 if (!defined($devices_list->{$bridge})) {
d559309f 4315 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
40f28a9f 4316 }
281fedb3 4317
40f28a9f
AD
4318 return 1;
4319}
4320
25088687
DM
4321sub qemu_set_link_status {
4322 my ($vmid, $device, $up) = @_;
4323
a1b7d579 4324 vm_mon_cmd($vmid, "set_link", name => $device,
25088687
DM
4325 up => $up ? JSON::true : JSON::false);
4326}
4327
2630d2a9 4328sub qemu_netdevadd {
d559309f 4329 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
2630d2a9 4330
d559309f 4331 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
73aa03b8 4332 my %options = split(/[=,]/, $netdev);
2630d2a9 4333
73aa03b8
AD
4334 vm_mon_cmd($vmid, "netdev_add", %options);
4335 return 1;
2630d2a9
DA
4336}
4337
4338sub qemu_netdevdel {
4339 my ($vmid, $deviceid) = @_;
4340
89c1e0f4 4341 vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
2630d2a9
DA
4342}
4343
16521d63 4344sub qemu_usb_hotplug {
d559309f 4345 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
16521d63
DC
4346
4347 return if !$device;
4348
4349 # remove the old one first
4350 vm_deviceunplug($vmid, $conf, $deviceid);
4351
4352 # check if xhci controller is necessary and available
4353 if ($device->{usb3}) {
4354
4355 my $devicelist = vm_devices_list($vmid);
4356
4357 if (!$devicelist->{xhci}) {
d559309f 4358 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
16521d63
DC
4359 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4360 }
4361 }
4362 my $d = parse_usb_device($device->{host});
4363 $d->{usb3} = $device->{usb3};
4364
4365 # add the new one
d559309f 4366 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
16521d63
DC
4367}
4368
838776ab 4369sub qemu_cpu_hotplug {
8edc9c08 4370 my ($vmid, $conf, $vcpus) = @_;
838776ab 4371
1e881b75
AD
4372 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4373
8edc9c08
AD
4374 my $sockets = 1;
4375 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4376 $sockets = $conf->{sockets} if $conf->{sockets};
4377 my $cores = $conf->{cores} || 1;
4378 my $maxcpus = $sockets * $cores;
838776ab 4379
8edc9c08 4380 $vcpus = $maxcpus if !$vcpus;
3a11fadb 4381
8edc9c08
AD
4382 die "you can't add more vcpus than maxcpus\n"
4383 if $vcpus > $maxcpus;
3a11fadb 4384
8edc9c08 4385 my $currentvcpus = $conf->{vcpus} || $maxcpus;
1e881b75 4386
eba3e64d 4387 if ($vcpus < $currentvcpus) {
1e881b75
AD
4388
4389 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4390
4391 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4392 qemu_devicedel($vmid, "cpu$i");
4393 my $retry = 0;
4394 my $currentrunningvcpus = undef;
4395 while (1) {
4396 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4397 last if scalar(@{$currentrunningvcpus}) == $i-1;
961af8a3 4398 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
1e881b75
AD
4399 $retry++;
4400 sleep 1;
4401 }
4402 #update conf after each succesfull cpu unplug
4403 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4404 PVE::QemuConfig->write_config($vmid, $conf);
4405 }
4406 } else {
961af8a3 4407 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
1e881b75
AD
4408 }
4409
4410 return;
4411 }
838776ab 4412
8edc9c08 4413 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
961af8a3 4414 die "vcpus in running vm does not match its configuration\n"
8edc9c08 4415 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
838776ab 4416
eba3e64d
AD
4417 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4418
4419 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4420 my $cpustr = print_cpu_device($conf, $i);
4421 qemu_deviceadd($vmid, $cpustr);
4422
4423 my $retry = 0;
4424 my $currentrunningvcpus = undef;
4425 while (1) {
4426 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4427 last if scalar(@{$currentrunningvcpus}) == $i;
961af8a3 4428 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
eba3e64d
AD
4429 sleep 1;
4430 $retry++;
4431 }
4432 #update conf after each succesfull cpu hotplug
4433 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4434 PVE::QemuConfig->write_config($vmid, $conf);
4435 }
4436 } else {
4437
4438 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4439 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4440 }
838776ab
AD
4441 }
4442}
4443
affd2f88 4444sub qemu_block_set_io_throttle {
277ca170
WB
4445 my ($vmid, $deviceid,
4446 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
9196a8ec
WB
4447 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4448 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4449 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
affd2f88 4450
f3f323a3
AD
4451 return if !check_running($vmid) ;
4452
277ca170
WB
4453 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4454 bps => int($bps),
4455 bps_rd => int($bps_rd),
4456 bps_wr => int($bps_wr),
4457 iops => int($iops),
4458 iops_rd => int($iops_rd),
4459 iops_wr => int($iops_wr),
4460 bps_max => int($bps_max),
4461 bps_rd_max => int($bps_rd_max),
4462 bps_wr_max => int($bps_wr_max),
4463 iops_max => int($iops_max),
4464 iops_rd_max => int($iops_rd_max),
9196a8ec
WB
4465 iops_wr_max => int($iops_wr_max),
4466 bps_max_length => int($bps_max_length),
4467 bps_rd_max_length => int($bps_rd_max_length),
4468 bps_wr_max_length => int($bps_wr_max_length),
4469 iops_max_length => int($iops_max_length),
4470 iops_rd_max_length => int($iops_rd_max_length),
4471 iops_wr_max_length => int($iops_wr_max_length),
277ca170 4472 );
f3f323a3 4473
affd2f88
AD
4474}
4475
f5eb281a 4476# old code, only used to shutdown old VM after update
dab36e1e
DM
4477sub __read_avail {
4478 my ($fh, $timeout) = @_;
4479
4480 my $sel = new IO::Select;
4481 $sel->add($fh);
4482
4483 my $res = '';
4484 my $buf;
4485
4486 my @ready;
4487 while (scalar (@ready = $sel->can_read($timeout))) {
4488 my $count;
4489 if ($count = $fh->sysread($buf, 8192)) {
4490 if ($buf =~ /^(.*)\(qemu\) $/s) {
4491 $res .= $1;
4492 last;
4493 } else {
4494 $res .= $buf;
4495 }
4496 } else {
4497 if (!defined($count)) {
4498 die "$!\n";
4499 }
4500 last;
4501 }
4502 }
4503
4504 die "monitor read timeout\n" if !scalar(@ready);
f5eb281a 4505
dab36e1e
DM
4506 return $res;
4507}
4508
c1175c92
AD
4509sub qemu_block_resize {
4510 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4511
ed221350 4512 my $running = check_running($vmid);
c1175c92 4513
7246e8f9 4514 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
c1175c92
AD
4515
4516 return if !$running;
4517
4518 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4519
4520}
4521
1ab0057c
AD
4522sub qemu_volume_snapshot {
4523 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4524
ed221350 4525 my $running = check_running($vmid);
1ab0057c 4526
e5eaa028 4527 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
eba2b721 4528 vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
e5eaa028
WL
4529 } else {
4530 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4531 }
1ab0057c
AD
4532}
4533
fc46aff9
AD
4534sub qemu_volume_snapshot_delete {
4535 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4536
ed221350 4537 my $running = check_running($vmid);
fc46aff9 4538
a2f1554b
AD
4539 if($running) {
4540
4541 $running = undef;
4542 my $conf = PVE::QemuConfig->load_config($vmid);
4543 foreach_drive($conf, sub {
4544 my ($ds, $drive) = @_;
4545 $running = 1 if $drive->{file} eq $volid;
4546 });
4547 }
4548
1ef7592f 4549 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
eba2b721 4550 vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
1ef7592f
AD
4551 } else {
4552 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4553 }
fc46aff9
AD
4554}
4555
264e519f
DM
4556sub set_migration_caps {
4557 my ($vmid) = @_;
a89fded1 4558
8b8345f3 4559 my $cap_ref = [];
a89fded1
AD
4560
4561 my $enabled_cap = {
8b8345f3 4562 "auto-converge" => 1,
0b0a47e8 4563 "xbzrle" => 1,
8b8345f3
DM
4564 "x-rdma-pin-all" => 0,
4565 "zero-blocks" => 0,
b62532e4 4566 "compress" => 0
a89fded1
AD
4567 };
4568
8b8345f3 4569 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
a89fded1 4570
8b8345f3 4571 for my $supported_capability (@$supported_capabilities) {
b463a3ce
SP
4572 push @$cap_ref, {
4573 capability => $supported_capability->{capability},
22430fa2
DM
4574 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4575 };
a89fded1
AD
4576 }
4577
8b8345f3
DM
4578 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4579}
a89fded1 4580
81d95ae1 4581my $fast_plug_option = {
7498eb64 4582 'lock' => 1,
81d95ae1 4583 'name' => 1,
a1b7d579 4584 'onboot' => 1,
81d95ae1
DM
4585 'shares' => 1,
4586 'startup' => 1,
b0ec896e 4587 'description' => 1,
ec647db4 4588 'protection' => 1,
8cad5e9b 4589 'vmstatestorage' => 1,
81d95ae1
DM
4590};
4591
3a11fadb
DM
4592# hotplug changes in [PENDING]
4593# $selection hash can be used to only apply specified options, for
4594# example: { cores => 1 } (only apply changed 'cores')
4595# $errors ref is used to return error messages
c427973b 4596sub vmconfig_hotplug_pending {
3a11fadb 4597 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
c427973b 4598
8e90138a 4599 my $defaults = load_defaults();
d559309f 4600 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
c427973b
DM
4601
4602 # commit values which do not have any impact on running VM first
3a11fadb
DM
4603 # Note: those option cannot raise errors, we we do not care about
4604 # $selection and always apply them.
4605
4606 my $add_error = sub {
4607 my ($opt, $msg) = @_;
4608 $errors->{$opt} = "hotplug problem - $msg";
4609 };
c427973b
DM
4610
4611 my $changes = 0;
4612 foreach my $opt (keys %{$conf->{pending}}) { # add/change
81d95ae1 4613 if ($fast_plug_option->{$opt}) {
c427973b
DM
4614 $conf->{$opt} = $conf->{pending}->{$opt};
4615 delete $conf->{pending}->{$opt};
4616 $changes = 1;
4617 }
4618 }
4619
4620 if ($changes) {
ffda963f
FG
4621 PVE::QemuConfig->write_config($vmid, $conf);
4622 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
c427973b
DM
4623 }
4624
b3c2bdd1 4625 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
c427973b 4626
3dc38fbb
WB
4627 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4628 while (my ($opt, $force) = each %$pending_delete_hash) {
3a11fadb 4629 next if $selection && !$selection->{$opt};
3a11fadb 4630 eval {
51a6f637
AD
4631 if ($opt eq 'hotplug') {
4632 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4633 } elsif ($opt eq 'tablet') {
b3c2bdd1 4634 die "skip\n" if !$hotplug_features->{usb};
3a11fadb 4635 if ($defaults->{tablet}) {
d559309f
WB
4636 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4637 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4638 if $arch eq 'aarch64';
3a11fadb 4639 } else {
d559309f
WB
4640 vm_deviceunplug($vmid, $conf, 'tablet');
4641 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
3a11fadb 4642 }
4eb68604 4643 } elsif ($opt =~ m/^usb\d+/) {
f745762b
DC
4644 die "skip\n";
4645 # since we cannot reliably hot unplug usb devices
4646 # we are disabling it
4eb68604
DC
4647 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4648 vm_deviceunplug($vmid, $conf, $opt);
8edc9c08 4649 } elsif ($opt eq 'vcpus') {
b3c2bdd1 4650 die "skip\n" if !$hotplug_features->{cpu};
8edc9c08 4651 qemu_cpu_hotplug($vmid, $conf, undef);
9c2f7069 4652 } elsif ($opt eq 'balloon') {
81d95ae1 4653 # enable balloon device is not hotpluggable
75b51053
DC
4654 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4655 # here we reset the ballooning value to memory
4656 my $balloon = $conf->{memory} || $defaults->{memory};
4657 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
81d95ae1
DM
4658 } elsif ($fast_plug_option->{$opt}) {
4659 # do nothing
3eec5767 4660 } elsif ($opt =~ m/^net(\d+)$/) {
b3c2bdd1 4661 die "skip\n" if !$hotplug_features->{network};
3eec5767 4662 vm_deviceunplug($vmid, $conf, $opt);
74479ee9 4663 } elsif (is_valid_drivename($opt)) {
b3c2bdd1 4664 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
19120f99 4665 vm_deviceunplug($vmid, $conf, $opt);
3dc38fbb 4666 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4d3f29ed
AD
4667 } elsif ($opt =~ m/^memory$/) {
4668 die "skip\n" if !$hotplug_features->{memory};
6779f1ac 4669 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
c8effec3
AD
4670 } elsif ($opt eq 'cpuunits') {
4671 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
58be00f1
AD
4672 } elsif ($opt eq 'cpulimit') {
4673 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3d7389fe 4674 } else {
e56beeda 4675 die "skip\n";
3d7389fe 4676 }
3a11fadb
DM
4677 };
4678 if (my $err = $@) {
e56beeda
DM
4679 &$add_error($opt, $err) if $err ne "skip\n";
4680 } else {
3a11fadb
DM
4681 # save new config if hotplug was successful
4682 delete $conf->{$opt};
4683 vmconfig_undelete_pending_option($conf, $opt);
ffda963f
FG
4684 PVE::QemuConfig->write_config($vmid, $conf);
4685 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3d7389fe 4686 }
3d7389fe
DM
4687 }
4688
9ed7a77c
WB
4689 my $apply_pending_cloudinit;
4690 $apply_pending_cloudinit = sub {
4691 my ($key, $value) = @_;
4692 $apply_pending_cloudinit = sub {}; # once is enough
4693
4694 my @cloudinit_opts = keys %$confdesc_cloudinit;
4695 foreach my $opt (keys %{$conf->{pending}}) {
4696 next if !grep { $_ eq $opt } @cloudinit_opts;
4697 $conf->{$opt} = delete $conf->{pending}->{$opt};
4698 }
4699
4700 my $new_conf = { %$conf };
4701 $new_conf->{$key} = $value;
4702 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4703 };
4704
3d7389fe 4705 foreach my $opt (keys %{$conf->{pending}}) {
3a11fadb 4706 next if $selection && !$selection->{$opt};
3d7389fe 4707 my $value = $conf->{pending}->{$opt};
3a11fadb 4708 eval {
51a6f637
AD
4709 if ($opt eq 'hotplug') {
4710 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4711 } elsif ($opt eq 'tablet') {
b3c2bdd1 4712 die "skip\n" if !$hotplug_features->{usb};
3a11fadb 4713 if ($value == 1) {
d559309f
WB
4714 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4715 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4716 if $arch eq 'aarch64';
3a11fadb 4717 } elsif ($value == 0) {
d559309f
WB
4718 vm_deviceunplug($vmid, $conf, 'tablet');
4719 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
3a11fadb 4720 }
4eb68604 4721 } elsif ($opt =~ m/^usb\d+$/) {
f745762b
DC
4722 die "skip\n";
4723 # since we cannot reliably hot unplug usb devices
4724 # we are disabling it
4eb68604
DC
4725 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4726 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4727 die "skip\n" if !$d;
d559309f 4728 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
8edc9c08 4729 } elsif ($opt eq 'vcpus') {
b3c2bdd1 4730 die "skip\n" if !$hotplug_features->{cpu};
3a11fadb
DM
4731 qemu_cpu_hotplug($vmid, $conf, $value);
4732 } elsif ($opt eq 'balloon') {
81d95ae1 4733 # enable/disable balloning device is not hotpluggable
8fe689e7 4734 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
a1b7d579 4735 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
81d95ae1
DM
4736 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4737
3a11fadb 4738 # allow manual ballooning if shares is set to zero
4cc1efa6 4739 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
9c2f7069
AD
4740 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4741 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4742 }
a1b7d579 4743 } elsif ($opt =~ m/^net(\d+)$/) {
3eec5767 4744 # some changes can be done without hotplug
a1b7d579 4745 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
d559309f 4746 $vmid, $opt, $value, $arch, $machine_type);
74479ee9 4747 } elsif (is_valid_drivename($opt)) {
a05cff86 4748 # some changes can be done without hotplug
9ed7a77c
WB
4749 my $drive = parse_drive($opt, $value);
4750 if (drive_is_cloudinit($drive)) {
4751 &$apply_pending_cloudinit($opt, $value);
4752 }
b3c2bdd1 4753 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
d559309f 4754 $vmid, $opt, $value, 1, $arch, $machine_type);
4d3f29ed
AD
4755 } elsif ($opt =~ m/^memory$/) { #dimms
4756 die "skip\n" if !$hotplug_features->{memory};
6779f1ac 4757 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
c8effec3
AD
4758 } elsif ($opt eq 'cpuunits') {
4759 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
58be00f1 4760 } elsif ($opt eq 'cpulimit') {
c6f773b8 4761 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
58be00f1 4762 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3a11fadb 4763 } else {
e56beeda 4764 die "skip\n"; # skip non-hot-pluggable options
3d7389fe 4765 }
3a11fadb
DM
4766 };
4767 if (my $err = $@) {
e56beeda
DM
4768 &$add_error($opt, $err) if $err ne "skip\n";
4769 } else {
3a11fadb
DM
4770 # save new config if hotplug was successful
4771 $conf->{$opt} = $value;
4772 delete $conf->{pending}->{$opt};
ffda963f
FG
4773 PVE::QemuConfig->write_config($vmid, $conf);
4774 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3d7389fe 4775 }
3d7389fe 4776 }
c427973b 4777}
055d554d 4778
3dc38fbb
WB
4779sub try_deallocate_drive {
4780 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4781
4782 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4783 my $volid = $drive->{file};
4784 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4785 my $sid = PVE::Storage::parse_volume_id($volid);
4786 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
cee01bcb
WB
4787
4788 # check if the disk is really unused
cee01bcb 4789 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
77019edf 4790 if is_volume_in_use($storecfg, $conf, $key, $volid);
cee01bcb 4791 PVE::Storage::vdisk_free($storecfg, $volid);
3dc38fbb 4792 return 1;
40b977f3
WL
4793 } else {
4794 # If vm is not owner of this disk remove from config
4795 return 1;
3dc38fbb
WB
4796 }
4797 }
4798
4799 return undef;
4800}
4801
4802sub vmconfig_delete_or_detach_drive {
4803 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4804
4805 my $drive = parse_drive($opt, $conf->{$opt});
4806
4807 my $rpcenv = PVE::RPCEnvironment::get();
4808 my $authuser = $rpcenv->get_user();
4809
4810 if ($force) {
4811 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4812 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4813 } else {
4814 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4815 }
4816}
4817
055d554d 4818sub vmconfig_apply_pending {
3a11fadb 4819 my ($vmid, $conf, $storecfg) = @_;
c427973b
DM
4820
4821 # cold plug
055d554d 4822
3dc38fbb
WB
4823 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4824 while (my ($opt, $force) = each %$pending_delete_hash) {
055d554d 4825 die "internal error" if $opt =~ m/^unused/;
ffda963f 4826 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
4827 if (!defined($conf->{$opt})) {
4828 vmconfig_undelete_pending_option($conf, $opt);
ffda963f 4829 PVE::QemuConfig->write_config($vmid, $conf);
74479ee9 4830 } elsif (is_valid_drivename($opt)) {
3dc38fbb 4831 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
055d554d
DM
4832 vmconfig_undelete_pending_option($conf, $opt);
4833 delete $conf->{$opt};
ffda963f 4834 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM
4835 } else {
4836 vmconfig_undelete_pending_option($conf, $opt);
4837 delete $conf->{$opt};
ffda963f 4838 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM
4839 }
4840 }
4841
ffda963f 4842 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
4843
4844 foreach my $opt (keys %{$conf->{pending}}) { # add/change
ffda963f 4845 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
4846
4847 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4848 # skip if nothing changed
74479ee9 4849 } elsif (is_valid_drivename($opt)) {
055d554d
DM
4850 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4851 if defined($conf->{$opt});
4852 $conf->{$opt} = $conf->{pending}->{$opt};
4853 } else {
4854 $conf->{$opt} = $conf->{pending}->{$opt};
4855 }
4856
4857 delete $conf->{pending}->{$opt};
ffda963f 4858 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM
4859 }
4860}
4861
3eec5767
DM
4862my $safe_num_ne = sub {
4863 my ($a, $b) = @_;
4864
4865 return 0 if !defined($a) && !defined($b);
4866 return 1 if !defined($a);
4867 return 1 if !defined($b);
4868
4869 return $a != $b;
4870};
4871
4872my $safe_string_ne = sub {
4873 my ($a, $b) = @_;
4874
4875 return 0 if !defined($a) && !defined($b);
4876 return 1 if !defined($a);
4877 return 1 if !defined($b);
4878
4879 return $a ne $b;
4880};
4881
4882sub vmconfig_update_net {
d559309f 4883 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
3eec5767
DM
4884
4885 my $newnet = parse_net($value);
4886
4887 if ($conf->{$opt}) {
4888 my $oldnet = parse_net($conf->{$opt});
4889
4890 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4891 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4892 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4893 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4894
4895 # for non online change, we try to hot-unplug
7196b757 4896 die "skip\n" if !$hotplug;
3eec5767
DM
4897 vm_deviceunplug($vmid, $conf, $opt);
4898 } else {
4899
4900 die "internal error" if $opt !~ m/net(\d+)/;
4901 my $iface = "tap${vmid}i$1";
a1b7d579 4902
25088687
DM
4903 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4904 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
16d08ecf 4905 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
25088687 4906 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
3eec5767 4907 PVE::Network::tap_unplug($iface);
4f4fbeb0
WB
4908 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4909 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4910 # Rate can be applied on its own but any change above needs to
4911 # include the rate in tap_plug since OVS resets everything.
4912 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
3eec5767 4913 }
38c590d9 4914
25088687
DM
4915 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4916 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4917 }
4918
38c590d9 4919 return 1;
3eec5767
DM
4920 }
4921 }
a1b7d579 4922
7196b757 4923 if ($hotplug) {
d559309f 4924 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
38c590d9
DM
4925 } else {
4926 die "skip\n";
4927 }
3eec5767
DM
4928}
4929
a05cff86 4930sub vmconfig_update_disk {
d559309f 4931 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
a05cff86
DM
4932
4933 # fixme: do we need force?
4934
4935 my $drive = parse_drive($opt, $value);
4936
4937 if ($conf->{$opt}) {
4938
4939 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4940
4941 my $media = $drive->{media} || 'disk';
4942 my $oldmedia = $old_drive->{media} || 'disk';
4943 die "unable to change media type\n" if $media ne $oldmedia;
4944
4945 if (!drive_is_cdrom($old_drive)) {
4946
a1b7d579 4947 if ($drive->{file} ne $old_drive->{file}) {
a05cff86 4948
7196b757 4949 die "skip\n" if !$hotplug;
a05cff86
DM
4950
4951 # unplug and register as unused
4952 vm_deviceunplug($vmid, $conf, $opt);
4953 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
a1b7d579 4954
a05cff86
DM
4955 } else {
4956 # update existing disk
4957
4958 # skip non hotpluggable value
6ecfbb44 4959 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
22de899a 4960 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
6e11f143 4961 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
a05cff86
DM
4962 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4963 die "skip\n";
4964 }
4965
4966 # apply throttle
4967 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4968 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4969 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4970 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4971 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4972 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4973 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4974 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4975 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4976 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4977 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
9196a8ec
WB
4978 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4979 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4980 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4981 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4982 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4983 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4984 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
a1b7d579 4985
a05cff86
DM
4986 qemu_block_set_io_throttle($vmid,"drive-$opt",
4987 ($drive->{mbps} || 0)*1024*1024,
4988 ($drive->{mbps_rd} || 0)*1024*1024,
4989 ($drive->{mbps_wr} || 0)*1024*1024,
4990 $drive->{iops} || 0,
4991 $drive->{iops_rd} || 0,
4992 $drive->{iops_wr} || 0,
4993 ($drive->{mbps_max} || 0)*1024*1024,
4994 ($drive->{mbps_rd_max} || 0)*1024*1024,
4995 ($drive->{mbps_wr_max} || 0)*1024*1024,
4996 $drive->{iops_max} || 0,
4997 $drive->{iops_rd_max} || 0,
9196a8ec
WB
4998 $drive->{iops_wr_max} || 0,
4999 $drive->{bps_max_length} || 1,
5000 $drive->{bps_rd_max_length} || 1,
5001 $drive->{bps_wr_max_length} || 1,
5002 $drive->{iops_max_length} || 1,
5003 $drive->{iops_rd_max_length} || 1,
5004 $drive->{iops_wr_max_length} || 1);
a05cff86
DM
5005
5006 }
a1b7d579 5007
a05cff86
DM
5008 return 1;
5009 }
4de1bb25
DM
5010
5011 } else { # cdrom
a1b7d579 5012
4de1bb25
DM
5013 if ($drive->{file} eq 'none') {
5014 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
2d9ddec5
WB
5015 if (drive_is_cloudinit($old_drive)) {
5016 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5017 }
4de1bb25
DM
5018 } else {
5019 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5020 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5021 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5022 }
a1b7d579 5023
34758d66 5024 return 1;
a05cff86
DM
5025 }
5026 }
5027 }
5028
a1b7d579 5029 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4de1bb25 5030 # hotplug new disks
f7b4356f 5031 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
d559309f 5032 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
a05cff86
DM
5033}
5034
1e3baf05 5035sub vm_start {
ba9e1000 5036 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
2189246c 5037 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
1e3baf05 5038
ffda963f
FG
5039 PVE::QemuConfig->lock_config($vmid, sub {
5040 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
1e3baf05 5041
ffda963f 5042 die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
3dcb98d5 5043
ffda963f 5044 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 5045
7e8dcf2c 5046 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
1e3baf05 5047
055d554d 5048 if (!$statefile && scalar(keys %{$conf->{pending}})) {
3a11fadb 5049 vmconfig_apply_pending($vmid, $conf, $storecfg);
ffda963f 5050 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
055d554d
DM
5051 }
5052
0c9a7596
AD
5053 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5054
6c47d546
DM
5055 my $defaults = load_defaults();
5056
5057 # set environment variable useful inside network script
5058 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5059
2189246c
AD
5060 my $local_volumes = {};
5061
3b4cf0f0 5062 if ($targetstorage) {
2189246c
AD
5063 foreach_drive($conf, sub {
5064 my ($ds, $drive) = @_;
5065
5066 return if drive_is_cdrom($drive);
5067
5068 my $volid = $drive->{file};
5069
5070 return if !$volid;
5071
5072 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5073
5074 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5075 return if $scfg->{shared};
5076 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5077 });
5078
5079 my $format = undef;
5080
5081 foreach my $opt (sort keys %$local_volumes) {
5082
5083 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5084 my $drive = parse_drive($opt, $conf->{$opt});
5085
5086 #if remote storage is specified, use default format
5087 if ($targetstorage && $targetstorage ne "1") {
5088 $storeid = $targetstorage;
5089 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5090 $format = $defFormat;
5091 } else {
5092 #else we use same format than original
5093 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5094 $format = qemu_img_format($scfg, $volid);
5095 }
5096
5097 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5098 my $newdrive = $drive;
5099 $newdrive->{format} = $format;
5100 $newdrive->{file} = $newvolid;
5101 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5102 $local_volumes->{$opt} = $drivestr;
5103 #pass drive to conf for command line
5104 $conf->{$opt} = $drivestr;
5105 }
5106 }
5107
67812f9c 5108 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
6c47d546 5109
1e3baf05 5110 my $migrate_port = 0;
5bc1e039 5111 my $migrate_uri;
1e3baf05
DM
5112 if ($statefile) {
5113 if ($statefile eq 'tcp') {
5bc1e039
SP
5114 my $localip = "localhost";
5115 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
af0eba7e 5116 my $nodename = PVE::INotify::nodename();
2de2d6f7 5117
b7a5a225
TL
5118 if (!defined($migration_type)) {
5119 if (defined($datacenterconf->{migration}->{type})) {
5120 $migration_type = $datacenterconf->{migration}->{type};
5121 } else {
5122 $migration_type = 'secure';
5123 }
5124 }
5125
2de2d6f7
TL
5126 if ($migration_type eq 'insecure') {
5127 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5128 if ($migrate_network_addr) {
5129 $localip = $migrate_network_addr;
5130 } else {
5bc1e039 5131 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
2de2d6f7
TL
5132 }
5133
5134 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5bc1e039 5135 }
2de2d6f7 5136
af0eba7e
WB
5137 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5138 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
407e0b8b 5139 $migrate_uri = "tcp:${localip}:${migrate_port}";
6c47d546
DM
5140 push @$cmd, '-incoming', $migrate_uri;
5141 push @$cmd, '-S';
1c9d54bf
TL
5142
5143 } elsif ($statefile eq 'unix') {
5144 # should be default for secure migrations as a ssh TCP forward
5145 # tunnel is not deterministic reliable ready and fails regurarly
5146 # to set up in time, so use UNIX socket forwards
54323eed
TL
5147 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5148 unlink $socket_addr;
5149
5150 $migrate_uri = "unix:$socket_addr";
1c9d54bf
TL
5151
5152 push @$cmd, '-incoming', $migrate_uri;
5153 push @$cmd, '-S';
5154
1e3baf05 5155 } else {
6c47d546 5156 push @$cmd, '-loadstate', $statefile;
1e3baf05 5157 }
91bd6c90
DM
5158 } elsif ($paused) {
5159 push @$cmd, '-S';
1e3baf05
DM
5160 }
5161
1e3baf05 5162 # host pci devices
040b06b7
DA
5163 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5164 my $d = parse_hostpci($conf->{"hostpci$i"});
5165 next if !$d;
b1f72af6
AD
5166 my $pcidevices = $d->{pciid};
5167 foreach my $pcidevice (@$pcidevices) {
2fd24788 5168 my $pciid = $pcidevice->{id};
000fc0a2 5169
b71351a7
DC
5170 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5171 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
b1f72af6 5172 die "no pci device info for device '$pciid'\n" if !$info;
6ab45bd7
DC
5173
5174 if ($d->{mdev}) {
5175 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5176 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5177 } else {
5178 die "can't unbind/bind pci group to vfio '$pciid'\n"
5179 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5180 die "can't reset pci device '$pciid'\n"
5181 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5182 }
b1f72af6 5183 }
040b06b7 5184 }
1e3baf05
DM
5185
5186 PVE::Storage::activate_volumes($storecfg, $vollist);
5187
3c23aa80
WB
5188 if (!check_running($vmid, 1)) {
5189 eval {
5190 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5191 outfunc => sub {}, errfunc => sub {});
5192 };
2b401189
AD
5193 }
5194
8e59d952
WB
5195 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5196 : $defaults->{cpuunits};
5197
f38de678
AD
5198 my $start_timeout = $conf->{hugepages} ? 300 : 30;
5199 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
7023f3ea
AD
5200
5201 my %properties = (
5202 Slice => 'qemu.slice',
5203 KillMode => 'none',
5204 CPUShares => $cpuunits
5205 );
5206
5207 if (my $cpulimit = $conf->{cpulimit}) {
5208 $properties{CPUQuota} = int($cpulimit * 100);
5209 }
5210 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5211
503308ed
WB
5212 my $run_qemu = sub {
5213 PVE::Tools::run_fork sub {
d04d6af1 5214 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
503308ed
WB
5215 run_command($cmd, %run_params);
5216 };
5217 };
5218
7023f3ea
AD
5219 if ($conf->{hugepages}) {
5220
5221 my $code = sub {
5222 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5223 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5224
5225 PVE::QemuServer::Memory::hugepages_mount();
5226 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5227
503308ed 5228 eval { $run_qemu->() };
7023f3ea
AD
5229 if (my $err = $@) {
5230 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5231 die $err;
5232 }
5233
5234 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5235 };
5236 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5237
5238 } else {
503308ed 5239 eval { $run_qemu->() };
7023f3ea 5240 }
77cde36b
DC
5241
5242 if (my $err = $@) {
5243 # deactivate volumes if start fails
5244 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5245 die "start failed: $err";
5246 }
1e3baf05 5247
5bc1e039 5248 print "migration listens on $migrate_uri\n" if $migrate_uri;
afdb31d5 5249
b37ecfe6 5250 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
95381ce0 5251 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
8c609afd 5252 warn $@ if $@;
62de2cbd
DM
5253 }
5254
2189246c
AD
5255 #start nbd server for storage migration
5256 if ($targetstorage) {
2189246c
AD
5257 my $nodename = PVE::INotify::nodename();
5258 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5259 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5260 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5261 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5262
5263 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5264
5265 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5266
5267 foreach my $opt (sort keys %$local_volumes) {
5268 my $volid = $local_volumes->{$opt};
5269 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5270 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5271 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5272 }
5273 }
5274
1d794448 5275 if ($migratedfrom) {
a89fded1 5276 eval {
8e90138a 5277 set_migration_caps($vmid);
a89fded1 5278 };
1d794448 5279 warn $@ if $@;
a89fded1 5280
1d794448
DM
5281 if ($spice_port) {
5282 print "spice listens on port $spice_port\n";
5283 if ($spice_ticket) {
8e90138a
DM
5284 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5285 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
95a4b4a9
AD
5286 }
5287 }
5288
1d794448 5289 } else {
51153f86
DC
5290 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5291 if !$statefile && $conf->{balloon};
25088687
DM
5292
5293 foreach my $opt (keys %$conf) {
5294 next if $opt !~ m/^net\d+$/;
5295 my $nicconf = parse_net($conf->{$opt});
5296 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5297 }
e18b0b99 5298 }
a1b7d579 5299
eb065317
AD
5300 vm_mon_cmd_nocheck($vmid, 'qom-set',
5301 path => "machine/peripheral/balloon0",
5302 property => "guest-stats-polling-interval",
5303 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5304
1e3baf05
DM
5305 });
5306}
5307
0eedc444
AD
5308sub vm_mon_cmd {
5309 my ($vmid, $execute, %params) = @_;
5310
26f11676
DM
5311 my $cmd = { execute => $execute, arguments => \%params };
5312 vm_qmp_command($vmid, $cmd);
0eedc444
AD
5313}
5314
5315sub vm_mon_cmd_nocheck {
5316 my ($vmid, $execute, %params) = @_;
5317
26f11676
DM
5318 my $cmd = { execute => $execute, arguments => \%params };
5319 vm_qmp_command($vmid, $cmd, 1);
0eedc444
AD
5320}
5321
c971c4f2 5322sub vm_qmp_command {
c5a07de5 5323 my ($vmid, $cmd, $nocheck) = @_;
97d62eb7 5324
c971c4f2 5325 my $res;
26f11676 5326
14db5366
DM
5327 my $timeout;
5328 if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5329 $timeout = $cmd->{arguments}->{timeout};
5330 delete $cmd->{arguments}->{timeout};
5331 }
be190583 5332
c971c4f2
AD
5333 eval {
5334 die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
7a6c2150
DM
5335 my $sname = qmp_socket($vmid);
5336 if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
c5a07de5 5337 my $qmpclient = PVE::QMPClient->new();
dab36e1e 5338
14db5366 5339 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
dab36e1e
DM
5340 } else {
5341 die "unable to open monitor socket\n";
5342 }
c971c4f2 5343 };
26f11676 5344 if (my $err = $@) {
c971c4f2
AD
5345 syslog("err", "VM $vmid qmp command failed - $err");
5346 die $err;
5347 }
5348
5349 return $res;
5350}
5351
9df5cbcc
DM
5352sub vm_human_monitor_command {
5353 my ($vmid, $cmdline) = @_;
5354
5355 my $res;
5356
f5eb281a 5357 my $cmd = {
9df5cbcc
DM
5358 execute => 'human-monitor-command',
5359 arguments => { 'command-line' => $cmdline},
5360 };
5361
5362 return vm_qmp_command($vmid, $cmd);
5363}
5364
1e3baf05
DM
5365sub vm_commandline {
5366 my ($storecfg, $vmid) = @_;
5367
ffda963f 5368 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05
DM
5369
5370 my $defaults = load_defaults();
5371
6b64503e 5372 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
1e3baf05 5373
5930c1ff 5374 return PVE::Tools::cmd2string($cmd);
1e3baf05
DM
5375}
5376
5377sub vm_reset {
5378 my ($vmid, $skiplock) = @_;
5379
ffda963f 5380 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5381
ffda963f 5382 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5383
ffda963f 5384 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 5385
816e2c4a 5386 vm_mon_cmd($vmid, "system_reset");
ff1a2432
DM
5387 });
5388}
5389
5390sub get_vm_volumes {
5391 my ($conf) = @_;
1e3baf05 5392
ff1a2432 5393 my $vollist = [];
d5769dc2 5394 foreach_volid($conf, sub {
392f8b5d 5395 my ($volid, $attr) = @_;
ff1a2432 5396
d5769dc2 5397 return if $volid =~ m|^/|;
ff1a2432 5398
d5769dc2
DM
5399 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5400 return if !$sid;
ff1a2432
DM
5401
5402 push @$vollist, $volid;
1e3baf05 5403 });
ff1a2432
DM
5404
5405 return $vollist;
5406}
5407
5408sub vm_stop_cleanup {
70b04821 5409 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
ff1a2432 5410
745fed70 5411 eval {
ff1a2432 5412
254575e9
DM
5413 if (!$keepActive) {
5414 my $vollist = get_vm_volumes($conf);
5415 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5416 }
a1b7d579 5417
ab6a046f 5418 foreach my $ext (qw(mon qmp pid vnc qga)) {
961bfcb2
DM
5419 unlink "/var/run/qemu-server/${vmid}.$ext";
5420 }
a1b7d579 5421
6ab45bd7
DC
5422 foreach my $key (keys %$conf) {
5423 next if $key !~ m/^hostpci(\d+)$/;
5424 my $hostpciindex = $1;
5425 my $d = parse_hostpci($conf->{$key});
5426 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5427
5428 foreach my $pci (@{$d->{pciid}}) {
2fd24788 5429 my $pciid = $pci->{id};
6ab45bd7
DC
5430 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5431 }
5432 }
5433
70b04821 5434 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
745fed70
DM
5435 };
5436 warn $@ if $@; # avoid errors - just warn
1e3baf05
DM
5437}
5438
e6c3b671 5439# Note: use $nockeck to skip tests if VM configuration file exists.
254575e9
DM
5440# We need that when migration VMs to other nodes (files already moved)
5441# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
1e3baf05 5442sub vm_stop {
af30308f 5443 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
9269013a 5444
9269013a 5445 $force = 1 if !defined($force) && !$shutdown;
1e3baf05 5446
af30308f
DM
5447 if ($migratedfrom){
5448 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5449 kill 15, $pid if $pid;
ffda963f 5450 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
70b04821 5451 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
af30308f
DM
5452 return;
5453 }
5454
ffda963f 5455 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5456
e6c3b671 5457 my $pid = check_running($vmid, $nocheck);
ff1a2432 5458 return if !$pid;
1e3baf05 5459
ff1a2432 5460 my $conf;
e6c3b671 5461 if (!$nocheck) {
ffda963f
FG
5462 $conf = PVE::QemuConfig->load_config($vmid);
5463 PVE::QemuConfig->check_lock($conf) if !$skiplock;
7f4a5b5a 5464 if (!defined($timeout) && $shutdown && $conf->{startup}) {
38f7f26c 5465 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
7f4a5b5a
DM
5466 $timeout = $opts->{down} if $opts->{down};
5467 }
e6c3b671 5468 }
19672434 5469
7f4a5b5a 5470 $timeout = 60 if !defined($timeout);
67fb9de6 5471
9269013a
DM
5472 eval {
5473 if ($shutdown) {
9d66b397 5474 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
2ea54503 5475 vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
1c0c1c17 5476 } else {
2ea54503 5477 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
1c0c1c17 5478 }
9269013a 5479 } else {
2ea54503 5480 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
afdb31d5 5481 }
9269013a 5482 };
1e3baf05
DM
5483 my $err = $@;
5484
5485 if (!$err) {
1e3baf05 5486 my $count = 0;
e6c3b671 5487 while (($count < $timeout) && check_running($vmid, $nocheck)) {
1e3baf05
DM
5488 $count++;
5489 sleep 1;
5490 }
5491
5492 if ($count >= $timeout) {
9269013a
DM
5493 if ($force) {
5494 warn "VM still running - terminating now with SIGTERM\n";
5495 kill 15, $pid;
5496 } else {
5497 die "VM quit/powerdown failed - got timeout\n";
5498 }
5499 } else {
70b04821 5500 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
9269013a 5501 return;
1e3baf05
DM
5502 }
5503 } else {
9269013a
DM
5504 if ($force) {
5505 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5506 kill 15, $pid;
5507 } else {
afdb31d5 5508 die "VM quit/powerdown failed\n";
9269013a 5509 }
1e3baf05
DM
5510 }
5511
5512 # wait again
ff1a2432 5513 $timeout = 10;
1e3baf05
DM
5514
5515 my $count = 0;
e6c3b671 5516 while (($count < $timeout) && check_running($vmid, $nocheck)) {
1e3baf05
DM
5517 $count++;
5518 sleep 1;
5519 }
5520
5521 if ($count >= $timeout) {
ff1a2432 5522 warn "VM still running - terminating now with SIGKILL\n";
1e3baf05 5523 kill 9, $pid;
ff1a2432 5524 sleep 1;
1e3baf05
DM
5525 }
5526
70b04821 5527 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
ff1a2432 5528 });
1e3baf05
DM
5529}
5530
5531sub vm_suspend {
5532 my ($vmid, $skiplock) = @_;
5533
ffda963f 5534 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5535
ffda963f 5536 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5537
e79706d4
FG
5538 PVE::QemuConfig->check_lock($conf)
5539 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
bcb7c9cf 5540
f77f91f3 5541 vm_mon_cmd($vmid, "stop");
1e3baf05
DM
5542 });
5543}
5544
5545sub vm_resume {
289e0b85 5546 my ($vmid, $skiplock, $nocheck) = @_;
1e3baf05 5547
ffda963f 5548 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5549
c2786bed
DC
5550 my $res = vm_mon_cmd($vmid, 'query-status');
5551 my $resume_cmd = 'cont';
5552
5553 if ($res->{status} && $res->{status} eq 'suspended') {
5554 $resume_cmd = 'system_wakeup';
5555 }
5556
289e0b85 5557 if (!$nocheck) {
1e3baf05 5558
ffda963f 5559 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5560
e79706d4
FG
5561 PVE::QemuConfig->check_lock($conf)
5562 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
289e0b85 5563
c2786bed 5564 vm_mon_cmd($vmid, $resume_cmd);
289e0b85
AD
5565
5566 } else {
c2786bed 5567 vm_mon_cmd_nocheck($vmid, $resume_cmd);
289e0b85 5568 }
1e3baf05
DM
5569 });
5570}
5571
5fdbe4f0
DM
5572sub vm_sendkey {
5573 my ($vmid, $skiplock, $key) = @_;
1e3baf05 5574
ffda963f 5575 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5576
ffda963f 5577 my $conf = PVE::QemuConfig->load_config($vmid);
f5eb281a 5578
7b7c6d1b
DM
5579 # there is no qmp command, so we use the human monitor command
5580 vm_human_monitor_command($vmid, "sendkey $key");
1e3baf05
DM
5581 });
5582}
5583
5584sub vm_destroy {
5585 my ($storecfg, $vmid, $skiplock) = @_;
5586
ffda963f 5587 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5588
ffda963f 5589 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5590
ff1a2432 5591 if (!check_running($vmid)) {
15cc8784 5592 destroy_vm($storecfg, $vmid, undef, $skiplock);
ff1a2432
DM
5593 } else {
5594 die "VM $vmid is running - destroy failed\n";
1e3baf05
DM
5595 }
5596 });
5597}
5598
3e16d5fc
DM
5599# vzdump restore implementaion
5600
ed221350 5601sub tar_archive_read_firstfile {
3e16d5fc 5602 my $archive = shift;
afdb31d5 5603
3e16d5fc
DM
5604 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5605
5606 # try to detect archive type first
387ba257 5607 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
3e16d5fc 5608 die "unable to open file '$archive'\n";
387ba257 5609 my $firstfile = <$fh>;
3e16d5fc 5610 kill 15, $pid;
387ba257 5611 close $fh;
3e16d5fc
DM
5612
5613 die "ERROR: archive contaions no data\n" if !$firstfile;
5614 chomp $firstfile;
5615
5616 return $firstfile;
5617}
5618
ed221350
DM
5619sub tar_restore_cleanup {
5620 my ($storecfg, $statfile) = @_;
3e16d5fc
DM
5621
5622 print STDERR "starting cleanup\n";
5623
5624 if (my $fd = IO::File->new($statfile, "r")) {
5625 while (defined(my $line = <$fd>)) {
5626 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5627 my $volid = $2;
5628 eval {
5629 if ($volid =~ m|^/|) {
5630 unlink $volid || die 'unlink failed\n';
5631 } else {
ed221350 5632 PVE::Storage::vdisk_free($storecfg, $volid);
3e16d5fc 5633 }
afdb31d5 5634 print STDERR "temporary volume '$volid' sucessfuly removed\n";
3e16d5fc
DM
5635 };
5636 print STDERR "unable to cleanup '$volid' - $@" if $@;
5637 } else {
5638 print STDERR "unable to parse line in statfile - $line";
afdb31d5 5639 }
3e16d5fc
DM
5640 }
5641 $fd->close();
5642 }
5643}
5644
5645sub restore_archive {
a0d1b1a2 5646 my ($archive, $vmid, $user, $opts) = @_;
3e16d5fc 5647
91bd6c90
DM
5648 my $format = $opts->{format};
5649 my $comp;
5650
5651 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5652 $format = 'tar' if !$format;
5653 $comp = 'gzip';
5654 } elsif ($archive =~ m/\.tar$/) {
5655 $format = 'tar' if !$format;
5656 } elsif ($archive =~ m/.tar.lzo$/) {
5657 $format = 'tar' if !$format;
5658 $comp = 'lzop';
5659 } elsif ($archive =~ m/\.vma$/) {
5660 $format = 'vma' if !$format;
5661 } elsif ($archive =~ m/\.vma\.gz$/) {
5662 $format = 'vma' if !$format;
5663 $comp = 'gzip';
5664 } elsif ($archive =~ m/\.vma\.lzo$/) {
5665 $format = 'vma' if !$format;
5666 $comp = 'lzop';
5667 } else {
5668 $format = 'vma' if !$format; # default
5669 }
5670
5671 # try to detect archive format
5672 if ($format eq 'tar') {
5673 return restore_tar_archive($archive, $vmid, $user, $opts);
5674 } else {
5675 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5676 }
5677}
5678
5679sub restore_update_config_line {
5680 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5681
5682 return if $line =~ m/^\#qmdump\#/;
5683 return if $line =~ m/^\#vzdump\#/;
5684 return if $line =~ m/^lock:/;
5685 return if $line =~ m/^unused\d+:/;
5686 return if $line =~ m/^parent:/;
ca3e4fa4 5687 return if $line =~ m/^template:/; # restored VM is never a template
91bd6c90 5688
b5b99790 5689 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
91bd6c90
DM
5690 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5691 # try to convert old 1.X settings
5692 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5693 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5694 my ($model, $macaddr) = split(/\=/, $devconfig);
b5b99790 5695 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
91bd6c90
DM
5696 my $net = {
5697 model => $model,
5698 bridge => "vmbr$ind",
5699 macaddr => $macaddr,
5700 };
5701 my $netstr = print_net($net);
5702
5703 print $outfd "net$cookie->{netcount}: $netstr\n";
5704 $cookie->{netcount}++;
5705 }
5706 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5707 my ($id, $netstr) = ($1, $2);
5708 my $net = parse_net($netstr);
b5b99790 5709 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
91bd6c90
DM
5710 $netstr = print_net($net);
5711 print $outfd "$id: $netstr\n";
6470743f 5712 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
91bd6c90 5713 my $virtdev = $1;
907ea891 5714 my $value = $3;
d9faf790
WB
5715 my $di = parse_drive($virtdev, $value);
5716 if (defined($di->{backup}) && !$di->{backup}) {
91bd6c90 5717 print $outfd "#$line";
c0f7406e 5718 } elsif ($map->{$virtdev}) {
8fd57431 5719 delete $di->{format}; # format can change on restore
91bd6c90 5720 $di->{file} = $map->{$virtdev};
ed221350 5721 $value = print_drive($vmid, $di);
91bd6c90
DM
5722 print $outfd "$virtdev: $value\n";
5723 } else {
5724 print $outfd $line;
5725 }
1a0c2f03 5726 } elsif (($line =~ m/^vmgenid: (.*)/)) {
babecffe 5727 my $vmgenid = $1;
6ee499ff 5728 if ($vmgenid ne '0') {
1a0c2f03 5729 # always generate a new vmgenid if there was a valid one setup
6ee499ff
DC
5730 $vmgenid = generate_uuid();
5731 }
1a0c2f03 5732 print $outfd "vmgenid: $vmgenid\n";
19a5dd55
WL
5733 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5734 my ($uuid, $uuid_str);
5735 UUID::generate($uuid);
5736 UUID::unparse($uuid, $uuid_str);
5737 my $smbios1 = parse_smbios1($2);
5738 $smbios1->{uuid} = $uuid_str;
5739 print $outfd $1.print_smbios1($smbios1)."\n";
91bd6c90
DM
5740 } else {
5741 print $outfd $line;
5742 }
5743}
5744
5745sub scan_volids {
5746 my ($cfg, $vmid) = @_;
5747
5748 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5749
5750 my $volid_hash = {};
5751 foreach my $storeid (keys %$info) {
5752 foreach my $item (@{$info->{$storeid}}) {
5753 next if !($item->{volid} && $item->{size});
5996a936 5754 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
91bd6c90
DM
5755 $volid_hash->{$item->{volid}} = $item;
5756 }
5757 }
5758
5759 return $volid_hash;
5760}
5761
77019edf
WB
5762sub is_volume_in_use {
5763 my ($storecfg, $conf, $skip_drive, $volid) = @_;
a8e2f942 5764
77019edf 5765 my $path = PVE::Storage::path($storecfg, $volid);
a8e2f942
DM
5766
5767 my $scan_config = sub {
5768 my ($cref, $snapname) = @_;
5769
5770 foreach my $key (keys %$cref) {
5771 my $value = $cref->{$key};
74479ee9 5772 if (is_valid_drivename($key)) {
a8e2f942
DM
5773 next if $skip_drive && $key eq $skip_drive;
5774 my $drive = parse_drive($key, $value);
5775 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
77019edf 5776 return 1 if $volid eq $drive->{file};
a8e2f942 5777 if ($drive->{file} =~ m!^/!) {
77019edf 5778 return 1 if $drive->{file} eq $path;
a8e2f942
DM
5779 } else {
5780 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5781 next if !$storeid;
5782 my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5783 next if !$scfg;
77019edf 5784 return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
a8e2f942
DM
5785 }
5786 }
5787 }
77019edf
WB
5788
5789 return 0;
a8e2f942
DM
5790 };
5791
77019edf 5792 return 1 if &$scan_config($conf);
a8e2f942
DM
5793
5794 undef $skip_drive;
5795
77019edf
WB
5796 foreach my $snapname (keys %{$conf->{snapshots}}) {
5797 return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
a8e2f942
DM
5798 }
5799
77019edf 5800 return 0;
a8e2f942
DM
5801}
5802
91bd6c90
DM
5803sub update_disksize {
5804 my ($vmid, $conf, $volid_hash) = @_;
be190583 5805
91bd6c90 5806 my $changes;
53b81297 5807 my $prefix = "VM $vmid:";
91bd6c90 5808
c449137a
DC
5809 # used and unused disks
5810 my $referenced = {};
91bd6c90 5811
5996a936
DM
5812 # Note: it is allowed to define multiple storages with same path (alias), so
5813 # we need to check both 'volid' and real 'path' (two different volid can point
5814 # to the same path).
5815
c449137a 5816 my $referencedpath = {};
be190583 5817
91bd6c90
DM
5818 # update size info
5819 foreach my $opt (keys %$conf) {
74479ee9 5820 if (is_valid_drivename($opt)) {
ed221350 5821 my $drive = parse_drive($opt, $conf->{$opt});
91bd6c90
DM
5822 my $volid = $drive->{file};
5823 next if !$volid;
5824
c449137a 5825 $referenced->{$volid} = 1;
be190583 5826 if ($volid_hash->{$volid} &&
5996a936 5827 (my $path = $volid_hash->{$volid}->{path})) {
c449137a 5828 $referencedpath->{$path} = 1;
5996a936 5829 }
91bd6c90 5830
ed221350 5831 next if drive_is_cdrom($drive);
91bd6c90
DM
5832 next if !$volid_hash->{$volid};
5833
5834 $drive->{size} = $volid_hash->{$volid}->{size};
7a907ce6
DM
5835 my $new = print_drive($vmid, $drive);
5836 if ($new ne $conf->{$opt}) {
5837 $changes = 1;
5838 $conf->{$opt} = $new;
53b81297 5839 print "$prefix update disk '$opt' information.\n";
7a907ce6 5840 }
91bd6c90
DM
5841 }
5842 }
5843
5996a936
DM
5844 # remove 'unusedX' entry if volume is used
5845 foreach my $opt (keys %$conf) {
5846 next if $opt !~ m/^unused\d+$/;
5847 my $volid = $conf->{$opt};
5848 my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
c449137a 5849 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
53b81297 5850 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5996a936
DM
5851 $changes = 1;
5852 delete $conf->{$opt};
5853 }
c449137a
DC
5854
5855 $referenced->{$volid} = 1;
5856 $referencedpath->{$path} = 1 if $path;
5996a936
DM
5857 }
5858
91bd6c90
DM
5859 foreach my $volid (sort keys %$volid_hash) {
5860 next if $volid =~ m/vm-$vmid-state-/;
c449137a 5861 next if $referenced->{$volid};
5996a936
DM
5862 my $path = $volid_hash->{$volid}->{path};
5863 next if !$path; # just to be sure
c449137a 5864 next if $referencedpath->{$path};
91bd6c90 5865 $changes = 1;
53b81297
TL
5866 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5867 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
c449137a 5868 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
91bd6c90
DM
5869 }
5870
5871 return $changes;
5872}
5873
5874sub rescan {
9224dcee 5875 my ($vmid, $nolock, $dryrun) = @_;
91bd6c90 5876
20519efc 5877 my $cfg = PVE::Storage::config();
91bd6c90 5878
b9a1a3ab
TL
5879 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5880 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
4771526a
AA
5881 foreach my $stor (keys %{$cfg->{ids}}) {
5882 delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5883 }
5884
53b81297 5885 print "rescan volumes...\n";
91bd6c90
DM
5886 my $volid_hash = scan_volids($cfg, $vmid);
5887
5888 my $updatefn = sub {
5889 my ($vmid) = @_;
5890
ffda963f 5891 my $conf = PVE::QemuConfig->load_config($vmid);
be190583 5892
ffda963f 5893 PVE::QemuConfig->check_lock($conf);
91bd6c90 5894
03da3f0d
DM
5895 my $vm_volids = {};
5896 foreach my $volid (keys %$volid_hash) {
5897 my $info = $volid_hash->{$volid};
5898 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5899 }
5900
5901 my $changes = update_disksize($vmid, $conf, $vm_volids);
91bd6c90 5902
9224dcee 5903 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
91bd6c90
DM
5904 };
5905
5906 if (defined($vmid)) {
5907 if ($nolock) {
5908 &$updatefn($vmid);
5909 } else {
ffda963f 5910 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
91bd6c90
DM
5911 }
5912 } else {
5913 my $vmlist = config_list();
5914 foreach my $vmid (keys %$vmlist) {
5915 if ($nolock) {
5916 &$updatefn($vmid);
5917 } else {
ffda963f 5918 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
be190583 5919 }
91bd6c90
DM
5920 }
5921 }
5922}
5923
5924sub restore_vma_archive {
5925 my ($archive, $vmid, $user, $opts, $comp) = @_;
5926
91bd6c90
DM
5927 my $readfrom = $archive;
5928
7c536e11
WB
5929 my $cfg = PVE::Storage::config();
5930 my $commands = [];
5931 my $bwlimit = $opts->{bwlimit};
5932
5933 my $dbg_cmdstring = '';
5934 my $add_pipe = sub {
5935 my ($cmd) = @_;
5936 push @$commands, $cmd;
5937 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5938 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
91bd6c90 5939 $readfrom = '-';
7c536e11
WB
5940 };
5941
5942 my $input = undef;
5943 if ($archive eq '-') {
5944 $input = '<&STDIN';
5945 } else {
5946 # If we use a backup from a PVE defined storage we also consider that
5947 # storage's rate limit:
5948 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5949 if (defined($volid)) {
5950 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5951 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5952 if ($readlimit) {
5953 print STDERR "applying read rate limit: $readlimit\n";
9444c6e4 5954 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7c536e11
WB
5955 $add_pipe->($cstream);
5956 }
5957 }
5958 }
5959
5960 if ($comp) {
5961 my $cmd;
91bd6c90 5962 if ($comp eq 'gzip') {
7c536e11 5963 $cmd = ['zcat', $readfrom];
91bd6c90 5964 } elsif ($comp eq 'lzop') {
7c536e11 5965 $cmd = ['lzop', '-d', '-c', $readfrom];
91bd6c90
DM
5966 } else {
5967 die "unknown compression method '$comp'\n";
5968 }
7c536e11 5969 $add_pipe->($cmd);
91bd6c90
DM
5970 }
5971
5972 my $tmpdir = "/var/tmp/vzdumptmp$$";
5973 rmtree $tmpdir;
5974
5975 # disable interrupts (always do cleanups)
5b97ef24
TL
5976 local $SIG{INT} =
5977 local $SIG{TERM} =
5978 local $SIG{QUIT} =
5979 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
91bd6c90
DM
5980
5981 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5982 POSIX::mkfifo($mapfifo, 0600);
5983 my $fifofh;
5984
5985 my $openfifo = sub {
5986 open($fifofh, '>', $mapfifo) || die $!;
5987 };
5988
7c536e11 5989 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
91bd6c90
DM
5990
5991 my $oldtimeout;
5992 my $timeout = 5;
5993
5994 my $devinfo = {};
5995
5996 my $rpcenv = PVE::RPCEnvironment::get();
5997
ffda963f 5998 my $conffile = PVE::QemuConfig->config_file($vmid);
91bd6c90
DM
5999 my $tmpfn = "$conffile.$$.tmp";
6000
ed221350 6001 # Note: $oldconf is undef if VM does not exists
ffda963f
FG
6002 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6003 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
ed221350 6004
7c536e11
WB
6005 my %storage_limits;
6006
91bd6c90
DM
6007 my $print_devmap = sub {
6008 my $virtdev_hash = {};
6009
6010 my $cfgfn = "$tmpdir/qemu-server.conf";
6011
6012 # we can read the config - that is already extracted
6013 my $fh = IO::File->new($cfgfn, "r") ||
6014 "unable to read qemu-server.conf - $!\n";
6015
6738ab9c 6016 my $fwcfgfn = "$tmpdir/qemu-server.fw";
3457d090
WL
6017 if (-f $fwcfgfn) {
6018 my $pve_firewall_dir = '/etc/pve/firewall';
6019 mkdir $pve_firewall_dir; # make sure the dir exists
6020 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6021 }
6738ab9c 6022
91bd6c90
DM
6023 while (defined(my $line = <$fh>)) {
6024 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6025 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6026 die "archive does not contain data for drive '$virtdev'\n"
6027 if !$devinfo->{$devname};
6028 if (defined($opts->{storage})) {
6029 $storeid = $opts->{storage} || 'local';
6030 } elsif (!$storeid) {
6031 $storeid = 'local';
6032 }
6033 $format = 'raw' if !$format;
6034 $devinfo->{$devname}->{devname} = $devname;
6035 $devinfo->{$devname}->{virtdev} = $virtdev;
6036 $devinfo->{$devname}->{format} = $format;
6037 $devinfo->{$devname}->{storeid} = $storeid;
6038
be190583 6039 # check permission on storage
91bd6c90
DM
6040 my $pool = $opts->{pool}; # todo: do we need that?
6041 if ($user ne 'root@pam') {
6042 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6043 }
6044
7c536e11
WB
6045 $storage_limits{$storeid} = $bwlimit;
6046
91bd6c90
DM
6047 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6048 }
6049 }
6050
7c536e11
WB
6051 foreach my $key (keys %storage_limits) {
6052 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6053 next if !$limit;
6054 print STDERR "rate limit for storage $key: $limit KiB/s\n";
6055 $storage_limits{$key} = $limit * 1024;
6056 }
6057
91bd6c90 6058 foreach my $devname (keys %$devinfo) {
be190583
DM
6059 die "found no device mapping information for device '$devname'\n"
6060 if !$devinfo->{$devname}->{virtdev};
91bd6c90
DM
6061 }
6062
ed221350 6063 # create empty/temp config
be190583 6064 if ($oldconf) {
ed221350
DM
6065 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6066 foreach_drive($oldconf, sub {
6067 my ($ds, $drive) = @_;
6068
6069 return if drive_is_cdrom($drive);
6070
6071 my $volid = $drive->{file};
6072
6073 return if !$volid || $volid =~ m|^/|;
6074
6075 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6076 return if !$path || !$owner || ($owner != $vmid);
6077
6078 # Note: only delete disk we want to restore
6079 # other volumes will become unused
6080 if ($virtdev_hash->{$ds}) {
6b72854b
FG
6081 eval { PVE::Storage::vdisk_free($cfg, $volid); };
6082 if (my $err = $@) {
6083 warn $err;
6084 }
ed221350
DM
6085 }
6086 });
381b8fae
DC
6087
6088 # delete vmstate files
6089 # since after the restore we have no snapshots anymore
6090 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6091 my $snap = $oldconf->{snapshots}->{$snapname};
6092 if ($snap->{vmstate}) {
6093 eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6094 if (my $err = $@) {
6095 warn $err;
6096 }
6097 }
6098 }
ed221350
DM
6099 }
6100
6101 my $map = {};
91bd6c90
DM
6102 foreach my $virtdev (sort keys %$virtdev_hash) {
6103 my $d = $virtdev_hash->{$virtdev};
6104 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
7c536e11
WB
6105 my $storeid = $d->{storeid};
6106 my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6107
6108 my $map_opts = '';
6109 if (my $limit = $storage_limits{$storeid}) {
6110 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6111 }
8fd57431
DM
6112
6113 # test if requested format is supported
7c536e11 6114 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
8fd57431
DM
6115 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6116 $d->{format} = $defFormat if !$supported;
6117
7c536e11 6118 my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
91bd6c90
DM
6119 $d->{format}, undef, $alloc_size);
6120 print STDERR "new volume ID is '$volid'\n";
6121 $d->{volid} = $volid;
6122 my $path = PVE::Storage::path($cfg, $volid);
6123
5f96f4df
WL
6124 PVE::Storage::activate_volumes($cfg,[$volid]);
6125
91bd6c90 6126 my $write_zeros = 1;
88240a83 6127 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
91bd6c90
DM
6128 $write_zeros = 0;
6129 }
6130
7c536e11 6131 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
91bd6c90
DM
6132
6133 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6134 $map->{$virtdev} = $volid;
6135 }
6136
6137 $fh->seek(0, 0) || die "seek failed - $!\n";
6138
6139 my $outfd = new IO::File ($tmpfn, "w") ||
6140 die "unable to write config for VM $vmid\n";
6141
6142 my $cookie = { netcount => 0 };
6143 while (defined(my $line = <$fh>)) {
be190583 6144 restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
91bd6c90
DM
6145 }
6146
6147 $fh->close();
6148 $outfd->close();
6149 };
6150
6151 eval {
6152 # enable interrupts
6cb0144a
EK
6153 local $SIG{INT} =
6154 local $SIG{TERM} =
6155 local $SIG{QUIT} =
6156 local $SIG{HUP} =
6157 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
91bd6c90
DM
6158 local $SIG{ALRM} = sub { die "got timeout\n"; };
6159
6160 $oldtimeout = alarm($timeout);
6161
6162 my $parser = sub {
6163 my $line = shift;
6164
6165 print "$line\n";
6166
6167 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6168 my ($dev_id, $size, $devname) = ($1, $2, $3);
6169 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6170 } elsif ($line =~ m/^CTIME: /) {
46f58b5f 6171 # we correctly received the vma config, so we can disable
3cf90d7a
DM
6172 # the timeout now for disk allocation (set to 10 minutes, so
6173 # that we always timeout if something goes wrong)
6174 alarm(600);
91bd6c90
DM
6175 &$print_devmap();
6176 print $fifofh "done\n";
6177 my $tmp = $oldtimeout || 0;
6178 $oldtimeout = undef;
6179 alarm($tmp);
6180 close($fifofh);
6181 }
6182 };
be190583 6183
7c536e11
WB
6184 print "restore vma archive: $dbg_cmdstring\n";
6185 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
91bd6c90
DM
6186 };
6187 my $err = $@;
6188
6189 alarm($oldtimeout) if $oldtimeout;
6190
5f96f4df
WL
6191 my $vollist = [];
6192 foreach my $devname (keys %$devinfo) {
6193 my $volid = $devinfo->{$devname}->{volid};
6194 push @$vollist, $volid if $volid;
6195 }
6196
5f96f4df
WL
6197 PVE::Storage::deactivate_volumes($cfg, $vollist);
6198
91bd6c90
DM
6199 unlink $mapfifo;
6200
6201 if ($err) {
6202 rmtree $tmpdir;
6203 unlink $tmpfn;
6204
91bd6c90
DM
6205 foreach my $devname (keys %$devinfo) {
6206 my $volid = $devinfo->{$devname}->{volid};
6207 next if !$volid;
6208 eval {
6209 if ($volid =~ m|^/|) {
6210 unlink $volid || die 'unlink failed\n';
6211 } else {
6212 PVE::Storage::vdisk_free($cfg, $volid);
6213 }
6214 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6215 };
6216 print STDERR "unable to cleanup '$volid' - $@" if $@;
6217 }
6218 die $err;
6219 }
6220
6221 rmtree $tmpdir;
ed221350
DM
6222
6223 rename($tmpfn, $conffile) ||
91bd6c90
DM
6224 die "unable to commit configuration file '$conffile'\n";
6225
ed221350
DM
6226 PVE::Cluster::cfs_update(); # make sure we read new file
6227
91bd6c90
DM
6228 eval { rescan($vmid, 1); };
6229 warn $@ if $@;
6230}
6231
6232sub restore_tar_archive {
6233 my ($archive, $vmid, $user, $opts) = @_;
6234
9c502e26 6235 if ($archive ne '-') {
ed221350 6236 my $firstfile = tar_archive_read_firstfile($archive);
9c502e26
DM
6237 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6238 if $firstfile ne 'qemu-server.conf';
6239 }
3e16d5fc 6240
20519efc 6241 my $storecfg = PVE::Storage::config();
ebb55558 6242
ed221350 6243 # destroy existing data - keep empty config
ffda963f 6244 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
ebb55558 6245 destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
ed221350 6246
3e16d5fc
DM
6247 my $tocmd = "/usr/lib/qemu-server/qmextract";
6248
2415a446 6249 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
a0d1b1a2 6250 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
3e16d5fc
DM
6251 $tocmd .= ' --prealloc' if $opts->{prealloc};
6252 $tocmd .= ' --info' if $opts->{info};
6253
a0d1b1a2 6254 # tar option "xf" does not autodetect compression when read from STDIN,
9c502e26 6255 # so we pipe to zcat
2415a446
DM
6256 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6257 PVE::Tools::shellquote("--to-command=$tocmd");
3e16d5fc
DM
6258
6259 my $tmpdir = "/var/tmp/vzdumptmp$$";
6260 mkpath $tmpdir;
6261
6262 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6263 local $ENV{VZDUMP_VMID} = $vmid;
a0d1b1a2 6264 local $ENV{VZDUMP_USER} = $user;
3e16d5fc 6265
ffda963f 6266 my $conffile = PVE::QemuConfig->config_file($vmid);
3e16d5fc
DM
6267 my $tmpfn = "$conffile.$$.tmp";
6268
6269 # disable interrupts (always do cleanups)
6cb0144a
EK
6270 local $SIG{INT} =
6271 local $SIG{TERM} =
6272 local $SIG{QUIT} =
6273 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
3e16d5fc 6274
afdb31d5 6275 eval {
3e16d5fc 6276 # enable interrupts
6cb0144a
EK
6277 local $SIG{INT} =
6278 local $SIG{TERM} =
6279 local $SIG{QUIT} =
6280 local $SIG{HUP} =
6281 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
3e16d5fc 6282
9c502e26
DM
6283 if ($archive eq '-') {
6284 print "extracting archive from STDIN\n";
6285 run_command($cmd, input => "<&STDIN");
6286 } else {
6287 print "extracting archive '$archive'\n";
6288 run_command($cmd);
6289 }
3e16d5fc
DM
6290
6291 return if $opts->{info};
6292
6293 # read new mapping
6294 my $map = {};
6295 my $statfile = "$tmpdir/qmrestore.stat";
6296 if (my $fd = IO::File->new($statfile, "r")) {
6297 while (defined (my $line = <$fd>)) {
6298 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6299 $map->{$1} = $2 if $1;
6300 } else {
6301 print STDERR "unable to parse line in statfile - $line\n";
6302 }
6303 }
6304 $fd->close();
6305 }
6306
6307 my $confsrc = "$tmpdir/qemu-server.conf";
6308
6309 my $srcfd = new IO::File($confsrc, "r") ||
6310 die "unable to open file '$confsrc'\n";
6311
6312 my $outfd = new IO::File ($tmpfn, "w") ||
6313 die "unable to write config for VM $vmid\n";
6314
91bd6c90 6315 my $cookie = { netcount => 0 };
3e16d5fc 6316 while (defined (my $line = <$srcfd>)) {
be190583 6317 restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
3e16d5fc
DM
6318 }
6319
6320 $srcfd->close();
6321 $outfd->close();
6322 };
6323 my $err = $@;
6324
afdb31d5 6325 if ($err) {
3e16d5fc
DM
6326
6327 unlink $tmpfn;
6328
ed221350 6329 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
afdb31d5 6330
3e16d5fc 6331 die $err;
afdb31d5 6332 }
3e16d5fc
DM
6333
6334 rmtree $tmpdir;
6335
6336 rename $tmpfn, $conffile ||
6337 die "unable to commit configuration file '$conffile'\n";
91bd6c90 6338
ed221350
DM
6339 PVE::Cluster::cfs_update(); # make sure we read new file
6340
91bd6c90
DM
6341 eval { rescan($vmid, 1); };
6342 warn $@ if $@;
3e16d5fc
DM
6343};
6344
65a5ce88 6345sub foreach_storage_used_by_vm {
18bfb361
DM
6346 my ($conf, $func) = @_;
6347
6348 my $sidhash = {};
6349
8ddbcf8b
FG
6350 foreach_drive($conf, sub {
6351 my ($ds, $drive) = @_;
6352 return if drive_is_cdrom($drive);
18bfb361
DM
6353
6354 my $volid = $drive->{file};
6355
6356 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
be190583 6357 $sidhash->{$sid} = $sid if $sid;
8ddbcf8b 6358 });
18bfb361
DM
6359
6360 foreach my $sid (sort keys %$sidhash) {
6361 &$func($sid);
6362 }
6363}
6364
e5eaa028
WL
6365sub do_snapshots_with_qemu {
6366 my ($storecfg, $volid) = @_;
6367
6368 my $storage_name = PVE::Storage::parse_volume_id($volid);
6369
116da784
WL
6370 if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6371 && !$storecfg->{ids}->{$storage_name}->{krbd}){
e5eaa028
WL
6372 return 1;
6373 }
6374
6375 if ($volid =~ m/\.(qcow2|qed)$/){
6376 return 1;
6377 }
6378
6379 return undef;
6380}
6381
4dcc780c 6382sub qga_check_running {
a4938c72 6383 my ($vmid, $nowarn) = @_;
4dcc780c
WL
6384
6385 eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6386 if ($@) {
a4938c72 6387 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
4dcc780c
WL
6388 return 0;
6389 }
6390 return 1;
6391}
6392
04a69bb4
AD
6393sub template_create {
6394 my ($vmid, $conf, $disk) = @_;
6395
04a69bb4 6396 my $storecfg = PVE::Storage::config();
04a69bb4 6397
9cd07842
DM
6398 foreach_drive($conf, sub {
6399 my ($ds, $drive) = @_;
6400
6401 return if drive_is_cdrom($drive);
6402 return if $disk && $ds ne $disk;
6403
6404 my $volid = $drive->{file};
bbd56097 6405 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
9cd07842 6406
04a69bb4
AD
6407 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6408 $drive->{file} = $voliddst;
152fe752 6409 $conf->{$ds} = print_drive($vmid, $drive);
ffda963f 6410 PVE::QemuConfig->write_config($vmid, $conf);
04a69bb4 6411 });
04a69bb4
AD
6412}
6413
5133de42 6414sub qemu_img_convert {
988e2714 6415 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5133de42
AD
6416
6417 my $storecfg = PVE::Storage::config();
6418 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6419 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6420
6421 if ($src_storeid && $dst_storeid) {
6bb91c17
DM
6422
6423 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6424
5133de42
AD
6425 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6426 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6427
6428 my $src_format = qemu_img_format($src_scfg, $src_volname);
6429 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6430
6431 my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6432 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6433
6434 my $cmd = [];
eed24303 6435 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
29004a20 6436 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
d4940091
FG
6437 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6438 push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
988e2714
WB
6439 push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6440 if ($is_zero_initialized) {
6441 push @$cmd, "zeroinit:$dst_path";
6442 } else {
6443 push @$cmd, $dst_path;
6444 }
5133de42
AD
6445
6446 my $parser = sub {
6447 my $line = shift;
6448 if($line =~ m/\((\S+)\/100\%\)/){
6449 my $percent = $1;
6450 my $transferred = int($size * $percent / 100);
6451 my $remaining = $size - $transferred;
6452
6453 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6454 }
6455
6456 };
6457
6458 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6459 my $err = $@;
6460 die "copy failed: $err" if $err;
6461 }
6462}
6463
6464sub qemu_img_format {
6465 my ($scfg, $volname) = @_;
6466
9c52f5ed 6467 if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
5133de42 6468 return $1;
be190583 6469 } else {
5133de42 6470 return "raw";
5133de42
AD
6471 }
6472}
6473
cfad42af 6474sub qemu_drive_mirror {
5619e74a 6475 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
cfad42af 6476
5a345967
AD
6477 $jobs = {} if !$jobs;
6478
6479 my $qemu_target;
6480 my $format;
35e4ab04 6481 $jobs->{"drive-$drive"} = {};
152fe752 6482
1e5143de 6483 if ($dst_volid =~ /^nbd:/) {
87955688 6484 $qemu_target = $dst_volid;
5a345967 6485 $format = "nbd";
5a345967 6486 } else {
5a345967
AD
6487 my $storecfg = PVE::Storage::config();
6488 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6489
6490 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
cfad42af 6491
5a345967 6492 $format = qemu_img_format($dst_scfg, $dst_volname);
21ccdb50 6493
5a345967 6494 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
21ccdb50 6495
5a345967
AD
6496 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6497 }
988e2714
WB
6498
6499 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
88383920
DM
6500 $opts->{format} = $format if $format;
6501
5a345967 6502 print "drive mirror is starting for drive-$drive\n";
21ccdb50 6503
5a345967 6504 eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
35e4ab04 6505
5a345967
AD
6506 if (my $err = $@) {
6507 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6508 die "mirroring error: $err";
6509 }
6510
5619e74a 6511 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5a345967
AD
6512}
6513
6514sub qemu_drive_mirror_monitor {
5619e74a 6515 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
2e953867 6516
08ac653f 6517 eval {
5a345967
AD
6518 my $err_complete = 0;
6519
08ac653f 6520 while (1) {
5a345967
AD
6521 die "storage migration timed out\n" if $err_complete > 300;
6522
08ac653f 6523 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
08ac653f 6524
5a345967
AD
6525 my $running_mirror_jobs = {};
6526 foreach my $stat (@$stats) {
6527 next if $stat->{type} ne 'mirror';
6528 $running_mirror_jobs->{$stat->{device}} = $stat;
6529 }
08ac653f 6530
5a345967 6531 my $readycounter = 0;
67fb9de6 6532
5a345967
AD
6533 foreach my $job (keys %$jobs) {
6534
6535 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6536 print "$job : finished\n";
6537 delete $jobs->{$job};
6538 next;
6539 }
6540
bd2d5fe6 6541 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
f34ebd52 6542
5a345967
AD
6543 my $busy = $running_mirror_jobs->{$job}->{busy};
6544 my $ready = $running_mirror_jobs->{$job}->{ready};
6545 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6546 my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6547 my $remaining = $total - $transferred;
6548 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
08ac653f 6549
5a345967
AD
6550 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6551 }
f34ebd52 6552
d1782eba 6553 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
5a345967 6554 }
b467f79a 6555
5a345967
AD
6556 last if scalar(keys %$jobs) == 0;
6557
6558 if ($readycounter == scalar(keys %$jobs)) {
6559 print "all mirroring jobs are ready \n";
6560 last if $skipcomplete; #do the complete later
6561
6562 if ($vmiddst && $vmiddst != $vmid) {
1a988fd2
DC
6563 my $agent_running = $qga && qga_check_running($vmid);
6564 if ($agent_running) {
5619e74a
AD
6565 print "freeze filesystem\n";
6566 eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6567 } else {
6568 print "suspend vm\n";
6569 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6570 }
6571
5a345967
AD
6572 # if we clone a disk for a new target vm, we don't switch the disk
6573 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
5619e74a 6574
1a988fd2 6575 if ($agent_running) {
5619e74a
AD
6576 print "unfreeze filesystem\n";
6577 eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6578 } else {
6579 print "resume vm\n";
6580 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
6581 }
6582
2e953867 6583 last;
5a345967
AD
6584 } else {
6585
6586 foreach my $job (keys %$jobs) {
6587 # try to switch the disk if source and destination are on the same guest
bd2d5fe6 6588 print "$job: Completing block job...\n";
5a345967
AD
6589
6590 eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6591 if ($@ =~ m/cannot be completed/) {
bd2d5fe6 6592 print "$job: Block job cannot be completed, try again.\n";
5a345967
AD
6593 $err_complete++;
6594 }else {
bd2d5fe6 6595 print "$job: Completed successfully.\n";
5a345967
AD
6596 $jobs->{$job}->{complete} = 1;
6597 }
6598 }
2e953867 6599 }
08ac653f 6600 }
08ac653f 6601 sleep 1;
cfad42af 6602 }
08ac653f 6603 };
88383920 6604 my $err = $@;
08ac653f 6605
88383920 6606 if ($err) {
5a345967 6607 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
88383920
DM
6608 die "mirroring error: $err";
6609 }
6610
5a345967
AD
6611}
6612
6613sub qemu_blockjobs_cancel {
6614 my ($vmid, $jobs) = @_;
6615
6616 foreach my $job (keys %$jobs) {
bd2d5fe6 6617 print "$job: Cancelling block job\n";
5a345967
AD
6618 eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6619 $jobs->{$job}->{cancel} = 1;
6620 }
6621
6622 while (1) {
6623 my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6624
6625 my $running_jobs = {};
6626 foreach my $stat (@$stats) {
6627 $running_jobs->{$stat->{device}} = $stat;
6628 }
6629
6630 foreach my $job (keys %$jobs) {
6631
bd2d5fe6
WB
6632 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6633 print "$job: Done.\n";
5a345967
AD
6634 delete $jobs->{$job};
6635 }
6636 }
6637
6638 last if scalar(keys %$jobs) == 0;
6639
6640 sleep 1;
cfad42af
AD
6641 }
6642}
6643
152fe752 6644sub clone_disk {
be190583 6645 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5619e74a 6646 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
152fe752
DM
6647
6648 my $newvolid;
6649
6650 if (!$full) {
6651 print "create linked clone of drive $drivename ($drive->{file})\n";
258e646c 6652 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
152fe752
DM
6653 push @$newvollist, $newvolid;
6654 } else {
5a345967 6655
152fe752
DM
6656 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6657 $storeid = $storage if $storage;
6658
44549149 6659 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
152fe752
DM
6660 my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6661
6662 print "create full clone of drive $drivename ($drive->{file})\n";
931432bd
WB
6663 my $name = undef;
6664 if (drive_is_cloudinit($drive)) {
6665 $name = "vm-$newvmid-cloudinit";
8fa6a851 6666 $snapname = undef;
931432bd
WB
6667 # cloudinit only supports raw and qcow2 atm:
6668 if ($dst_format eq 'qcow2') {
6669 $name .= '.qcow2';
6670 } elsif ($dst_format ne 'raw') {
6671 die "clone: unhandled format for cloudinit image\n";
6672 }
6673 }
6674 $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
152fe752
DM
6675 push @$newvollist, $newvolid;
6676
3999f370 6677 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
1dbd6d30 6678
988e2714 6679 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
152fe752 6680 if (!$running || $snapname) {
988e2714 6681 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
152fe752 6682 } else {
2e541679
AD
6683
6684 my $kvmver = get_running_qemu_version ($vmid);
6685 if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
961af8a3
WB
6686 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6687 if $drive->{iothread};
2e541679 6688 }
2af848a2 6689
5619e74a 6690 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
be190583 6691 }
152fe752
DM
6692 }
6693
6694 my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6695
6696 my $disk = $drive;
6697 $disk->{format} = undef;
6698 $disk->{file} = $newvolid;
6699 $disk->{size} = $size;
6700
6701 return $disk;
6702}
6703
ff556cf2
DM
6704# this only works if VM is running
6705sub get_current_qemu_machine {
6706 my ($vmid) = @_;
6707
6708 my $cmd = { execute => 'query-machines', arguments => {} };
8e90138a 6709 my $res = vm_qmp_command($vmid, $cmd);
ff556cf2
DM
6710
6711 my ($current, $default);
6712 foreach my $e (@$res) {
6713 $default = $e->{name} if $e->{'is-default'};
6714 $current = $e->{name} if $e->{'is-current'};
6715 }
6716
6717 # fallback to the default machine if current is not supported by qemu
6718 return $current || $default || 'pc';
6719}
6720
98cfd8b6
AD
6721sub get_running_qemu_version {
6722 my ($vmid) = @_;
6723 my $cmd = { execute => 'query-version', arguments => {} };
6724 my $res = vm_qmp_command($vmid, $cmd);
6725 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6726}
6727
23f73120
AD
6728sub qemu_machine_feature_enabled {
6729 my ($machine, $kvmver, $version_major, $version_minor) = @_;
6730
6731 my $current_major;
6732 my $current_minor;
6733
d731ecbe 6734 if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
23f73120
AD
6735
6736 $current_major = $3;
6737 $current_minor = $4;
6738
6739 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6740
6741 $current_major = $1;
6742 $current_minor = $2;
6743 }
6744
dd84e5ec
WB
6745 return 1 if $current_major > $version_major ||
6746 ($current_major == $version_major &&
6747 $current_minor >= $version_minor);
23f73120
AD
6748}
6749
42dbd2ee
AD
6750sub qemu_machine_pxe {
6751 my ($vmid, $conf, $machine) = @_;
6752
6753 $machine = PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6754
3807f3e4
DC
6755 if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6756 $machine .= '.pxe';
42dbd2ee
AD
6757 }
6758
d1363934 6759 return $machine;
42dbd2ee
AD
6760}
6761
249c4a6c
AD
6762sub qemu_use_old_bios_files {
6763 my ($machine_type) = @_;
6764
6765 return if !$machine_type;
6766
6767 my $use_old_bios_files = undef;
6768
6769 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6770 $machine_type = $1;
6771 $use_old_bios_files = 1;
6772 } else {
74cc511f 6773 my $kvmver = kvm_user_version();
249c4a6c
AD
6774 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6775 # load new efi bios files on migration. So this hack is required to allow
6776 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6777 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
74cc511f 6778 $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
249c4a6c
AD
6779 }
6780
6781 return ($use_old_bios_files, $machine_type);
6782}
6783
96ed3574
WB
6784sub create_efidisk($$$$$) {
6785 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
3e1f1122 6786
96ed3574
WB
6787 my (undef, $ovmf_vars) = get_ovmf_files($arch);
6788 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
3e1f1122 6789
96ed3574 6790 my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
3e1f1122
TL
6791 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6792 PVE::Storage::activate_volumes($storecfg, [$volid]);
6793
6794 my $path = PVE::Storage::path($storecfg, $volid);
6795 eval {
96ed3574 6796 run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
3e1f1122
TL
6797 };
6798 die "Copying EFI vars image failed: $@" if $@;
6799
6800 return ($volid, $vars_size);
6801}
6802
22de899a
AD
6803sub vm_iothreads_list {
6804 my ($vmid) = @_;
6805
6806 my $res = vm_mon_cmd($vmid, 'query-iothreads');
6807
6808 my $iothreads = {};
6809 foreach my $iothread (@$res) {
6810 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6811 }
6812
6813 return $iothreads;
6814}
6815
ee034f5c
AD
6816sub scsihw_infos {
6817 my ($conf, $drive) = @_;
6818
6819 my $maxdev = 0;
6820
7fe1b688 6821 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
ee034f5c 6822 $maxdev = 7;
a1511b3c 6823 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
ee034f5c
AD
6824 $maxdev = 1;
6825 } else {
6826 $maxdev = 256;
6827 }
6828
6829 my $controller = int($drive->{index} / $maxdev);
a1511b3c 6830 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
ee034f5c
AD
6831
6832 return ($maxdev, $controller, $controller_prefix);
6833}
a1511b3c 6834
075e8249
PA
6835sub add_hyperv_enlightenments {
6836 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
4317f69f 6837
4317f69f
AD
6838 return if $winversion < 6;
6839 return if $bios && $bios eq 'ovmf' && $winversion < 8;
6840
5aba3953
AD
6841 push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6842
4317f69f
AD
6843 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6844 push @$cpuFlags , 'hv_spinlocks=0x1fff';
6845 push @$cpuFlags , 'hv_vapic';
6846 push @$cpuFlags , 'hv_time';
6847 } else {
6848 push @$cpuFlags , 'hv_spinlocks=0xffff';
6849 }
6850
6851 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6852 push @$cpuFlags , 'hv_reset';
6853 push @$cpuFlags , 'hv_vpindex';
6854 push @$cpuFlags , 'hv_runtime';
6855 }
6856
6857 if ($winversion >= 7) {
6858 push @$cpuFlags , 'hv_relaxed';
ebb346d6 6859
df648a6a 6860 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
ebb346d6
AD
6861 push @$cpuFlags , 'hv_synic';
6862 push @$cpuFlags , 'hv_stimer';
6863 }
4317f69f
AD
6864 }
6865}
6866
6867sub windows_version {
6868 my ($ostype) = @_;
6869
6870 return 0 if !$ostype;
6871
6872 my $winversion = 0;
6873
6874 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6875 $winversion = 5;
6876 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6877 $winversion = 6;
6878 } elsif ($ostype =~ m/^win(\d+)$/) {
6879 $winversion = $1;
6880 }
6881
6882 return $winversion;
6883}
6884
44549149
EK
6885sub resolve_dst_disk_format {
6886 my ($storecfg, $storeid, $src_volname, $format) = @_;
6887 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6888
6889 if (!$format) {
6890 # if no target format is specified, use the source disk format as hint
6891 if ($src_volname) {
6892 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6893 $format = qemu_img_format($scfg, $src_volname);
6894 } else {
6895 return $defFormat;
6896 }
6897 }
6898
6899 # test if requested format is supported - else use default
6900 my $supported = grep { $_ eq $format } @$validFormats;
6901 $format = $defFormat if !$supported;
6902 return $format;
6903}
6904
ae2fcb3b
EK
6905sub resolve_first_disk {
6906 my $conf = shift;
6907 my @disks = PVE::QemuServer::valid_drive_names();
6908 my $firstdisk;
6909 foreach my $ds (reverse @disks) {
6910 next if !$conf->{$ds};
6911 my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6912 next if PVE::QemuServer::drive_is_cdrom($disk);
6913 $firstdisk = $ds;
6914 }
6915 return $firstdisk;
6916}
6917
6ee499ff 6918sub generate_uuid {
ae2fcb3b
EK
6919 my ($uuid, $uuid_str);
6920 UUID::generate($uuid);
6921 UUID::unparse($uuid, $uuid_str);
6ee499ff
DC
6922 return $uuid_str;
6923}
6924
6925sub generate_smbios1_uuid {
6926 return "uuid=".generate_uuid();
ae2fcb3b
EK
6927}
6928
9c152e87
TL
6929sub nbd_stop {
6930 my ($vmid) = @_;
6931
6932 vm_mon_cmd($vmid, 'nbd-server-stop');
6933}
6934
65e866e5
DM
6935# bash completion helper
6936
6937sub complete_backup_archives {
6938 my ($cmdname, $pname, $cvalue) = @_;
6939
6940 my $cfg = PVE::Storage::config();
6941
6942 my $storeid;
6943
6944 if ($cvalue =~ m/^([^:]+):/) {
6945 $storeid = $1;
6946 }
6947
6948 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6949
6950 my $res = [];
6951 foreach my $id (keys %$data) {
6952 foreach my $item (@{$data->{$id}}) {
6953 next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6954 push @$res, $item->{volid} if defined($item->{volid});
6955 }
6956 }
6957
6958 return $res;
6959}
6960
6961my $complete_vmid_full = sub {
6962 my ($running) = @_;
6963
6964 my $idlist = vmstatus();
6965
6966 my $res = [];
6967
6968 foreach my $id (keys %$idlist) {
6969 my $d = $idlist->{$id};
6970 if (defined($running)) {
6971 next if $d->{template};
6972 next if $running && $d->{status} ne 'running';
6973 next if !$running && $d->{status} eq 'running';
6974 }
6975 push @$res, $id;
6976
6977 }
6978 return $res;
6979};
6980
6981sub complete_vmid {
6982 return &$complete_vmid_full();
6983}
6984
6985sub complete_vmid_stopped {
6986 return &$complete_vmid_full(0);
6987}
6988
6989sub complete_vmid_running {
6990 return &$complete_vmid_full(1);
6991}
6992
335af808
DM
6993sub complete_storage {
6994
6995 my $cfg = PVE::Storage::config();
6996 my $ids = $cfg->{ids};
6997
6998 my $res = [];
6999 foreach my $sid (keys %$ids) {
7000 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
c4c844ef 7001 next if !$ids->{$sid}->{content}->{images};
335af808
DM
7002 push @$res, $sid;
7003 }
7004
7005 return $res;
7006}
7007
1e3baf05 70081;