]> git.proxmox.com Git - qemu-server.git/blame - PVE/QemuServer.pm
projects / qemu-server.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
fix #2570: add 'keephugepages' config
[qemu-server.git] / PVE / QemuServer.pm
CommitLineData
1e3baf05
DM		 1package PVE::QemuServer;
2
3use strict;
990fc5e2 4use warnings;
3ff84d6f 5
5da072fb
TL		 6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
1e3baf05 10use File::Basename;
5da072fb 11use File::Copy qw(copy);
1e3baf05
DM		 12use File::Path;
13use File::stat;
14use Getopt::Long;
5da072fb
TL		 15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
1e3baf05 20use IPC::Open3;
c971c4f2 21use JSON;
1f30ac3a 22use MIME::Base64;
5da072fb
TL		 23use POSIX;
24use Storable qw(dclone);
25use Time::HiRes qw(gettimeofday);
26use URI::Escape;
425441e6 27use UUID;
5da072fb 28
82841214 29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
48cf040f 30use PVE::DataCenterConfig;
5da072fb 31use PVE::Exception qw(raise raise_param_exc);
81d6e4e1 32use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
1e3baf05 33use PVE::INotify;
5da072fb 34use PVE::JSONSchema qw(get_standard_option);
1e3baf05 35use PVE::ProcFSTools;
91bd6c90 36use PVE::RPCEnvironment;
5da072fb 37use PVE::Storage;
b71351a7 38use PVE::SysFSTools;
d04d6af1 39use PVE::Systemd;
82841214 40use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
5da072fb
TL		 41
42use PVE::QMPClient;
43use PVE::QemuConfig;
2f18c84d 44use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
5da072fb 45use PVE::QemuServer::Cloudinit;
d786a274 46use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
912792e2 47use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
2ea5fb7e 48use PVE::QemuServer::Machine;
5da072fb 49use PVE::QemuServer::Memory;
0a13e08e 50use PVE::QemuServer::Monitor qw(mon_cmd);
74c17b7a 51use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
5da072fb 52use PVE::QemuServer::USB qw(parse_usb_device);
1e3baf05 53
28e129cc
AD		 54my $have_sdn;
55eval {
56 require PVE::Network::SDN::Zones;
57 $have_sdn = 1;
58};
59
102cf9d8 60my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
96ed3574
WB		 61my $OVMF = {
62 x86_64 => [
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
65 ],
66 aarch64 => [
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
69 ],
70};
2ddc0a5c 71
7f0b5beb 72my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
1e3baf05 73
19672434 74# Note about locking: we use flock on the config file protect
1e3baf05
DM		 75# against concurent actions.
76# Aditionaly, we have a 'lock' setting in the config file. This
22c377f0 77# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
1e3baf05
DM		 78# allowed when such lock is set. But you can ignore this kind of
79# lock with the --skiplock flag.
80
97d62eb7 81cfs_register_file('/qemu-server/',
1858638f
DM		 82 \&parse_vm_config,
83 \&write_vm_config);
1e3baf05 84
3ea94c60
DM		 85PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
86 description => "Some command save/restore state from this location.",
87 type => 'string',
88 maxLength => 128,
89 optional => 1,
90});
91
c6737ef1
DC		 92PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
93 description => "Specifies the Qemu machine type.",
94 type => 'string',
9471e48b 95 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
c6737ef1
DC		 96 maxLength => 40,
97 optional => 1,
98});
99
bf8fc5a3
FG		 100
101sub map_storage {
102 my ($map, $source) = @_;
103
104 return $source if !defined($map);
105
106 return $map->{entries}->{$source}
b6d9b54b 107 if $map->{entries} && defined($map->{entries}->{$source});
bf8fc5a3
FG		 108
109 return $map->{default} if $map->{default};
110
111 # identity (fallback)
112 return $source;
113}
114
115PVE::JSONSchema::register_standard_option('pve-targetstorage', {
116 description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
117 type => 'string',
118 format => 'storagepair-list',
119 optional => 1,
120});
121
1e3baf05
DM		 122#no warnings 'redefine';
123
c8effec3
AD		 124sub cgroups_write {
125 my ($controller, $vmid, $option, $value) = @_;
126
3a515a88
DM		 127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE::ProcFSTools::write_proc_entry($path, $value);
c8effec3
AD		 129
130}
131
38277afc
TL		 132my $nodename_cache;
133sub nodename {
134 $nodename_cache //= PVE::INotify::nodename();
135 return $nodename_cache;
136}
1e3baf05 137
ec3582b5
WB		 138my $watchdog_fmt = {
139 model => {
140 default_key => 1,
141 type => 'string',
142 enum => [qw(i6300esb ib700)],
143 description => "Watchdog type to emulate.",
144 default => 'i6300esb',
145 optional => 1,
146 },
147 action => {
148 type => 'string',
149 enum => [qw(reset shutdown poweroff pause debug none)],
150 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
151 optional => 1,
152 },
153};
154PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
155
9d66b397
SI		 156my $agent_fmt = {
157 enabled => {
158 description => "Enable/disable Qemu GuestAgent.",
159 type => 'boolean',
160 default => 0,
161 default_key => 1,
162 },
163 fstrim_cloned_disks => {
164 description => "Run fstrim after cloning/moving a disk.",
165 type => 'boolean',
166 optional => 1,
167 default => 0
168 },
48657158
MD		 169 type => {
170 description => "Select the agent type",
171 type => 'string',
172 default => 'virtio',
173 optional => 1,
174 enum => [qw(virtio isa)],
175 },
9d66b397
SI		 176};
177
55655ebc
DC		 178my $vga_fmt = {
179 type => {
180 description => "Select the VGA type.",
181 type => 'string',
182 default => 'std',
183 optional => 1,
184 default_key => 1,
7c954c42 185 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
55655ebc
DC		 186 },
187 memory => {
188 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
189 type => 'integer',
190 optional => 1,
191 minimum => 4,
192 maximum => 512,
193 },
194};
195
6dbcb073
DC		 196my $ivshmem_fmt = {
197 size => {
198 type => 'integer',
199 minimum => 1,
200 description => "The size of the file in MB.",
201 },
202 name => {
203 type => 'string',
204 pattern => '[a-zA-Z0-9\-]+',
205 optional => 1,
206 format_description => 'string',
207 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
208 },
209};
210
1448547f
AL		 211my $audio_fmt = {
212 device => {
213 type => 'string',
214 enum => [qw(ich9-intel-hda intel-hda AC97)],
215 description => "Configure an audio device."
216 },
217 driver => {
218 type => 'string',
219 enum => ['spice'],
220 default => 'spice',
221 optional => 1,
222 description => "Driver backend for the audio device."
223 },
224};
225
c4df18db
AL		 226my $spice_enhancements_fmt = {
227 foldersharing => {
228 type => 'boolean',
229 optional => 1,
d282a24d 230 default => '0',
c4df18db
AL		 231 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
232 },
233 videostreaming => {
234 type => 'string',
235 enum => ['off', 'all', 'filter'],
d282a24d 236 default => 'off',
c4df18db
AL		 237 optional => 1,
238 description => "Enable video streaming. Uses compression for detected video streams."
239 },
240};
241
2cf61f33
SR		 242my $rng_fmt = {
243 source => {
244 type => 'string',
245 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
246 default_key => 1,
247 description => "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
255 },
256 max_bytes => {
257 type => 'integer',
258 description => "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
262 optional => 1,
263
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
267 default => 1024,
268 },
269 period => {
270 type => 'integer',
271 description => "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
274 optional => 1,
275 default => 1000,
276 },
277};
278
1e3baf05
DM		 279my $confdesc = {
280 onboot => {
281 optional => 1,
282 type => 'boolean',
283 description => "Specifies whether a VM will be started during system bootup.",
284 default => 0,
285 },
286 autostart => {
287 optional => 1,
288 type => 'boolean',
289 description => "Automatic restart after crash (currently ignored).",
290 default => 0,
291 },
2ff09f52
DA		 292 hotplug => {
293 optional => 1,
b3c2bdd1
DM		 294 type => 'string', format => 'pve-hotplug-features',
295 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
2ff09f52 297 },
1e3baf05
DM		 298 reboot => {
299 optional => 1,
300 type => 'boolean',
301 description => "Allow reboot. If set to '0' the VM exit on reboot.",
302 default => 1,
303 },
304 lock => {
305 optional => 1,
306 type => 'string',
307 description => "Lock/unlock the VM.",
159719e5 308 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
1e3baf05
DM		 309 },
310 cpulimit => {
311 optional => 1,
c6f773b8 312 type => 'number',
52261945
DM		 313 description => "Limit of CPU usage.",
314 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
1e3baf05 315 minimum => 0,
c6f773b8 316 maximum => 128,
52261945 317 default => 0,
1e3baf05
DM		 318 },
319 cpuunits => {
320 optional => 1,
321 type => 'integer',
52261945 322 description => "CPU weight for a VM.",
237239bf
PA		 323 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
324 minimum => 2,
325 maximum => 262144,
613d76a1 326 default => 1024,
1e3baf05
DM		 327 },
328 memory => {
329 optional => 1,
330 type => 'integer',
7878afeb 331 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
1e3baf05
DM		 332 minimum => 16,
333 default => 512,
334 },
13a48620
DA		 335 balloon => {
336 optional => 1,
337 type => 'integer',
8b1accf7
DM		 338 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
339 minimum => 0,
340 },
341 shares => {
342 optional => 1,
343 type => 'integer',
82329cd5 344 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
8b1accf7
DM		 345 minimum => 0,
346 maximum => 50000,
347 default => 1000,
13a48620 348 },
1e3baf05
DM		 349 keyboard => {
350 optional => 1,
351 type => 'string',
f889aa0f 352 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
aea47dd6 353 "It should not be necessary to set it.",
e95fe75f 354 enum => PVE::Tools::kvmkeymaplist(),
aea47dd6 355 default => undef,
1e3baf05
DM		 356 },
357 name => {
358 optional => 1,
7fabe17d 359 type => 'string', format => 'dns-name',
1e3baf05
DM		 360 description => "Set a name for the VM. Only used on the configuration web interface.",
361 },
cdd20088
AD		 362 scsihw => {
363 optional => 1,
364 type => 'string',
52261945 365 description => "SCSI controller model",
6731a4cf 366 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
cdd20088
AD		 367 default => 'lsi',
368 },
1e3baf05
DM		 369 description => {
370 optional => 1,
371 type => 'string',
0581fe4f 372 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
1e3baf05
DM		 373 },
374 ostype => {
375 optional => 1,
376 type => 'string',
0cb9971e 377 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
52261945
DM		 378 description => "Specify guest operating system.",
379 verbose_description => <<EODESC,
380Specify guest operating system. This is used to enable special
381optimization/features for specific operating systems:
382
383[horizontal]
384other;; unspecified OS
385wxp;; Microsoft Windows XP
386w2k;; Microsoft Windows 2000
387w2k3;; Microsoft Windows 2003
388w2k8;; Microsoft Windows 2008
389wvista;; Microsoft Windows Vista
390win7;; Microsoft Windows 7
44c2a647
TL		 391win8;; Microsoft Windows 8/2012/2012r2
392win10;; Microsoft Windows 10/2016
52261945 393l24;; Linux 2.4 Kernel
a5269260 394l26;; Linux 2.6 - 5.X Kernel
52261945 395solaris;; Solaris/OpenSolaris/OpenIndiania kernel
1e3baf05
DM		 396EODESC
397 },
398 boot => {
399 optional => 1,
400 type => 'string',
401 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
402 pattern => '[acdn]{1,4}',
32baffb4 403 default => 'cdn',
1e3baf05
DM		 404 },
405 bootdisk => {
406 optional => 1,
407 type => 'string', format => 'pve-qm-bootdisk',
408 description => "Enable booting from specified disk.",
03e480fc 409 pattern => '(ide|sata|scsi|virtio)\d+',
1e3baf05
DM		 410 },
411 smp => {
412 optional => 1,
413 type => 'integer',
414 description => "The number of CPUs. Please use option -sockets instead.",
415 minimum => 1,
416 default => 1,
417 },
418 sockets => {
419 optional => 1,
420 type => 'integer',
421 description => "The number of CPU sockets.",
422 minimum => 1,
423 default => 1,
424 },
425 cores => {
426 optional => 1,
427 type => 'integer',
428 description => "The number of cores per socket.",
429 minimum => 1,
430 default => 1,
431 },
8a010eae
AD		 432 numa => {
433 optional => 1,
434 type => 'boolean',
1917695c 435 description => "Enable/disable NUMA.",
8a010eae
AD		 436 default => 0,
437 },
7023f3ea
AD		 438 hugepages => {
439 optional => 1,
440 type => 'string',
441 description => "Enable/disable hugepages memory.",
442 enum => [qw(any 2 1024)],
443 },
f36e9894
SR		 444 keephugepages => {
445 optional => 1,
446 type => 'boolean',
447 default => 0,
448 description => "Use together with hugepages. If enabled, hugepages will"
449 . " not be deleted after VM shutdown and can be used for"
450 . " subsequent starts.",
451 },
de9d1e55 452 vcpus => {
3bd18e48
AD		 453 optional => 1,
454 type => 'integer',
de9d1e55 455 description => "Number of hotplugged vcpus.",
3bd18e48 456 minimum => 1,
de9d1e55 457 default => 0,
3bd18e48 458 },
1e3baf05
DM		 459 acpi => {
460 optional => 1,
461 type => 'boolean',
462 description => "Enable/disable ACPI.",
463 default => 1,
464 },
bc84dcca 465 agent => {
ab6a046f 466 optional => 1,
9d66b397
SI		 467 description => "Enable/disable Qemu GuestAgent and its properties.",
468 type => 'string',
469 format => $agent_fmt,
ab6a046f 470 },
1e3baf05
DM		 471 kvm => {
472 optional => 1,
473 type => 'boolean',
474 description => "Enable/disable KVM hardware virtualization.",
475 default => 1,
476 },
477 tdf => {
478 optional => 1,
479 type => 'boolean',
8c559505
DM		 480 description => "Enable/disable time drift fix.",
481 default => 0,
1e3baf05 482 },
19672434 483 localtime => {
1e3baf05
DM		 484 optional => 1,
485 type => 'boolean',
486 description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
487 },
488 freeze => {
489 optional => 1,
490 type => 'boolean',
491 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
492 },
493 vga => {
494 optional => 1,
55655ebc
DC		 495 type => 'string', format => $vga_fmt,
496 description => "Configure the VGA hardware.",
497 verbose_description => "Configure the VGA Hardware. If you want to use ".
498 "high resolution modes (>= 1280x1024x16) you may need to increase " .
499 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
500 "is 'std' for all OS types besides some Windows versions (XP and " .
501 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
502 "display server. For win* OS you can select how many independent " .
503 "displays you want, Linux guests can add displays them self.\n".
504 "You can also run without any graphic card, using a serial device as terminal.",
1e3baf05 505 },
0ea9541d
DM		 506 watchdog => {
507 optional => 1,
508 type => 'string', format => 'pve-qm-watchdog',
52261945
DM		 509 description => "Create a virtual hardware watchdog device.",
510 verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
1917695c
TL		 511 " (by a guest action), the watchdog must be periodically polled " .
512 "by an agent inside the guest or else the watchdog will reset " .
513 "the guest (or execute the respective action specified)",
0ea9541d 514 },
1e3baf05
DM		 515 startdate => {
516 optional => 1,
19672434 517 type => 'string',
1e3baf05
DM		 518 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
519 description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
520 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
521 default => 'now',
522 },
43574f73 523 startup => get_standard_option('pve-startup-order'),
68eda3ab
AD		 524 template => {
525 optional => 1,
526 type => 'boolean',
527 description => "Enable/disable Template.",
528 default => 0,
529 },
1e3baf05
DM		 530 args => {
531 optional => 1,
532 type => 'string',
52261945
DM		 533 description => "Arbitrary arguments passed to kvm.",
534 verbose_description => <<EODESCR,
c7a8aad6 535Arbitrary arguments passed to kvm, for example:
1e3baf05
DM		 536
537args: -no-reboot -no-hpet
c7a8aad6
FG		 538
539NOTE: this option is for experts only.
1e3baf05
DM		 540EODESCR
541 },
542 tablet => {
543 optional => 1,
544 type => 'boolean',
545 default => 1,
52261945
DM		 546 description => "Enable/disable the USB tablet device.",
547 verbose_description => "Enable/disable the USB tablet device. This device is " .
1917695c
TL		 548 "usually needed to allow absolute mouse positioning with VNC. " .
549 "Else the mouse runs out of sync with normal VNC clients. " .
550 "If you're running lots of console-only guests on one host, " .
551 "you may consider disabling this to save some context switches. " .
552 "This is turned off by default if you use spice (-vga=qxl).",
1e3baf05
DM		 553 },
554 migrate_speed => {
555 optional => 1,
556 type => 'integer',
557 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
558 minimum => 0,
559 default => 0,
560 },
561 migrate_downtime => {
562 optional => 1,
04432191 563 type => 'number',
1e3baf05
DM		 564 description => "Set maximum tolerated downtime (in seconds) for migrations.",
565 minimum => 0,
04432191 566 default => 0.1,
1e3baf05
DM		 567 },
568 cdrom => {
569 optional => 1,
b799312f 570 type => 'string', format => 'pve-qm-ide',
8485b9ba 571 typetext => '<volume>',
1e3baf05
DM		 572 description => "This is an alias for option -ide2",
573 },
574 cpu => {
575 optional => 1,
576 description => "Emulated CPU type.",
577 type => 'string',
5d008ad3 578 format => 'pve-vm-cpu-conf',
1e3baf05 579 },
b7ba6b79
DM		 580 parent => get_standard_option('pve-snapshot-name', {
581 optional => 1,
582 description => "Parent snapshot name. This is used internally, and should not be modified.",
583 }),
982c7f12
DM		 584 snaptime => {
585 optional => 1,
586 description => "Timestamp for snapshots.",
587 type => 'integer',
588 minimum => 0,
589 },
18bfb361
DM		 590 vmstate => {
591 optional => 1,
592 type => 'string', format => 'pve-volume-id',
593 description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
594 },
253624c7
FG		 595 vmstatestorage => get_standard_option('pve-storage-id', {
596 description => "Default storage for VM state volumes/files.",
597 optional => 1,
598 }),
c6737ef1 599 runningmachine => get_standard_option('pve-qemu-machine', {
ea1c2110 600 description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
c6737ef1 601 }),
ea1c2110
SR		 602 runningcpu => {
603 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
604 optional => 1,
605 type => 'string',
606 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
607 format_description => 'QEMU -cpu parameter'
608 },
c6737ef1 609 machine => get_standard_option('pve-qemu-machine'),
d731ecbe
WB		 610 arch => {
611 description => "Virtual processor architecture. Defaults to the host.",
612 optional => 1,
613 type => 'string',
614 enum => [qw(x86_64 aarch64)],
615 },
2796e7d5
DM		 616 smbios1 => {
617 description => "Specify SMBIOS type 1 fields.",
618 type => 'string', format => 'pve-qm-smbios1',
5d004b00 619 maxLength => 512,
2796e7d5
DM		 620 optional => 1,
621 },
cb0e4540
AG		 622 protection => {
623 optional => 1,
624 type => 'boolean',
52261945 625 description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
cb0e4540
AG		 626 default => 0,
627 },
3edb45e7 628 bios => {
a783c78e 629 optional => 1,
3edb45e7
DM		 630 type => 'string',
631 enum => [ qw(seabios ovmf) ],
632 description => "Select BIOS implementation.",
633 default => 'seabios',
a783c78e 634 },
6ee499ff
DC		 635 vmgenid => {
636 type => 'string',
637 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
638 format_description => 'UUID',
f7ed64e7
TL		 639 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
640 verbose_description => "The VM generation ID (vmgenid) device exposes a".
641 " 128-bit integer value identifier to the guest OS. This allows to".
642 " notify the guest operating system when the virtual machine is".
643 " executed with a different configuration (e.g. snapshot execution".
644 " or creation from a template). The guest operating system notices".
645 " the change, and is then able to react as appropriate by marking".
646 " its copies of distributed databases as dirty, re-initializing its".
647 " random number generator, etc.\n".
648 "Note that auto-creation only works when done throug API/CLI create".
649 " or update methods, but not when manually editing the config file.",
650 default => "1 (autogenerated)",
6ee499ff
DC		 651 optional => 1,
652 },
9e784b11
DC		 653 hookscript => {
654 type => 'string',
655 format => 'pve-volume-id',
656 optional => 1,
657 description => "Script that will be executed during various steps in the vms lifetime.",
658 },
6dbcb073
DC		 659 ivshmem => {
660 type => 'string',
661 format => $ivshmem_fmt,
662 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
663 optional => 1,
2e7b5925
AL		 664 },
665 audio0 => {
666 type => 'string',
1448547f 667 format => $audio_fmt,
194b65f1 668 description => "Configure a audio device, useful in combination with QXL/Spice.",
2e7b5925
AL		 669 optional => 1
670 },
c4df18db
AL		 671 spice_enhancements => {
672 type => 'string',
673 format => $spice_enhancements_fmt,
674 description => "Configure additional enhancements for SPICE.",
675 optional => 1
676 },
b8e7068a
DC		 677 tags => {
678 type => 'string', format => 'pve-tag-list',
679 description => 'Tags of the VM. This is only meta information.',
680 optional => 1,
681 },
2cf61f33
SR		 682 rng0 => {
683 type => 'string',
684 format => $rng_fmt,
685 description => "Configure a VirtIO-based Random Number Generator.",
686 optional => 1,
687 },
9ed7a77c
WB		 688};
689
cb702ebe
DL		 690my $cicustom_fmt = {
691 meta => {
692 type => 'string',
693 optional => 1,
694 description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
695 format => 'pve-volume-id',
696 format_description => 'volume',
697 },
698 network => {
699 type => 'string',
700 optional => 1,
701 description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
702 format => 'pve-volume-id',
703 format_description => 'volume',
704 },
705 user => {
706 type => 'string',
707 optional => 1,
708 description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
709 format => 'pve-volume-id',
710 format_description => 'volume',
711 },
712};
713PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
714
9ed7a77c 715my $confdesc_cloudinit = {
41cd94a0
WB		 716 citype => {
717 optional => 1,
718 type => 'string',
498cdc36 719 description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
41cd94a0
WB		 720 enum => ['configdrive2', 'nocloud'],
721 },
7b42f951
WB		 722 ciuser => {
723 optional => 1,
724 type => 'string',
725 description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
726 },
727 cipassword => {
728 optional => 1,
729 type => 'string',
1d1c4e1c 730 description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
7b42f951 731 },
cb702ebe
DL		 732 cicustom => {
733 optional => 1,
734 type => 'string',
735 description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
736 format => 'pve-qm-cicustom',
737 },
0c9a7596
AD		 738 searchdomain => {
739 optional => 1,
740 type => 'string',
741 description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
742 },
743 nameserver => {
744 optional => 1,
745 type => 'string', format => 'address-list',
746 description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
747 },
748 sshkeys => {
749 optional => 1,
750 type => 'string',
751 format => 'urlencoded',
1d1c4e1c 752 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
0c9a7596 753 },
1e3baf05
DM		 754};
755
756# what about other qemu settings ?
757#cpu => 'string',
758#machine => 'string',
759#fda => 'file',
760#fdb => 'file',
761#mtdblock => 'file',
762#sd => 'file',
763#pflash => 'file',
764#snapshot => 'bool',
765#bootp => 'file',
766##tftp => 'dir',
767##smb => 'dir',
768#kernel => 'file',
769#append => 'string',
770#initrd => 'file',
771##soundhw => 'string',
772
773while (my ($k, $v) = each %$confdesc) {
774 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
775}
776
1e3baf05 777my $MAX_USB_DEVICES = 5;
5bdcf937 778my $MAX_NETS = 32;
bae179aa 779my $MAX_SERIAL_PORTS = 4;
1989a89c 780my $MAX_PARALLEL_PORTS = 3;
2ed5d572
AD		 781my $MAX_NUMA = 8;
782
ffc0d8c7
WB		 783my $numa_fmt = {
784 cpus => {
785 type => "string",
786 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
52261945 787 description => "CPUs accessing this NUMA node.",
ffc0d8c7
WB		 788 format_description => "id[-id];...",
789 },
790 memory => {
791 type => "number",
52261945 792 description => "Amount of memory this NUMA node provides.",
ffc0d8c7
WB		 793 optional => 1,
794 },
795 hostnodes => {
796 type => "string",
797 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
52261945 798 description => "Host NUMA nodes to use.",
ffc0d8c7
WB		 799 format_description => "id[-id];...",
800 optional => 1,
801 },
802 policy => {
803 type => 'string',
804 enum => [qw(preferred bind interleave)],
52261945 805 description => "NUMA allocation policy.",
ffc0d8c7
WB		 806 optional => 1,
807 },
808};
809PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
2ed5d572
AD		 810my $numadesc = {
811 optional => 1,
ffc0d8c7 812 type => 'string', format => $numa_fmt,
52261945 813 description => "NUMA topology.",
2ed5d572
AD		 814};
815PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
816
817for (my $i = 0; $i < $MAX_NUMA; $i++) {
818 $confdesc->{"numa$i"} = $numadesc;
819}
1e3baf05
DM		 820
821my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
55034103
KT		 822 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
823 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
6b64503e 824my $nic_model_list_txt = join(' ', sort @$nic_model_list);
1e3baf05 825
52261945
DM		 826my $net_fmt_bridge_descr = <<__EOD__;
827Bridge to attach the network device to. The Proxmox VE standard bridge
828is called 'vmbr0'.
829
830If you do not specify a bridge, we create a kvm user (NATed) network
831device, which provides DHCP and DNS services. The following addresses
832are used:
833
834 10.0.2.2 Gateway
835 10.0.2.3 DNS Server
836 10.0.2.4 SMB Server
837
838The DHCP server assign addresses to the guest starting from 10.0.2.15.
839__EOD__
840
cd9c34d1 841my $net_fmt = {
399d96db 842 macaddr => get_standard_option('mac-addr', {
52261945 843 description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
399d96db 844 }),
7f694a71
DM		 845 model => {
846 type => 'string',
52261945 847 description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
7f694a71
DM		 848 enum => $nic_model_list,
849 default_key => 1,
850 },
851 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
cd9c34d1
WB		 852 bridge => {
853 type => 'string',
52261945 854 description => $net_fmt_bridge_descr,
cd9c34d1 855 format_description => 'bridge',
e1cfa02e 856 pattern => '[-_.\w\d]+',
cd9c34d1
WB		 857 optional => 1,
858 },
859 queues => {
860 type => 'integer',
861 minimum => 0, maximum => 16,
862 description => 'Number of packet queues to be used on the device.',
cd9c34d1
WB		 863 optional => 1,
864 },
865 rate => {
866 type => 'number',
867 minimum => 0,
52261945 868 description => "Rate limit in mbps (megabytes per second) as floating point number.",
cd9c34d1
WB		 869 optional => 1,
870 },
871 tag => {
872 type => 'integer',
9f41a659 873 minimum => 1, maximum => 4094,
cd9c34d1 874 description => 'VLAN tag to apply to packets on this interface.',
cd9c34d1
WB		 875 optional => 1,
876 },
877 trunks => {
878 type => 'string',
879 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
880 description => 'VLAN trunks to pass through this interface.',
7f694a71 881 format_description => 'vlanid[;vlanid...]',
cd9c34d1
WB		 882 optional => 1,
883 },
884 firewall => {
885 type => 'boolean',
886 description => 'Whether this interface should be protected by the firewall.',
cd9c34d1
WB		 887 optional => 1,
888 },
889 link_down => {
890 type => 'boolean',
52261945 891 description => 'Whether this interface should be disconnected (like pulling the plug).',
cd9c34d1
WB		 892 optional => 1,
893 },
61a14cde
AD		 894 mtu => {
895 type => 'integer',
896 minimum => 1, maximum => 65520,
0530177b 897 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
61a14cde
AD		 898 optional => 1,
899 },
cd9c34d1 900};
52261945 901
1e3baf05
DM		 902my $netdesc = {
903 optional => 1,
7f694a71 904 type => 'string', format => $net_fmt,
52261945 905 description => "Specify network devices.",
1e3baf05 906};
52261945 907
1e3baf05
DM		 908PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
909
0c9a7596
AD		 910my $ipconfig_fmt = {
911 ip => {
912 type => 'string',
913 format => 'pve-ipv4-config',
914 format_description => 'IPv4Format/CIDR',
915 description => 'IPv4 address in CIDR format.',
916 optional => 1,
917 default => 'dhcp',
918 },
919 gw => {
920 type => 'string',
921 format => 'ipv4',
922 format_description => 'GatewayIPv4',
923 description => 'Default gateway for IPv4 traffic.',
924 optional => 1,
925 requires => 'ip',
926 },
927 ip6 => {
928 type => 'string',
929 format => 'pve-ipv6-config',
930 format_description => 'IPv6Format/CIDR',
931 description => 'IPv6 address in CIDR format.',
932 optional => 1,
933 default => 'dhcp',
934 },
935 gw6 => {
936 type => 'string',
937 format => 'ipv6',
938 format_description => 'GatewayIPv6',
939 description => 'Default gateway for IPv6 traffic.',
940 optional => 1,
941 requires => 'ip6',
942 },
943};
944PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
945my $ipconfigdesc = {
946 optional => 1,
947 type => 'string', format => 'pve-qm-ipconfig',
948 description => <<'EODESCR',
949cloud-init: Specify IP addresses and gateways for the corresponding interface.
950
951IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
952
953The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
954For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
955
956If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
957EODESCR
958};
959PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
960
1e3baf05
DM		 961for (my $i = 0; $i < $MAX_NETS; $i++) {
962 $confdesc->{"net$i"} = $netdesc;
9ed7a77c
WB		 963 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
964}
965
966foreach my $key (keys %$confdesc_cloudinit) {
967 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1e3baf05
DM		 968}
969
ffa42b86
DC		 970PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
971sub verify_volume_id_or_qm_path {
822c8a07
WB		 972 my ($volid, $noerr) = @_;
973
ffa42b86
DC		 974 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
975 return $volid;
976 }
977
978 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
822c8a07
WB		 979 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
980 if ($@) {
981 return undef if $noerr;
982 die $@;
983 }
984 return $volid;
985}
986
ff6ffe20 987my $usb_fmt = {
a6b9aee4
DC		 988 host => {
989 default_key => 1,
990 type => 'string', format => 'pve-qm-usb-device',
991 format_description => 'HOSTUSBDEVICE|spice',
52261945
DM		 992 description => <<EODESCR,
993The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
994
995 'bus-port(.port)*' (decimal numbers) or
996 'vendor_id:product_id' (hexadeciaml numbers) or
997 'spice'
998
999You can use the 'lsusb -t' command to list existing usb devices.
1000
1001NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1002
1003The value 'spice' can be used to add a usb redirection devices for spice.
1004EODESCR
a6b9aee4
DC		 1005 },
1006 usb3 => {
1007 optional => 1,
1008 type => 'boolean',
97ef5356 1009 description => "Specifies whether if given host option is a USB3 device or port.",
52261945 1010 default => 0,
a6b9aee4
DC		 1011 },
1012};
1013
1e3baf05
DM		 1014my $usbdesc = {
1015 optional => 1,
ff6ffe20 1016 type => 'string', format => $usb_fmt,
52261945 1017 description => "Configure an USB device (n is 0 to 4).",
1e3baf05
DM		 1018};
1019PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1020
bae179aa
DA		 1021my $serialdesc = {
1022 optional => 1,
ca0cef26 1023 type => 'string',
1b0b51ed 1024 pattern => '(/dev/.+|socket)',
52261945
DM		 1025 description => "Create a serial device inside the VM (n is 0 to 3)",
1026 verbose_description => <<EODESCR,
52261945
DM		 1027Create a serial device inside the VM (n is 0 to 3), and pass through a
1028host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1029host side (use 'qm terminal' to open a terminal connection).
bae179aa 1030
8a61e0fd 1031NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
bae179aa 1032
52261945 1033CAUTION: Experimental! User reported problems with this option.
bae179aa
DA		 1034EODESCR
1035};
bae179aa 1036
1989a89c
DA		 1037my $paralleldesc= {
1038 optional => 1,
ca0cef26 1039 type => 'string',
9ecc8431 1040 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
52261945
DM		 1041 description => "Map host parallel devices (n is 0 to 2).",
1042 verbose_description => <<EODESCR,
19672434 1043Map host parallel devices (n is 0 to 2).
1989a89c 1044
8a61e0fd 1045NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1989a89c 1046
52261945 1047CAUTION: Experimental! User reported problems with this option.
1989a89c
DA		 1048EODESCR
1049};
1989a89c
DA		 1050
1051for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1052 $confdesc->{"parallel$i"} = $paralleldesc;
1053}
1054
bae179aa
DA		 1055for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1056 $confdesc->{"serial$i"} = $serialdesc;
1057}
1058
74c17b7a
SR		 1059for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1060 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
040b06b7 1061}
1e3baf05 1062
e0fd2b2f
FE		 1063for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1064 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
cdb0931f
DA		 1065}
1066
1e3baf05
DM		 1067for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1068 $confdesc->{"usb$i"} = $usbdesc;
1069}
1070
1e3baf05
DM		 1071my $kvm_api_version = 0;
1072
1073sub kvm_version {
1e3baf05
DM		 1074 return $kvm_api_version if $kvm_api_version;
1075
646f2df4
WB		 1076 open my $fh, '<', '/dev/kvm'
1077 or return undef;
1e3baf05 1078
646f2df4
WB		 1079 # 0xae00 => KVM_GET_API_VERSION
1080 $kvm_api_version = ioctl($fh, 0xae00, 0);
1e3baf05 1081
646f2df4 1082 return $kvm_api_version;
1e3baf05
DM		 1083}
1084
1476b99f
DC		 1085my $kvm_user_version = {};
1086my $kvm_mtime = {};
1e3baf05
DM		 1087
1088sub kvm_user_version {
1476b99f 1089 my ($binary) = @_;
1e3baf05 1090
1476b99f
DC		 1091 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1092 my $st = stat($binary);
1e3baf05 1093
1476b99f
DC		 1094 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1095 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1096 $cachedmtime == $st->mtime;
1097
1098 $kvm_user_version->{$binary} = 'unknown';
1099 $kvm_mtime->{$binary} = $st->mtime;
1e3baf05 1100
09b11429
TL		 1101 my $code = sub {
1102 my $line = shift;
1103 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1476b99f 1104 $kvm_user_version->{$binary} = $2;
09b11429
TL		 1105 }
1106 };
19672434 1107
1476b99f 1108 eval { run_command([$binary, '--version'], outfunc => $code); };
09b11429 1109 warn $@ if $@;
1e3baf05 1110
1476b99f 1111 return $kvm_user_version->{$binary};
1e3baf05
DM		 1112
1113}
1114
db70021b
TL		 1115sub kernel_has_vhost_net {
1116 return -c '/dev/vhost-net';
1117}
1e3baf05 1118
1e3baf05
DM		 1119sub option_exists {
1120 my $key = shift;
1121 return defined($confdesc->{$key});
19672434 1122}
1e3baf05 1123
1e3baf05 1124my $cdrom_path;
1e3baf05
DM		 1125sub get_cdrom_path {
1126
1127 return $cdrom_path if $cdrom_path;
1128
1129 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1130 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1131 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1132}
1133
1134sub get_iso_path {
1135 my ($storecfg, $vmid, $cdrom) = @_;
1136
1137 if ($cdrom eq 'cdrom') {
1138 return get_cdrom_path();
1139 } elsif ($cdrom eq 'none') {
1140 return '';
1141 } elsif ($cdrom =~ m|^/|) {
1142 return $cdrom;
1143 } else {
6b64503e 1144 return PVE::Storage::path($storecfg, $cdrom);
1e3baf05
DM		 1145 }
1146}
1147
1148# try to convert old style file names to volume IDs
1149sub filename_to_volume_id {
1150 my ($vmid, $file, $media) = @_;
1151
0c9a7596 1152 if (!($file eq 'none' || $file eq 'cdrom' ||
1e3baf05 1153 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
19672434 1154
1e3baf05 1155 return undef if $file =~ m|/|;
19672434 1156
1e3baf05
DM		 1157 if ($media && $media eq 'cdrom') {
1158 $file = "local:iso/$file";
1159 } else {
1160 $file = "local:$vmid/$file";
1161 }
1162 }
1163
1164 return $file;
1165}
1166
1167sub verify_media_type {
1168 my ($opt, $vtype, $media) = @_;
1169
1170 return if !$media;
1171
1172 my $etype;
1173 if ($media eq 'disk') {
a125592c 1174 $etype = 'images';
1e3baf05
DM		 1175 } elsif ($media eq 'cdrom') {
1176 $etype = 'iso';
1177 } else {
1178 die "internal error";
1179 }
1180
1181 return if ($vtype eq $etype);
19672434 1182
1e3baf05
DM		 1183 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1184}
1185
1186sub cleanup_drive_path {
1187 my ($opt, $storecfg, $drive) = @_;
1188
1189 # try to convert filesystem paths to volume IDs
1190
1191 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1192 ($drive->{file} !~ m|^/dev/.+|) &&
1193 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
19672434 1194 ($drive->{file} !~ m/^\d+$/)) {
1e3baf05
DM		 1195 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1196 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1197 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1198 verify_media_type($opt, $vtype, $drive->{media});
1199 $drive->{file} = $volid;
1200 }
1201
1202 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1203}
1204
b3c2bdd1
DM		 1205sub parse_hotplug_features {
1206 my ($data) = @_;
1207
1208 my $res = {};
1209
1210 return $res if $data eq '0';
a1b7d579 1211
b3c2bdd1
DM		 1212 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1213
45827685 1214 foreach my $feature (PVE::Tools::split_list($data)) {
b3c2bdd1
DM		 1215 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1216 $res->{$1} = 1;
1217 } else {
596a0a20 1218 die "invalid hotplug feature '$feature'\n";
b3c2bdd1
DM		 1219 }
1220 }
1221 return $res;
1222}
1223
1224PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1225sub pve_verify_hotplug_features {
1226 my ($value, $noerr) = @_;
1227
1228 return $value if parse_hotplug_features($value);
1229
1230 return undef if $noerr;
1231
1232 die "unable to parse hotplug option\n";
1233}
1234
28ef82d3
DM		 1235sub scsi_inquiry {
1236 my($fh, $noerr) = @_;
1237
1238 my $SG_IO = 0x2285;
1239 my $SG_GET_VERSION_NUM = 0x2282;
1240
1241 my $versionbuf = "\x00" x 8;
1242 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1243 if (!$ret) {
1244 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1245 return undef;
1246 }
97d62eb7 1247 my $version = unpack("I", $versionbuf);
28ef82d3
DM		 1248 if ($version < 30000) {
1249 die "scsi generic interface too old\n" if !$noerr;
1250 return undef;
1251 }
97d62eb7 1252
28ef82d3
DM		 1253 my $buf = "\x00" x 36;
1254 my $sensebuf = "\x00" x 8;
f334aa3e 1255 my $cmd = pack("C x3 C x1", 0x12, 36);
97d62eb7 1256
28ef82d3
DM		 1257 # see /usr/include/scsi/sg.h
1258 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1259
97d62eb7
DM		 1260 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1261 length($sensebuf), 0, length($buf), $buf,
28ef82d3
DM		 1262 $cmd, $sensebuf, 6000);
1263
1264 $ret = ioctl($fh, $SG_IO, $packet);
1265 if (!$ret) {
1266 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1267 return undef;
1268 }
97d62eb7 1269
28ef82d3
DM		 1270 my @res = unpack($sg_io_hdr_t, $packet);
1271 if ($res[17] || $res[18]) {
1272 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1273 return undef;
1274 }
1275
1276 my $res = {};
09984754 1277 (my $byte0, my $byte1, $res->{vendor},
28ef82d3
DM		 1278 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1279
09984754
DM		 1280 $res->{removable} = $byte1 & 128 ? 1 : 0;
1281 $res->{type} = $byte0 & 31;
1282
28ef82d3
DM		 1283 return $res;
1284}
1285
1286sub path_is_scsi {
1287 my ($path) = @_;
1288
1289 my $fh = IO::File->new("+<$path") || return undef;
1290 my $res = scsi_inquiry($fh, 1);
1291 close($fh);
1292
1293 return $res;
1294}
1295
db656e5f 1296sub print_tabletdevice_full {
d559309f 1297 my ($conf, $arch) = @_;
b467f79a 1298
3392d6ca 1299 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
db656e5f
DM		 1300
1301 # we use uhci for old VMs because tablet driver was buggy in older qemu
d559309f 1302 my $usbbus;
3392d6ca 1303 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
d559309f
WB		 1304 $usbbus = 'ehci';
1305 } else {
1306 $usbbus = 'uhci';
1307 }
b467f79a 1308
db656e5f
DM		 1309 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1310}
1311
d559309f
WB		 1312sub print_keyboarddevice_full {
1313 my ($conf, $arch, $machine) = @_;
1314
1315 return undef if $arch ne 'aarch64';
1316
1317 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1318}
1319
ca916ecc 1320sub print_drivedevice_full {
d559309f 1321 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
ca916ecc
DA		 1322
1323 my $device = '';
1324 my $maxdev = 0;
19672434 1325
ca916ecc 1326 if ($drive->{interface} eq 'virtio') {
d559309f 1327 my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
2ed36a41 1328 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
51f492cd 1329 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
2ed36a41 1330 } elsif ($drive->{interface} eq 'scsi') {
6731a4cf 1331
ee034f5c 1332 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
2ed36a41
DM		 1333 my $unit = $drive->{index} % $maxdev;
1334 my $devicetype = 'hd';
69bcf246
WB		 1335 my $path = '';
1336 if (drive_is_cdrom($drive)) {
1337 $devicetype = 'cd';
29b19529 1338 } else {
69bcf246
WB		 1339 if ($drive->{file} =~ m|^/|) {
1340 $path = $drive->{file};
1341 if (my $info = path_is_scsi($path)) {
8e3c33ab 1342 if ($info->{type} == 0 && $drive->{scsiblock}) {
69bcf246
WB		 1343 $devicetype = 'block';
1344 } elsif ($info->{type} == 1) { # tape
1345 $devicetype = 'generic';
1346 }
1347 }
1348 } else {
1349 $path = PVE::Storage::path($storecfg, $drive->{file});
1350 }
1351
a034e3d6 1352 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
9471e48b 1353 my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
a034e3d6 1354 if ($path =~ m/^iscsi\:\/\// &&
2ea5fb7e 1355 !min_version($version, 4, 1)) {
69bcf246
WB		 1356 $devicetype = 'generic';
1357 }
1358 }
1359
1360 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1361 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1362 } else {
1363 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1364 }
cdd20088 1365
6c875f9f
NC		 1366 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1367 $device .= ",rotation_rate=1";
1368 }
e741c516 1369 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
6c875f9f
NC		 1370
1371 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
e0fd2b2f 1372 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
2ed36a41
DM		 1373 my $controller = int($drive->{index} / $maxdev);
1374 my $unit = $drive->{index} % $maxdev;
1375 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1376
6c875f9f
NC		 1377 $device = "ide-$devicetype";
1378 if ($drive->{interface} eq 'ide') {
1379 $device .= ",bus=ide.$controller,unit=$unit";
1380 } else {
1381 $device .= ",bus=ahci$controller.$unit";
1382 }
1383 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1384
1385 if ($devicetype eq 'hd') {
1386 if (my $model = $drive->{model}) {
1387 $model = URI::Escape::uri_unescape($model);
1388 $device .= ",model=$model";
1389 }
1390 if ($drive->{ssd}) {
1391 $device .= ",rotation_rate=1";
1392 }
0f2812c2 1393 }
e741c516 1394 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
2ed36a41
DM		 1395 } elsif ($drive->{interface} eq 'usb') {
1396 die "implement me";
1397 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1398 } else {
1399 die "unsupported interface type";
ca916ecc
DA		 1400 }
1401
3b408e82
DM		 1402 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1403
a70e7e6c
TL		 1404 if (my $serial = $drive->{serial}) {
1405 $serial = URI::Escape::uri_unescape($serial);
1406 $device .= ",serial=$serial";
1407 }
1408
1409
ca916ecc
DA		 1410 return $device;
1411}
1412
15b21acc 1413sub get_initiator_name {
46f58b5f 1414 my $initiator;
15b21acc 1415
46f58b5f
DM		 1416 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1417 while (defined(my $line = <$fh>)) {
1418 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
15b21acc
MR		 1419 $initiator = $1;
1420 last;
1421 }
46f58b5f
DM		 1422 $fh->close();
1423
15b21acc
MR		 1424 return $initiator;
1425}
1426
776c5f50 1427sub print_drive_commandline_full {
1e3baf05
DM		 1428 my ($storecfg, $vmid, $drive) = @_;
1429
d81f0f09
DM		 1430 my $path;
1431 my $volid = $drive->{file};
1432 my $format;
370b05e7 1433
d81f0f09
DM		 1434 if (drive_is_cdrom($drive)) {
1435 $path = get_iso_path($storecfg, $vmid, $volid);
1436 } else {
1437 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1438 if ($storeid) {
1439 $path = PVE::Storage::path($storecfg, $volid);
1440 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1441 $format = qemu_img_format($scfg, $volname);
1442 } else {
1443 $path = $volid;
5b61bff2 1444 $format = "raw";
d81f0f09
DM		 1445 }
1446 }
1447
1e3baf05 1448 my $opts = '';
8a267708 1449 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1e3baf05 1450 foreach my $o (@qemu_drive_options) {
5fc74861 1451 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
19672434 1452 }
8a267708
DC		 1453
1454 # snapshot only accepts on|off
1455 if (defined($drive->{snapshot})) {
1456 my $v = $drive->{snapshot} ? 'on' : 'off';
1457 $opts .= ",snapshot=$v";
1458 }
1459
fb8e95a2
WB		 1460 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1461 my ($dir, $qmpname) = @$type;
1462 if (my $v = $drive->{"mbps$dir"}) {
1463 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1464 }
1465 if (my $v = $drive->{"mbps${dir}_max"}) {
1466 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1467 }
1468 if (my $v = $drive->{"bps${dir}_max_length"}) {
1469 $opts .= ",throttling.bps$qmpname-max-length=$v";
1470 }
1471 if (my $v = $drive->{"iops${dir}"}) {
1472 $opts .= ",throttling.iops$qmpname=$v";
1473 }
1474 if (my $v = $drive->{"iops${dir}_max"}) {
8aca1654 1475 $opts .= ",throttling.iops$qmpname-max=$v";
fb8e95a2
WB		 1476 }
1477 if (my $v = $drive->{"iops${dir}_max_length"}) {
8aca1654 1478 $opts .= ",throttling.iops$qmpname-max-length=$v";
fb8e95a2
WB		 1479 }
1480 }
1481
d81f0f09
DM		 1482 $opts .= ",format=$format" if $format && !$drive->{format};
1483
b2ee900e
WB		 1484 my $cache_direct = 0;
1485
1486 if (my $cache = $drive->{cache}) {
1487 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1488 } elsif (!drive_is_cdrom($drive)) {
1489 $opts .= ",cache=none";
1490 $cache_direct = 1;
1491 }
1492
1493 # aio native works only with O_DIRECT
1494 if (!$drive->{aio}) {
1495 if($cache_direct) {
1496 $opts .= ",aio=native";
1497 } else {
1498 $opts .= ",aio=threads";
1499 }
1500 }
11490cf2 1501
6e47c3b4
WB		 1502 if (!drive_is_cdrom($drive)) {
1503 my $detectzeroes;
7d4e30f3 1504 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
6e47c3b4
WB		 1505 $detectzeroes = 'off';
1506 } elsif ($drive->{discard}) {
1507 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1508 } else {
1509 # This used to be our default with discard not being specified:
1510 $detectzeroes = 'on';
1511 }
1512 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1513 }
f1e05305 1514
1e3baf05
DM		 1515 my $pathinfo = $path ? "file=$path," : '';
1516
3ebfcc86 1517 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1e3baf05
DM		 1518}
1519
cc4d6182 1520sub print_netdevice_full {
d559309f 1521 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
cc4d6182
DA		 1522
1523 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1524
1525 my $device = $net->{model};
1526 if ($net->{model} eq 'virtio') {
1527 $device = 'virtio-net-pci';
1528 };
1529
d559309f 1530 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
5e2068d2 1531 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
a9410357
AD		 1532 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1533 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1534 my $vectors = $net->{queues} * 2 + 2;
1535 $tmpstr .= ",vectors=$vectors,mq=on";
1536 }
cc4d6182 1537 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
ba9e1000 1538
0530177b
TL		 1539 if (my $mtu = $net->{mtu}) {
1540 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1541 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1542 if ($mtu == 1) {
1543 $mtu = $bridge_mtu;
1544 } elsif ($mtu < 576) {
1545 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1546 } elsif ($mtu > $bridge_mtu) {
1547 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1548 }
1549 $tmpstr .= ",host_mtu=$mtu";
1550 } else {
1551 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
61a14cde 1552 }
61a14cde
AD		 1553 }
1554
ba9e1000
DM		 1555 if ($use_old_bios_files) {
1556 my $romfile;
1557 if ($device eq 'virtio-net-pci') {
1558 $romfile = 'pxe-virtio.rom';
1559 } elsif ($device eq 'e1000') {
1560 $romfile = 'pxe-e1000.rom';
1561 } elsif ($device eq 'ne2k') {
1562 $romfile = 'pxe-ne2k_pci.rom';
1563 } elsif ($device eq 'pcnet') {
1564 $romfile = 'pxe-pcnet.rom';
1565 } elsif ($device eq 'rtl8139') {
1566 $romfile = 'pxe-rtl8139.rom';
1567 }
1568 $tmpstr .= ",romfile=$romfile" if $romfile;
1569 }
1570
cc4d6182
DA		 1571 return $tmpstr;
1572}
1573
1574sub print_netdev_full {
d559309f 1575 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
cc4d6182
DA		 1576
1577 my $i = '';
1578 if ($netid =~ m/^net(\d+)$/) {
1579 $i = int($1);
1580 }
1581
1582 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1583
1584 my $ifname = "tap${vmid}i$i";
1585
1586 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1587 die "interface name '$ifname' is too long (max 15 character)\n"
1588 if length($ifname) >= 16;
1589
1590 my $vhostparam = '';
6f0cb675 1591 if (is_native($arch)) {
db70021b 1592 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
6f0cb675 1593 }
cc4d6182
DA		 1594
1595 my $vmname = $conf->{name} || "vm$vmid";
1596
a9410357 1597 my $netdev = "";
208ba94e 1598 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
a9410357 1599
cc4d6182 1600 if ($net->{bridge}) {
208ba94e 1601 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
cc4d6182 1602 } else {
a9410357 1603 $netdev = "type=user,id=$netid,hostname=$vmname";
cc4d6182 1604 }
a9410357
AD		 1605
1606 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1607
1608 return $netdev;
cc4d6182 1609}
1e3baf05 1610
55655ebc
DC		 1611my $vga_map = {
1612 'cirrus' => 'cirrus-vga',
1613 'std' => 'VGA',
1614 'vmware' => 'vmware-svga',
1615 'virtio' => 'virtio-vga',
1616};
1617
1618sub print_vga_device {
2ea5fb7e 1619 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
55655ebc
DC		 1620
1621 my $type = $vga_map->{$vga->{type}};
86c9fafe 1622 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
d559309f
WB		 1623 $type = 'virtio-gpu';
1624 }
55655ebc 1625 my $vgamem_mb = $vga->{memory};
6021c7a5
AL		 1626
1627 my $max_outputs = '';
55655ebc
DC		 1628 if ($qxlnum) {
1629 $type = $id ? 'qxl' : 'qxl-vga';
6021c7a5 1630
c5a4c92c 1631 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
6021c7a5 1632 # set max outputs so linux can have up to 4 qxl displays with one device
2ea5fb7e 1633 if (min_version($machine_version, 4, 1)) {
9e8976ea
TL		 1634 $max_outputs = ",max_outputs=4";
1635 }
6021c7a5 1636 }
55655ebc 1637 }
6021c7a5 1638
55655ebc
DC		 1639 die "no devicetype for $vga->{type}\n" if !$type;
1640
1641 my $memory = "";
1642 if ($vgamem_mb) {
1643 if ($vga->{type} eq 'virtio') {
1644 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1645 $memory = ",max_hostmem=$bytes";
1646 } elsif ($qxlnum) {
1647 # from https://www.spice-space.org/multiple-monitors.html
1648 $memory = ",vgamem_mb=$vga->{memory}";
1649 my $ram = $vgamem_mb * 4;
1650 my $vram = $vgamem_mb * 2;
1651 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1652 } else {
1653 $memory = ",vgamem_mb=$vga->{memory}";
1654 }
1655 } elsif ($qxlnum && $id) {
1656 $memory = ",ram_size=67108864,vram_size=33554432";
1657 }
1658
789fe8e8
AL		 1659 my $edidoff = "";
1660 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
b5d32c6b 1661 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
789fe8e8
AL		 1662 }
1663
3392d6ca 1664 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
55655ebc
DC		 1665 my $vgaid = "vga" . ($id // '');
1666 my $pciaddr;
daadd5a4 1667
55655ebc 1668 if ($q35 && $vgaid eq 'vga') {
daadd5a4 1669 # the first display uses pcie.0 bus on q35 machines
d559309f 1670 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
55655ebc 1671 } else {
d559309f 1672 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
55655ebc
DC		 1673 }
1674
789fe8e8 1675 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
55655ebc
DC		 1676}
1677
ffc0d8c7
WB		 1678sub parse_number_sets {
1679 my ($set) = @_;
1680 my $res = [];
1681 foreach my $part (split(/;/, $set)) {
1682 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1683 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1684 push @$res, [ $1, $2 ];
2ed5d572 1685 } else {
ffc0d8c7 1686 die "invalid range: $part\n";
2ed5d572
AD		 1687 }
1688 }
ffc0d8c7
WB		 1689 return $res;
1690}
2ed5d572 1691
ffc0d8c7
WB		 1692sub parse_numa {
1693 my ($data) = @_;
1694
1695 my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1696 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1697 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2ed5d572
AD		 1698 return $res;
1699}
1700
1e3baf05
DM		 1701# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1702sub parse_net {
1703 my ($data) = @_;
1704
cd9c34d1
WB		 1705 my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1706 if ($@) {
1707 warn $@;
1708 return undef;
1e3baf05 1709 }
b5b99790
WB		 1710 if (!defined($res->{macaddr})) {
1711 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1712 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1713 }
0c9a7596
AD		 1714 return $res;
1715}
1716
1717# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1718sub parse_ipconfig {
1719 my ($data) = @_;
1720
1721 my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1722 if ($@) {
1723 warn $@;
1724 return undef;
1725 }
1726
1727 if ($res->{gw} && !$res->{ip}) {
1728 warn 'gateway specified without specifying an IP address';
1729 return undef;
1730 }
1731 if ($res->{gw6} && !$res->{ip6}) {
1732 warn 'IPv6 gateway specified without specifying an IPv6 address';
1733 return undef;
1734 }
1735 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1736 warn 'gateway specified together with DHCP';
1737 return undef;
1738 }
1739 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1740 # gw6 + auto/dhcp
1741 warn "IPv6 gateway specified together with $res->{ip6} address";
1742 return undef;
1743 }
1744
1745 if (!$res->{ip} && !$res->{ip6}) {
1746 return { ip => 'dhcp', ip6 => 'dhcp' };
1747 }
1748
1e3baf05
DM		 1749 return $res;
1750}
1751
1752sub print_net {
1753 my $net = shift;
1754
cd9c34d1 1755 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1e3baf05
DM		 1756}
1757
1758sub add_random_macs {
1759 my ($settings) = @_;
1760
1761 foreach my $opt (keys %$settings) {
1762 next if $opt !~ m/^net(\d+)$/;
1763 my $net = parse_net($settings->{$opt});
1764 next if !$net;
1765 $settings->{$opt} = print_net($net);
1766 }
1767}
1768
055d554d
DM		 1769sub vm_is_volid_owner {
1770 my ($storecfg, $vmid, $volid) = @_;
1771
1772 if ($volid !~ m|^/|) {
1773 my ($path, $owner);
1774 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1775 if ($owner && ($owner == $vmid)) {
1776 return 1;
1777 }
1778 }
1779
1780 return undef;
1781}
1782
055d554d
DM		 1783sub vmconfig_register_unused_drive {
1784 my ($storecfg, $vmid, $conf, $drive) = @_;
1785
2d9ddec5
WB		 1786 if (drive_is_cloudinit($drive)) {
1787 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1788 warn $@ if $@;
1789 } elsif (!drive_is_cdrom($drive)) {
055d554d
DM		 1790 my $volid = $drive->{file};
1791 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
8793d495 1792 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
055d554d
DM		 1793 }
1794 }
1795}
1796
1f30ac3a 1797# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
ff6ffe20 1798my $smbios1_fmt = {
bd27e851
WB		 1799 uuid => {
1800 type => 'string',
1801 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1802 format_description => 'UUID',
52261945 1803 description => "Set SMBIOS1 UUID.",
bd27e851
WB		 1804 optional => 1,
1805 },
1806 version => {
1807 type => 'string',
1f30ac3a
CE		 1808 pattern => '[A-Za-z0-9+\/]+={0,2}',
1809 format_description => 'Base64 encoded string',
52261945 1810 description => "Set SMBIOS1 version.",
bd27e851
WB		 1811 optional => 1,
1812 },
1813 serial => {
1814 type => 'string',
1f30ac3a
CE		 1815 pattern => '[A-Za-z0-9+\/]+={0,2}',
1816 format_description => 'Base64 encoded string',
52261945 1817 description => "Set SMBIOS1 serial number.",
bd27e851
WB		 1818 optional => 1,
1819 },
1820 manufacturer => {
1821 type => 'string',
1f30ac3a
CE		 1822 pattern => '[A-Za-z0-9+\/]+={0,2}',
1823 format_description => 'Base64 encoded string',
52261945 1824 description => "Set SMBIOS1 manufacturer.",
bd27e851
WB		 1825 optional => 1,
1826 },
1827 product => {
1828 type => 'string',
1f30ac3a
CE		 1829 pattern => '[A-Za-z0-9+\/]+={0,2}',
1830 format_description => 'Base64 encoded string',
52261945 1831 description => "Set SMBIOS1 product ID.",
bd27e851
WB		 1832 optional => 1,
1833 },
1834 sku => {
1835 type => 'string',
1f30ac3a
CE		 1836 pattern => '[A-Za-z0-9+\/]+={0,2}',
1837 format_description => 'Base64 encoded string',
52261945 1838 description => "Set SMBIOS1 SKU string.",
bd27e851
WB		 1839 optional => 1,
1840 },
1841 family => {
1842 type => 'string',
1f30ac3a
CE		 1843 pattern => '[A-Za-z0-9+\/]+={0,2}',
1844 format_description => 'Base64 encoded string',
52261945 1845 description => "Set SMBIOS1 family string.",
bd27e851
WB		 1846 optional => 1,
1847 },
1f30ac3a
CE		 1848 base64 => {
1849 type => 'boolean',
1850 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1851 optional => 1,
1852 },
2796e7d5
DM		 1853};
1854
2796e7d5
DM		 1855sub parse_smbios1 {
1856 my ($data) = @_;
1857
ff6ffe20 1858 my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
bd27e851 1859 warn $@ if $@;
2796e7d5
DM		 1860 return $res;
1861}
1862
cd11416f
DM		 1863sub print_smbios1 {
1864 my ($smbios1) = @_;
ff6ffe20 1865 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
cd11416f
DM		 1866}
1867
ff6ffe20 1868PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2796e7d5 1869
0ea9541d
DM		 1870sub parse_watchdog {
1871 my ($value) = @_;
1872
1873 return undef if !$value;
1874
ec3582b5
WB		 1875 my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1876 warn $@ if $@;
0ea9541d
DM		 1877 return $res;
1878}
1879
9d66b397
SI		 1880sub parse_guest_agent {
1881 my ($value) = @_;
1882
1883 return {} if !defined($value->{agent});
1884
1885 my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1886 warn $@ if $@;
1887
1888 # if the agent is disabled ignore the other potentially set properties
1889 return {} if !$res->{enabled};
1890 return $res;
1891}
1892
55655ebc
DC		 1893sub parse_vga {
1894 my ($value) = @_;
1895
1896 return {} if !$value;
1897 my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1898 warn $@ if $@;
1899 return $res;
1900}
1901
2cf61f33
SR		 1902sub parse_rng {
1903 my ($value) = @_;
1904
1905 return undef if !$value;
1906
1907 my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1908 warn $@ if $@;
1909 return $res;
1910}
1911
1e3baf05
DM		 1912PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1913sub verify_usb_device {
1914 my ($value, $noerr) = @_;
1915
1916 return $value if parse_usb_device($value);
1917
1918 return undef if $noerr;
19672434 1919
1e3baf05
DM		 1920 die "unable to parse usb device\n";
1921}
1922
1e3baf05
DM		 1923# add JSON properties for create and set function
1924sub json_config_properties {
1925 my $prop = shift;
1926
1927 foreach my $opt (keys %$confdesc) {
ea1c2110
SR		 1928 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1929 $opt eq 'runningmachine' || $opt eq 'runningcpu';
1e3baf05
DM		 1930 $prop->{$opt} = $confdesc->{$opt};
1931 }
1932
1933 return $prop;
1934}
1935
d41121fd
DM		 1936# return copy of $confdesc_cloudinit to generate documentation
1937sub cloudinit_config_properties {
1938
1939 return dclone($confdesc_cloudinit);
1940}
1941
1e3baf05
DM		 1942sub check_type {
1943 my ($key, $value) = @_;
1944
1945 die "unknown setting '$key'\n" if !$confdesc->{$key};
1946
1947 my $type = $confdesc->{$key}->{type};
1948
6b64503e 1949 if (!defined($value)) {
1e3baf05
DM		 1950 die "got undefined value\n";
1951 }
1952
1953 if ($value =~ m/[\n\r]/) {
1954 die "property contains a line feed\n";
1955 }
1956
1957 if ($type eq 'boolean') {
19672434
DM		 1958 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1959 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1960 die "type check ('boolean') failed - got '$value'\n";
1e3baf05
DM		 1961 } elsif ($type eq 'integer') {
1962 return int($1) if $value =~ m/^(\d+)$/;
1963 die "type check ('integer') failed - got '$value'\n";
04432191
AD		 1964 } elsif ($type eq 'number') {
1965 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1966 die "type check ('number') failed - got '$value'\n";
1e3baf05
DM		 1967 } elsif ($type eq 'string') {
1968 if (my $fmt = $confdesc->{$key}->{format}) {
1e3baf05 1969 PVE::JSONSchema::check_format($fmt, $value);
19672434
DM		 1970 return $value;
1971 }
1e3baf05 1972 $value =~ s/^\"(.*)\"$/$1/;
19672434 1973 return $value;
1e3baf05
DM		 1974 } else {
1975 die "internal error"
1976 }
1977}
1978
1e3baf05 1979sub destroy_vm {
b04ea584 1980 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1e3baf05 1981
ffda963f 1982 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 1983
ffda963f 1984 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 1985
5e67a2d2
DC		 1986 if ($conf->{template}) {
1987 # check if any base image is still used by a linked clone
912792e2 1988 PVE::QemuConfig->foreach_volume($conf, sub {
5e67a2d2 1989 my ($ds, $drive) = @_;
5e67a2d2
DC		 1990 return if drive_is_cdrom($drive);
1991
1992 my $volid = $drive->{file};
5e67a2d2
DC		 1993 return if !$volid || $volid =~ m|^/|;
1994
1995 die "base volume '$volid' is still in use by linked cloned\n"
1996 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
1997
1998 });
1999 }
2000
19672434 2001 # only remove disks owned by this VM
912792e2 2002 PVE::QemuConfig->foreach_volume($conf, sub {
1e3baf05 2003 my ($ds, $drive) = @_;
9c52f5ed 2004 return if drive_is_cdrom($drive, 1);
1e3baf05
DM		 2005
2006 my $volid = $drive->{file};
ff1a2432 2007 return if !$volid || $volid =~ m|^/|;
1e3baf05 2008
6b64503e 2009 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
ff1a2432 2010 return if !$path || !$owner || ($owner != $vmid);
1e3baf05 2011
a2f50f01 2012 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
31b52247 2013 warn "Could not remove disk '$volid', check manually: $@" if $@;
1e3baf05 2014 });
19672434 2015
1e3baf05 2016 # also remove unused disk
a2f50f01
TL		 2017 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2018 PVE::Storage::foreach_volid($vmdisks, sub {
2019 my ($volid, $sid, $volname, $d) = @_;
2020 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
1e3baf05 2021 warn $@ if $@;
a2f50f01 2022 });
dfda979e 2023
b04ea584 2024 if (defined $replacement_conf) {
8baf8bc7 2025 PVE::QemuConfig->write_config($vmid, $replacement_conf);
dfda979e
DJ		 2026 } else {
2027 PVE::QemuConfig->destroy_config($vmid);
2028 }
1e3baf05
DM		 2029}
2030
1e3baf05
DM		 2031sub parse_vm_config {
2032 my ($filename, $raw) = @_;
2033
2034 return undef if !defined($raw);
2035
554ac7e7 2036 my $res = {
fc1ddcdc 2037 digest => Digest::SHA::sha1_hex($raw),
0d18dcfc 2038 snapshots => {},
0d732d16 2039 pending => {},
554ac7e7 2040 };
1e3baf05 2041
19672434 2042 $filename =~ m|/qemu-server/(\d+)\.conf$|
1e3baf05
DM		 2043 || die "got strange filename '$filename'";
2044
2045 my $vmid = $1;
2046
0d18dcfc 2047 my $conf = $res;
b0ec896e 2048 my $descr;
e297c490 2049 my $section = '';
0581fe4f 2050
0d18dcfc
DM		 2051 my @lines = split(/\n/, $raw);
2052 foreach my $line (@lines) {
1e3baf05 2053 next if $line =~ m/^\s*$/;
be190583 2054
eab09f4e 2055 if ($line =~ m/^\[PENDING\]\s*$/i) {
e297c490 2056 $section = 'pending';
b0ec896e
DM		 2057 if (defined($descr)) {
2058 $descr =~ s/\s+$//;
2059 $conf->{description} = $descr;
2060 }
2061 $descr = undef;
e297c490 2062 $conf = $res->{$section} = {};
eab09f4e
AD		 2063 next;
2064
0d732d16 2065 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
e297c490 2066 $section = $1;
b0ec896e
DM		 2067 if (defined($descr)) {
2068 $descr =~ s/\s+$//;
2069 $conf->{description} = $descr;
2070 }
2071 $descr = undef;
e297c490 2072 $conf = $res->{snapshots}->{$section} = {};
0d18dcfc
DM		 2073 next;
2074 }
1e3baf05 2075
0581fe4f 2076 if ($line =~ m/^\#(.*)\s*$/) {
b0ec896e 2077 $descr = '' if !defined($descr);
0581fe4f
DM		 2078 $descr .= PVE::Tools::decode_text($1) . "\n";
2079 next;
2080 }
2081
1e3baf05 2082 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
b0ec896e 2083 $descr = '' if !defined($descr);
0581fe4f 2084 $descr .= PVE::Tools::decode_text($2);
0d18dcfc
DM		 2085 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2086 $conf->{snapstate} = $1;
1e3baf05
DM		 2087 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2088 my $key = $1;
2089 my $value = $2;
0d18dcfc 2090 $conf->{$key} = $value;
ef824322 2091 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
e297c490 2092 my $value = $1;
ef824322
DM		 2093 if ($section eq 'pending') {
2094 $conf->{delete} = $value; # we parse this later
2095 } else {
2096 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
eab09f4e 2097 }
15cf7698 2098 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
1e3baf05
DM		 2099 my $key = $1;
2100 my $value = $2;
2101 eval { $value = check_type($key, $value); };
2102 if ($@) {
2103 warn "vm $vmid - unable to parse value of '$key' - $@";
2104 } else {
b799312f 2105 $key = 'ide2' if $key eq 'cdrom';
1e3baf05 2106 my $fmt = $confdesc->{$key}->{format};
b799312f 2107 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
1e3baf05
DM		 2108 my $v = parse_drive($key, $value);
2109 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2110 $v->{file} = $volid;
71c58bb7 2111 $value = print_drive($v);
1e3baf05
DM		 2112 } else {
2113 warn "vm $vmid - unable to parse value of '$key'\n";
2114 next;
2115 }
2116 }
2117
b799312f 2118 $conf->{$key} = $value;
1e3baf05
DM		 2119 }
2120 }
2121 }
2122
b0ec896e
DM		 2123 if (defined($descr)) {
2124 $descr =~ s/\s+$//;
2125 $conf->{description} = $descr;
2126 }
0d18dcfc 2127 delete $res->{snapstate}; # just to be sure
1e3baf05
DM		 2128
2129 return $res;
2130}
2131
1858638f
DM		 2132sub write_vm_config {
2133 my ($filename, $conf) = @_;
1e3baf05 2134
0d18dcfc
DM		 2135 delete $conf->{snapstate}; # just to be sure
2136
1858638f
DM		 2137 if ($conf->{cdrom}) {
2138 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2139 $conf->{ide2} = $conf->{cdrom};
2140 delete $conf->{cdrom};
2141 }
1e3baf05
DM		 2142
2143 # we do not use 'smp' any longer
1858638f
DM		 2144 if ($conf->{sockets}) {
2145 delete $conf->{smp};
2146 } elsif ($conf->{smp}) {
2147 $conf->{sockets} = $conf->{smp};
2148 delete $conf->{cores};
2149 delete $conf->{smp};
1e3baf05
DM		 2150 }
2151
ee2f90b1 2152 my $used_volids = {};
0d18dcfc 2153
ee2f90b1 2154 my $cleanup_config = sub {
ef824322 2155 my ($cref, $pending, $snapname) = @_;
1858638f 2156
ee2f90b1
DM		 2157 foreach my $key (keys %$cref) {
2158 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
ef824322 2159 $key eq 'snapstate' || $key eq 'pending';
ee2f90b1 2160 my $value = $cref->{$key};
ef824322
DM		 2161 if ($key eq 'delete') {
2162 die "propertry 'delete' is only allowed in [PENDING]\n"
2163 if !$pending;
2164 # fixme: check syntax?
2165 next;
2166 }
ee2f90b1
DM		 2167 eval { $value = check_type($key, $value); };
2168 die "unable to parse value of '$key' - $@" if $@;
1858638f 2169
ee2f90b1
DM		 2170 $cref->{$key} = $value;
2171
74479ee9 2172 if (!$snapname && is_valid_drivename($key)) {
ed221350 2173 my $drive = parse_drive($key, $value);
ee2f90b1
DM		 2174 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2175 }
1e3baf05 2176 }
ee2f90b1
DM		 2177 };
2178
2179 &$cleanup_config($conf);
ef824322
DM		 2180
2181 &$cleanup_config($conf->{pending}, 1);
2182
ee2f90b1 2183 foreach my $snapname (keys %{$conf->{snapshots}}) {
15c6e277 2184 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
ef824322 2185 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
1e3baf05
DM		 2186 }
2187
1858638f
DM		 2188 # remove 'unusedX' settings if we re-add a volume
2189 foreach my $key (keys %$conf) {
2190 my $value = $conf->{$key};
ee2f90b1 2191 if ($key =~ m/^unused/ && $used_volids->{$value}) {
1858638f 2192 delete $conf->{$key};
1e3baf05 2193 }
1858638f 2194 }
be190583 2195
0d18dcfc 2196 my $generate_raw_config = sub {
b0ec896e 2197 my ($conf, $pending) = @_;
0581fe4f 2198
0d18dcfc
DM		 2199 my $raw = '';
2200
2201 # add description as comment to top of file
b0ec896e
DM		 2202 if (defined(my $descr = $conf->{description})) {
2203 if ($descr) {
2204 foreach my $cl (split(/\n/, $descr)) {
2205 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2206 }
2207 } else {
2208 $raw .= "#\n" if $pending;
2209 }
0d18dcfc
DM		 2210 }
2211
2212 foreach my $key (sort keys %$conf) {
ef824322 2213 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
0d18dcfc
DM		 2214 $raw .= "$key: $conf->{$key}\n";
2215 }
2216 return $raw;
2217 };
0581fe4f 2218
0d18dcfc 2219 my $raw = &$generate_raw_config($conf);
ef824322
DM		 2220
2221 if (scalar(keys %{$conf->{pending}})){
2222 $raw .= "\n[PENDING]\n";
b0ec896e 2223 $raw .= &$generate_raw_config($conf->{pending}, 1);
ef824322
DM		 2224 }
2225
0d18dcfc
DM		 2226 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2227 $raw .= "\n[$snapname]\n";
2228 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
1858638f 2229 }
1e3baf05 2230
1858638f
DM		 2231 return $raw;
2232}
1e3baf05 2233
19672434 2234sub load_defaults {
1e3baf05
DM		 2235
2236 my $res = {};
2237
2238 # we use static defaults from our JSON schema configuration
2239 foreach my $key (keys %$confdesc) {
2240 if (defined(my $default = $confdesc->{$key}->{default})) {
2241 $res->{$key} = $default;
2242 }
2243 }
19672434 2244
1e3baf05
DM		 2245 return $res;
2246}
2247
2248sub config_list {
2249 my $vmlist = PVE::Cluster::get_vmlist();
2250 my $res = {};
2251 return $res if !$vmlist || !$vmlist->{ids};
2252 my $ids = $vmlist->{ids};
38277afc 2253 my $nodename = nodename();
1e3baf05 2254
1e3baf05
DM		 2255 foreach my $vmid (keys %$ids) {
2256 my $d = $ids->{$vmid};
2257 next if !$d->{node} || $d->{node} ne $nodename;
5ee957cc 2258 next if !$d->{type} || $d->{type} ne 'qemu';
1e3baf05
DM		 2259 $res->{$vmid}->{exists} = 1;
2260 }
2261 return $res;
2262}
2263
64e13401
DM		 2264# test if VM uses local resources (to prevent migration)
2265sub check_local_resources {
2266 my ($conf, $noerr) = @_;
2267
ca6abacf 2268 my @loc_res = ();
19672434 2269
ca6abacf
TM		 2270 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2271 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
64e13401 2272
ca6abacf 2273 push @loc_res, "ivshmem" if $conf->{ivshmem};
6dbcb073 2274
0d29ab3b 2275 foreach my $k (keys %$conf) {
a9ce7583 2276 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
d44712fc
EK		 2277 # sockets are safe: they will recreated be on the target side post-migrate
2278 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
ca6abacf 2279 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
64e13401
DM		 2280 }
2281
ca6abacf 2282 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
64e13401 2283
ca6abacf 2284 return \@loc_res;
64e13401
DM		 2285}
2286
719893a9 2287# check if used storages are available on all nodes (use by migrate)
47152e2e
DM		 2288sub check_storage_availability {
2289 my ($storecfg, $conf, $node) = @_;
2290
912792e2 2291 PVE::QemuConfig->foreach_volume($conf, sub {
47152e2e
DM		 2292 my ($ds, $drive) = @_;
2293
2294 my $volid = $drive->{file};
2295 return if !$volid;
2296
2297 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2298 return if !$sid;
2299
2300 # check if storage is available on both nodes
2301 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2302 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2303 });
2304}
2305
719893a9
DM		 2306# list nodes where all VM images are available (used by has_feature API)
2307sub shared_nodes {
2308 my ($conf, $storecfg) = @_;
2309
2310 my $nodelist = PVE::Cluster::get_nodelist();
2311 my $nodehash = { map { $_ => 1 } @$nodelist };
38277afc 2312 my $nodename = nodename();
be190583 2313
912792e2 2314 PVE::QemuConfig->foreach_volume($conf, sub {
719893a9
DM		 2315 my ($ds, $drive) = @_;
2316
2317 my $volid = $drive->{file};
2318 return if !$volid;
2319
2320 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2321 if ($storeid) {
2322 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2323 if ($scfg->{disable}) {
2324 $nodehash = {};
2325 } elsif (my $avail = $scfg->{nodes}) {
2326 foreach my $node (keys %$nodehash) {
2327 delete $nodehash->{$node} if !$avail->{$node};
2328 }
2329 } elsif (!$scfg->{shared}) {
2330 foreach my $node (keys %$nodehash) {
2331 delete $nodehash->{$node} if $node ne $nodename
2332 }
2333 }
2334 }
2335 });
2336
2337 return $nodehash
2338}
2339
f25852c2
TM		 2340sub check_local_storage_availability {
2341 my ($conf, $storecfg) = @_;
2342
2343 my $nodelist = PVE::Cluster::get_nodelist();
2344 my $nodehash = { map { $_ => {} } @$nodelist };
2345
912792e2 2346 PVE::QemuConfig->foreach_volume($conf, sub {
f25852c2
TM		 2347 my ($ds, $drive) = @_;
2348
2349 my $volid = $drive->{file};
2350 return if !$volid;
2351
2352 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2353 if ($storeid) {
2354 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2355
2356 if ($scfg->{disable}) {
2357 foreach my $node (keys %$nodehash) {
32075a2c 2358 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
f25852c2
TM		 2359 }
2360 } elsif (my $avail = $scfg->{nodes}) {
2361 foreach my $node (keys %$nodehash) {
2362 if (!$avail->{$node}) {
32075a2c 2363 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
f25852c2
TM		 2364 }
2365 }
2366 }
2367 }
2368 });
2369
32075a2c
TL		 2370 foreach my $node (values %$nodehash) {
2371 if (my $unavail = $node->{unavailable_storages}) {
2372 $node->{unavailable_storages} = [ sort keys %$unavail ];
2373 }
2374 }
2375
f25852c2
TM		 2376 return $nodehash
2377}
2378
babf613a 2379# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
1e3baf05 2380sub check_running {
7e8dcf2c 2381 my ($vmid, $nocheck, $node) = @_;
1e3baf05 2382
babf613a
SR		 2383 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2384 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
1e3baf05
DM		 2385}
2386
2387sub vzlist {
19672434 2388
1e3baf05
DM		 2389 my $vzlist = config_list();
2390
d036e418 2391 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
1e3baf05 2392
19672434 2393 while (defined(my $de = $fd->read)) {
1e3baf05
DM		 2394 next if $de !~ m/^(\d+)\.pid$/;
2395 my $vmid = $1;
6b64503e
DM		 2396 next if !defined($vzlist->{$vmid});
2397 if (my $pid = check_running($vmid)) {
1e3baf05
DM		 2398 $vzlist->{$vmid}->{pid} = $pid;
2399 }
2400 }
2401
2402 return $vzlist;
2403}
2404
b1a70cab
DM		 2405our $vmstatus_return_properties = {
2406 vmid => get_standard_option('pve-vmid'),
2407 status => {
2408 description => "Qemu process status.",
2409 type => 'string',
2410 enum => ['stopped', 'running'],
2411 },
2412 maxmem => {
2413 description => "Maximum memory in bytes.",
2414 type => 'integer',
2415 optional => 1,
2416 renderer => 'bytes',
2417 },
2418 maxdisk => {
2419 description => "Root disk size in bytes.",
2420 type => 'integer',
2421 optional => 1,
2422 renderer => 'bytes',
2423 },
2424 name => {
2425 description => "VM name.",
2426 type => 'string',
2427 optional => 1,
2428 },
2429 qmpstatus => {
2430 description => "Qemu QMP agent status.",
2431 type => 'string',
2432 optional => 1,
2433 },
2434 pid => {
2435 description => "PID of running qemu process.",
2436 type => 'integer',
2437 optional => 1,
2438 },
2439 uptime => {
2440 description => "Uptime.",
2441 type => 'integer',
2442 optional => 1,
2443 renderer => 'duration',
2444 },
2445 cpus => {
2446 description => "Maximum usable CPUs.",
2447 type => 'number',
2448 optional => 1,
2449 },
e6ed61b4 2450 lock => {
11efdfa5 2451 description => "The current config lock, if any.",
e6ed61b4
DC		 2452 type => 'string',
2453 optional => 1,
b8e7068a
DC		 2454 },
2455 tags => {
2456 description => "The current configured tags, if any",
2457 type => 'string',
2458 optional => 1,
2459 },
b1a70cab
DM		 2460};
2461
1e3baf05
DM		 2462my $last_proc_pid_stat;
2463
03a33f30
DM		 2464# get VM status information
2465# This must be fast and should not block ($full == false)
2466# We only query KVM using QMP if $full == true (this can be slow)
1e3baf05 2467sub vmstatus {
03a33f30 2468 my ($opt_vmid, $full) = @_;
1e3baf05
DM		 2469
2470 my $res = {};
2471
19672434 2472 my $storecfg = PVE::Storage::config();
1e3baf05
DM		 2473
2474 my $list = vzlist();
3618ee99
EK		 2475 my $defaults = load_defaults();
2476
694fcad4 2477 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
1e3baf05 2478
ae4915a2
DM		 2479 my $cpucount = $cpuinfo->{cpus} || 1;
2480
1e3baf05
DM		 2481 foreach my $vmid (keys %$list) {
2482 next if $opt_vmid && ($vmid ne $opt_vmid);
2483
9f78b695 2484 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 2485
b1a70cab 2486 my $d = { vmid => $vmid };
1e3baf05
DM		 2487 $d->{pid} = $list->{$vmid}->{pid};
2488
2489 # fixme: better status?
2490 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2491
776c5f50 2492 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
af990afe
DM		 2493 if (defined($size)) {
2494 $d->{disk} = 0; # no info available
1e3baf05
DM		 2495 $d->{maxdisk} = $size;
2496 } else {
2497 $d->{disk} = 0;
2498 $d->{maxdisk} = 0;
2499 }
2500
3618ee99
EK		 2501 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2502 * ($conf->{cores} || $defaults->{cores});
ae4915a2 2503 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
d7c8364b 2504 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
ae4915a2 2505
1e3baf05 2506 $d->{name} = $conf->{name} || "VM $vmid";
3618ee99
EK		 2507 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2508 : $defaults->{memory}*(1024*1024);
1e3baf05 2509
8b1accf7 2510 if ($conf->{balloon}) {
4bdb0514 2511 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3618ee99
EK		 2512 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2513 : $defaults->{shares};
8b1accf7
DM		 2514 }
2515
1e3baf05
DM		 2516 $d->{uptime} = 0;
2517 $d->{cpu} = 0;
1e3baf05
DM		 2518 $d->{mem} = 0;
2519
2520 $d->{netout} = 0;
2521 $d->{netin} = 0;
2522
2523 $d->{diskread} = 0;
2524 $d->{diskwrite} = 0;
2525
ffda963f 2526 $d->{template} = PVE::QemuConfig->is_template($conf);
4d8c851b 2527
8107b378 2528 $d->{serial} = 1 if conf_has_serial($conf);
e6ed61b4 2529 $d->{lock} = $conf->{lock} if $conf->{lock};
b8e7068a 2530 $d->{tags} = $conf->{tags} if defined($conf->{tags});
8107b378 2531
1e3baf05
DM		 2532 $res->{$vmid} = $d;
2533 }
2534
2535 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2536 foreach my $dev (keys %$netdev) {
2537 next if $dev !~ m/^tap([1-9]\d*)i/;
2538 my $vmid = $1;
2539 my $d = $res->{$vmid};
2540 next if !$d;
19672434 2541
1e3baf05
DM		 2542 $d->{netout} += $netdev->{$dev}->{receive};
2543 $d->{netin} += $netdev->{$dev}->{transmit};
604ea644
AD		 2544
2545 if ($full) {
2546 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2547 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2548 }
2549
1e3baf05
DM		 2550 }
2551
1e3baf05
DM		 2552 my $ctime = gettimeofday;
2553
2554 foreach my $vmid (keys %$list) {
2555
2556 my $d = $res->{$vmid};
2557 my $pid = $d->{pid};
2558 next if !$pid;
2559
694fcad4
DM		 2560 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2561 next if !$pstat; # not running
19672434 2562
694fcad4 2563 my $used = $pstat->{utime} + $pstat->{stime};
1e3baf05 2564
694fcad4 2565 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
1e3baf05 2566
694fcad4 2567 if ($pstat->{vsize}) {
6b64503e 2568 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
1e3baf05
DM		 2569 }
2570
2571 my $old = $last_proc_pid_stat->{$pid};
2572 if (!$old) {
19672434
DM		 2573 $last_proc_pid_stat->{$pid} = {
2574 time => $ctime,
1e3baf05
DM		 2575 used => $used,
2576 cpu => 0,
1e3baf05
DM		 2577 };
2578 next;
2579 }
2580
7f0b5beb 2581 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
1e3baf05
DM		 2582
2583 if ($dtime > 1000) {
2584 my $dutime = $used - $old->{used};
2585
ae4915a2 2586 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
1e3baf05 2587 $last_proc_pid_stat->{$pid} = {
19672434 2588 time => $ctime,
1e3baf05
DM		 2589 used => $used,
2590 cpu => $d->{cpu},
1e3baf05
DM		 2591 };
2592 } else {
2593 $d->{cpu} = $old->{cpu};
1e3baf05
DM		 2594 }
2595 }
2596
f5eb281a 2597 return $res if !$full;
03a33f30
DM		 2598
2599 my $qmpclient = PVE::QMPClient->new();
2600
64e7fcf2
DM		 2601 my $ballooncb = sub {
2602 my ($vmid, $resp) = @_;
2603
2604 my $info = $resp->{'return'};
38babf81
DM		 2605 return if !$info->{max_mem};
2606
64e7fcf2
DM		 2607 my $d = $res->{$vmid};
2608
38babf81
DM		 2609 # use memory assigned to VM
2610 $d->{maxmem} = $info->{max_mem};
2611 $d->{balloon} = $info->{actual};
2612
2613 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2614 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2615 $d->{freemem} = $info->{free_mem};
64e7fcf2
DM		 2616 }
2617
604ea644 2618 $d->{ballooninfo} = $info;
64e7fcf2
DM		 2619 };
2620
03a33f30
DM		 2621 my $blockstatscb = sub {
2622 my ($vmid, $resp) = @_;
2623 my $data = $resp->{'return'} || [];
2624 my $totalrdbytes = 0;
2625 my $totalwrbytes = 0;
604ea644 2626
03a33f30
DM		 2627 for my $blockstat (@$data) {
2628 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2629 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
604ea644
AD		 2630
2631 $blockstat->{device} =~ s/drive-//;
2632 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
03a33f30
DM		 2633 }
2634 $res->{$vmid}->{diskread} = $totalrdbytes;
2635 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2636 };
2637
2638 my $statuscb = sub {
2639 my ($vmid, $resp) = @_;
64e7fcf2 2640
03a33f30 2641 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
64e7fcf2
DM		 2642 # this fails if ballon driver is not loaded, so this must be
2643 # the last commnand (following command are aborted if this fails).
38babf81 2644 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
03a33f30
DM		 2645
2646 my $status = 'unknown';
2647 if (!defined($status = $resp->{'return'}->{status})) {
2648 warn "unable to get VM status\n";
2649 return;
2650 }
2651
2652 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2653 };
2654
2655 foreach my $vmid (keys %$list) {
2656 next if $opt_vmid && ($vmid ne $opt_vmid);
2657 next if !$res->{$vmid}->{pid}; # not running
2658 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2659 }
2660
b017fbda 2661 $qmpclient->queue_execute(undef, 2);
03a33f30
DM		 2662
2663 foreach my $vmid (keys %$list) {
2664 next if $opt_vmid && ($vmid ne $opt_vmid);
2665 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2666 }
2667
1e3baf05
DM		 2668 return $res;
2669}
2670
8107b378
DC		 2671sub conf_has_serial {
2672 my ($conf) = @_;
2673
2674 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2675 if ($conf->{"serial$i"}) {
2676 return 1;
2677 }
2678 }
2679
2680 return 0;
2681}
2682
d5535a00
TL		 2683sub conf_has_audio {
2684 my ($conf, $id) = @_;
2685
2686 $id //= 0;
2687 my $audio = $conf->{"audio$id"};
2688 return undef if !defined($audio);
2689
2690 my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2691 my $audiodriver = $audioproperties->{driver} // 'spice';
2692
2693 return {
2694 dev => $audioproperties->{device},
b0f96836 2695 dev_id => "audiodev$id",
d5535a00
TL		 2696 backend => $audiodriver,
2697 backend_id => "$audiodriver-backend${id}",
2698 };
2699}
2700
b01de199 2701sub audio_devs {
1cc5ed1b 2702 my ($audio, $audiopciaddr, $machine_version) = @_;
b01de199
TL		 2703
2704 my $devs = [];
2705
2706 my $id = $audio->{dev_id};
1cc5ed1b
AL		 2707 my $audiodev = "";
2708 if (min_version($machine_version, 4, 2)) {
2709 $audiodev = ",audiodev=$audio->{backend_id}";
2710 }
b01de199
TL		 2711
2712 if ($audio->{dev} eq 'AC97') {
1cc5ed1b 2713 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
b01de199
TL		 2714 } elsif ($audio->{dev} =~ /intel\-hda$/) {
2715 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
1cc5ed1b
AL		 2716 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2717 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
b01de199
TL		 2718 } else {
2719 die "unkown audio device '$audio->{dev}', implement me!";
2720 }
2721
2722 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2723
2724 return $devs;
2725}
2726
86b8228b
DM		 2727sub vga_conf_has_spice {
2728 my ($vga) = @_;
2729
55655ebc
DC		 2730 my $vgaconf = parse_vga($vga);
2731 my $vgatype = $vgaconf->{type};
2732 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
590e698c
DM		 2733
2734 return $1 || 1;
86b8228b
DM		 2735}
2736
d731ecbe
WB		 2737sub is_native($) {
2738 my ($arch) = @_;
2739 return get_host_arch() eq $arch;
2740}
2741
045749f2
TL		 2742sub get_vm_arch {
2743 my ($conf) = @_;
2744 return $conf->{arch} // get_host_arch();
2745}
2746
d731ecbe
WB		 2747my $default_machines = {
2748 x86_64 => 'pc',
2749 aarch64 => 'virt',
2750};
2751
045749f2 2752sub get_vm_machine {
ac0077cc 2753 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
045749f2
TL		 2754
2755 my $machine = $forcemachine || $conf->{machine};
d731ecbe 2756
9471e48b 2757 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
045749f2
TL		 2758 $arch //= 'x86_64';
2759 $machine ||= $default_machines->{$arch};
ac0077cc
SR		 2760 if ($add_pve_version) {
2761 $kvmversion //= kvm_user_version();
2762 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2763 $machine .= "+pve$pvever";
2764 }
2765 }
2766
2767 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2768 # for version-pinned machines that do not include a pve-version (e.g.
2769 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2770 $machine .= '+pve0';
045749f2
TL		 2771 }
2772
2773 return $machine;
d731ecbe
WB		 2774}
2775
96ed3574
WB		 2776sub get_ovmf_files($) {
2777 my ($arch) = @_;
2778
2779 my $ovmf = $OVMF->{$arch}
2780 or die "no OVMF images known for architecture '$arch'\n";
2781
2782 return @$ovmf;
2783}
2784
6908fd9b
WB		 2785my $Arch2Qemu = {
2786 aarch64 => '/usr/bin/qemu-system-aarch64',
2787 x86_64 => '/usr/bin/qemu-system-x86_64',
2788};
2789sub get_command_for_arch($) {
2790 my ($arch) = @_;
2791 return '/usr/bin/kvm' if is_native($arch);
2792
2793 my $cmd = $Arch2Qemu->{$arch}
2794 or die "don't know how to emulate architecture '$arch'\n";
2795 return $cmd;
2796}
2797
05a4c550
SR		 2798# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2799# to use in a QEMU command line (-cpu element), first array_intersect the result
2800# of query_supported_ with query_understood_. This is necessary because:
2801#
2802# a) query_understood_ returns flags the host cannot use and
2803# b) query_supported_ (rather the QMP call) doesn't actually return CPU
2804# flags, but CPU settings - with most of them being flags. Those settings
2805# (and some flags, curiously) cannot be specified as a "-cpu" argument.
2806#
2807# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2808# expensive. If you need the value returned from this, you can get it much
2809# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2810# $accel being 'kvm' or 'tcg'.
2811#
2812# pvestatd calls this function on startup and whenever the QEMU/KVM version
2813# changes, automatically populating pmxcfs.
2814#
2815# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2816# since kvm and tcg machines support different flags
2817#
2818sub query_supported_cpu_flags {
52cffab6 2819 my ($arch) = @_;
05a4c550 2820
52cffab6
SR		 2821 $arch //= get_host_arch();
2822 my $default_machine = $default_machines->{$arch};
2823
2824 my $flags = {};
05a4c550
SR		 2825
2826 # FIXME: Once this is merged, the code below should work for ARM as well:
2827 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2828 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2829 $arch eq "aarch64";
2830
2831 my $kvm_supported = defined(kvm_version());
2832 my $qemu_cmd = get_command_for_arch($arch);
2833 my $fakevmid = -1;
2834 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2835
2836 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2837 my $query_supported_run_qemu = sub {
2838 my ($kvm) = @_;
2839
2840 my $flags = {};
2841 my $cmd = [
2842 $qemu_cmd,
2843 '-machine', $default_machine,
2844 '-display', 'none',
2845 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2846 '-mon', 'chardev=qmp,mode=control',
2847 '-pidfile', $pidfile,
2848 '-S', '-daemonize'
2849 ];
2850
2851 if (!$kvm) {
2852 push @$cmd, '-accel', 'tcg';
2853 }
2854
2855 my $rc = run_command($cmd, noerr => 1, quiet => 0);
2856 die "QEMU flag querying VM exited with code " . $rc if $rc;
2857
2858 eval {
2859 my $cmd_result = mon_cmd(
2860 $fakevmid,
2861 'query-cpu-model-expansion',
2862 type => 'full',
2863 model => { name => 'host' }
2864 );
2865
2866 my $props = $cmd_result->{model}->{props};
2867 foreach my $prop (keys %$props) {
2868 next if $props->{$prop} ne '1';
2869 # QEMU returns some flags multiple times, with '_', '.' or '-'
2870 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2871 # We only keep those with underscores, to match /proc/cpuinfo
2872 $prop =~ s/\.|-/_/g;
2873 $flags->{$prop} = 1;
2874 }
2875 };
2876 my $err = $@;
2877
2878 # force stop with 10 sec timeout and 'nocheck'
2879 # always stop, even if QMP failed
2880 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2881
2882 die $err if $err;
2883
2884 return [ sort keys %$flags ];
2885 };
2886
2887 # We need to query QEMU twice, since KVM and TCG have different supported flags
2888 PVE::QemuConfig->lock_config($fakevmid, sub {
2889 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2890 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2891
2892 if ($kvm_supported) {
2893 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2894 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2895 }
2896 });
2897
2898 return $flags;
2899}
2900
2901# Understood CPU flags are written to a file at 'pve-qemu' compile time
2902my $understood_cpu_flag_dir = "/usr/share/kvm";
2903sub query_understood_cpu_flags {
2904 my $arch = get_host_arch();
2905 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2906
2907 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2908 if ! -e $filepath;
2909
2910 my $raw = file_get_contents($filepath);
2911 $raw =~ s/^\s+|\s+$//g;
2912 my @flags = split(/\s+/, $raw);
2913
2914 return \@flags;
2915}
2916
1e3baf05 2917sub config_to_command {
58c64ad5 2918 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
1e3baf05
DM		 2919
2920 my $cmd = [];
8c559505
DM		 2921 my $globalFlags = [];
2922 my $machineFlags = [];
2923 my $rtcFlags = [];
5bdcf937 2924 my $devices = [];
b78ebef7 2925 my $pciaddr = '';
5bdcf937 2926 my $bridges = {};
b42d3cf9 2927 my $ostype = $conf->{ostype};
4317f69f 2928 my $winversion = windows_version($ostype);
d731ecbe 2929 my $kvm = $conf->{kvm};
38277afc 2930 my $nodename = nodename();
d731ecbe 2931
045749f2 2932 my $arch = get_vm_arch($conf);
1476b99f
DC		 2933 my $kvm_binary = get_command_for_arch($arch);
2934 my $kvmver = kvm_user_version($kvm_binary);
045749f2 2935
a04dd5c4
SR		 2936 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2937 $kvmver //= "undefined";
2938 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2939 }
2940
9471e48b
TL		 2941 my $add_pve_version = min_version($kvmver, 4, 1);
2942
2943 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2944 my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
d731ecbe 2945 $kvm //= 1 if is_native($arch);
4317f69f 2946
a77a53ae 2947 $machine_version =~ m/(\d+)\.(\d+)/;
ac0077cc 2948 my ($machine_major, $machine_minor) = ($1, $2);
ac0077cc 2949
b516c848
SR		 2950 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2951 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2952 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2953 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2954 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
ac0077cc
SR		 2955 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2956 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
b516c848 2957 ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
ac0077cc
SR		 2958 }
2959
2960 # if a specific +pve version is required for a feature, use $version_guard
2961 # instead of min_version to allow machines to be run with the minimum
2962 # required version
2963 my $required_pve_version = 0;
2964 my $version_guard = sub {
2965 my ($major, $minor, $pve) = @_;
2966 return 0 if !min_version($machine_version, $major, $minor, $pve);
47f35977
SR		 2967 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2968 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
ac0077cc
SR		 2969 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2970 return 1;
2971 };
a77a53ae 2972
d731ecbe
WB		 2973 if ($kvm) {
2974 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2975 if !defined kvm_version();
2976 }
bfcd9b7e 2977
3392d6ca 2978 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4d3f29ed 2979 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
249c4a6c
AD		 2980 my $use_old_bios_files = undef;
2981 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
db656e5f 2982
f08e17c7
AD		 2983 my $cpuunits = defined($conf->{cpuunits}) ?
2984 $conf->{cpuunits} : $defaults->{cpuunits};
2985
1476b99f 2986 push @$cmd, $kvm_binary;
1e3baf05
DM		 2987
2988 push @$cmd, '-id', $vmid;
2989
e4d4cda1
HR		 2990 my $vmname = $conf->{name} || "vm$vmid";
2991
2992 push @$cmd, '-name', $vmname;
2993
1e3baf05
DM		 2994 my $use_virtio = 0;
2995
d036e418 2996 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
c971c4f2
AD		 2997 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2998 push @$cmd, '-mon', "chardev=qmp,mode=control";
2999
2ea5fb7e 3000 if (min_version($machine_version, 2, 12)) {
b4496b9e 3001 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
71bd73b5
DC		 3002 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3003 }
1e3baf05 3004
d036e418 3005 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
19672434 3006
1e3baf05
DM		 3007 push @$cmd, '-daemonize';
3008
2796e7d5 3009 if ($conf->{smbios1}) {
1f30ac3a
CE		 3010 my $smbios_conf = parse_smbios1($conf->{smbios1});
3011 if ($smbios_conf->{base64}) {
3012 # Do not pass base64 flag to qemu
3013 delete $smbios_conf->{base64};
3014 my $smbios_string = "";
3015 foreach my $key (keys %$smbios_conf) {
3016 my $value;
3017 if ($key eq "uuid") {
3018 $value = $smbios_conf->{uuid}
3019 } else {
3020 $value = decode_base64($smbios_conf->{$key});
3021 }
3022 # qemu accepts any binary data, only commas need escaping by double comma
3023 $value =~ s/,/,,/g;
3024 $smbios_string .= "," . $key . "=" . $value if $value;
3025 }
3026 push @$cmd, '-smbios', "type=1" . $smbios_string;
3027 } else {
3028 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3029 }
2796e7d5
DM		 3030 }
3031
3edb45e7 3032 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
818c3b8d
TL		 3033 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3034 die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
2ddc0a5c 3035
818c3b8d 3036 my ($path, $format);
b57d4863
SR		 3037 if (my $efidisk = $conf->{efidisk0}) {
3038 my $d = parse_drive('efidisk0', $efidisk);
2ddc0a5c 3039 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
13bca7b4 3040 $format = $d->{format};
2ddc0a5c
DC		 3041 if ($storeid) {
3042 $path = PVE::Storage::path($storecfg, $d->{file});
13bca7b4
WB		 3043 if (!defined($format)) {
3044 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3045 $format = qemu_img_format($scfg, $volname);
3046 }
2ddc0a5c
DC		 3047 } else {
3048 $path = $d->{file};
13bca7b4
WB		 3049 die "efidisk format must be specified\n"
3050 if !defined($format);
2ddc0a5c 3051 }
2ddc0a5c 3052 } else {
4dcce9ee
TL		 3053 warn "no efidisk configured! Using temporary efivars disk.\n";
3054 $path = "/tmp/$vmid-ovmf.fd";
96ed3574 3055 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
13bca7b4 3056 $format = 'raw';
2ddc0a5c 3057 }
4dcce9ee 3058
818ce80e
DC		 3059 my $size_str = "";
3060
3061 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3062 $size_str = ",size=" . (-s $ovmf_vars);
3063 }
3064
96ed3574 3065 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
818ce80e 3066 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
a783c78e
AD		 3067 }
3068
7583d156
DC		 3069 # load q35 config
3070 if ($q35) {
3071 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
2ea5fb7e 3072 if (min_version($machine_version, 4, 0)) {
7583d156
DC		 3073 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3074 } else {
3075 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3076 }
3077 }
da8b4189 3078
844d8fa6
DC		 3079 if ($conf->{vmgenid}) {
3080 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3081 }
3082
d40e5e18 3083 # add usb controllers
d559309f 3084 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
d40e5e18 3085 push @$devices, @usbcontrollers if @usbcontrollers;
55655ebc 3086 my $vga = parse_vga($conf->{vga});
2fa3151e 3087
55655ebc
DC		 3088 my $qxlnum = vga_conf_has_spice($conf->{vga});
3089 $vga->{type} = 'qxl' if $qxlnum;
2fa3151e 3090
55655ebc 3091 if (!$vga->{type}) {
869ad4a7
WB		 3092 if ($arch eq 'aarch64') {
3093 $vga->{type} = 'virtio';
2ea5fb7e 3094 } elsif (min_version($machine_version, 2, 9)) {
55655ebc 3095 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
a2a5cd64 3096 } else {
55655ebc 3097 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
a2a5cd64 3098 }
5acbfe9e
DM		 3099 }
3100
1e3baf05 3101 # enable absolute mouse coordinates (needed by vnc)
5acbfe9e
DM		 3102 my $tablet;
3103 if (defined($conf->{tablet})) {
3104 $tablet = $conf->{tablet};
3105 } else {
3106 $tablet = $defaults->{tablet};
590e698c 3107 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
55655ebc 3108 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
5acbfe9e
DM		 3109 }
3110
d559309f
WB		 3111 if ($tablet) {
3112 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3113 my $kbd = print_keyboarddevice_full($conf, $arch);
3114 push @$devices, '-device', $kbd if defined($kbd);
3115 }
b467f79a 3116
74c17b7a 3117 # host pci device passthrough
13d68979 3118 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
7de7f675 3119 $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
1e3baf05
DM		 3120
3121 # usb devices
ae36393d 3122 my $usb_dev_features = {};
2ea5fb7e 3123 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
ae36393d
AL		 3124
3125 my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
d40e5e18 3126 push @$devices, @usbdevices if @usbdevices;
1e3baf05 3127 # serial devices
bae179aa 3128 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
34978be3 3129 if (my $path = $conf->{"serial$i"}) {
9f9d2fb2
DM		 3130 if ($path eq 'socket') {
3131 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3132 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
91b01bbb
WB		 3133 # On aarch64, serial0 is the UART device. Qemu only allows
3134 # connecting UART devices via the '-serial' command line, as
3135 # the device has a fixed slot on the hardware...
3136 if ($arch eq 'aarch64' && $i == 0) {
3137 push @$devices, '-serial', "chardev:serial$i";
3138 } else {
3139 push @$devices, '-device', "isa-serial,chardev=serial$i";
3140 }
9f9d2fb2
DM		 3141 } else {
3142 die "no such serial device\n" if ! -c $path;
3143 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3144 push @$devices, '-device', "isa-serial,chardev=serial$i";
3145 }
34978be3 3146 }
1e3baf05
DM		 3147 }
3148
3149 # parallel devices
1989a89c 3150 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
34978be3 3151 if (my $path = $conf->{"parallel$i"}) {
19672434 3152 die "no such parallel device\n" if ! -c $path;
32e69805 3153 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
4c5dbaf6 3154 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
5bdcf937 3155 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
34978be3 3156 }
1e3baf05
DM		 3157 }
3158
b01de199 3159 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
2e7b5925 3160 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
1cc5ed1b 3161 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
b01de199 3162 push @$devices, @$audio_devs;
2e7b5925 3163 }
19672434 3164
1e3baf05
DM		 3165 my $sockets = 1;
3166 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3167 $sockets = $conf->{sockets} if $conf->{sockets};
3168
3169 my $cores = $conf->{cores} || 1;
3bd18e48 3170
de9d1e55 3171 my $maxcpus = $sockets * $cores;
76267728 3172
de9d1e55 3173 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
76267728 3174
de9d1e55
AD		 3175 my $allowed_vcpus = $cpuinfo->{cpus};
3176
6965d5d1 3177 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
de9d1e55
AD		 3178 if ($allowed_vcpus < $maxcpus);
3179
2ea5fb7e 3180 if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
1e3baf05 3181
69c81430
AD		 3182 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3183 for (my $i = 2; $i <= $vcpus; $i++) {
3184 my $cpustr = print_cpu_device($conf,$i);
3185 push @$cmd, '-device', $cpustr;
3186 }
3187
3188 } else {
3189
3190 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3191 }
1e3baf05
DM		 3192 push @$cmd, '-nodefaults';
3193
32baffb4 3194 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3b408e82 3195
0888fdce
DM		 3196 my $bootindex_hash = {};
3197 my $i = 1;
3198 foreach my $o (split(//, $bootorder)) {
3199 $bootindex_hash->{$o} = $i*100;
3200 $i++;
afdb31d5 3201 }
3b408e82 3202
dbea4415 3203 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
1e3baf05 3204
6b64503e 3205 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
1e3baf05 3206
6b64503e 3207 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
1e3baf05 3208
84902837 3209 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
2ea5fb7e 3210 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
d036e418 3211 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
dc62a7fa 3212 push @$cmd, '-vnc', "unix:$socket,password";
b7be4ba9 3213 } else {
55655ebc 3214 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
b7be4ba9
AD		 3215 push @$cmd, '-nographic';
3216 }
3217
1e3baf05 3218 # time drift fix
6b64503e 3219 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
8c559505 3220 my $useLocaltime = $conf->{localtime};
1e3baf05 3221
4317f69f
AD		 3222 if ($winversion >= 5) { # windows
3223 $useLocaltime = 1 if !defined($conf->{localtime});
7a131888 3224
4317f69f
AD		 3225 # use time drift fix when acpi is enabled
3226 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3227 $tdf = 1 if !defined($conf->{tdf});
462e8d19 3228 }
4317f69f 3229 }
462e8d19 3230
4317f69f
AD		 3231 if ($winversion >= 6) {
3232 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3233 push @$cmd, '-no-hpet';
1e3baf05
DM		 3234 }
3235
8c559505
DM		 3236 push @$rtcFlags, 'driftfix=slew' if $tdf;
3237
2f6f002c 3238 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
8c559505
DM		 3239 push @$rtcFlags, "base=$conf->{startdate}";
3240 } elsif ($useLocaltime) {
3241 push @$rtcFlags, 'base=localtime';
3242 }
1e3baf05 3243
58c64ad5
SR		 3244 if ($forcecpu) {
3245 push @$cmd, '-cpu', $forcecpu;
3246 } else {
2f6f002c 3247 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
58c64ad5 3248 }
519ed28c 3249
0567a4d5 3250 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
370b05e7 3251
1e3baf05
DM		 3252 push @$cmd, '-S' if $conf->{freeze};
3253
b20df606 3254 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
1e3baf05 3255
48657158
MD		 3256 my $guest_agent = parse_guest_agent($conf);
3257
3258 if ($guest_agent->{enabled}) {
d036e418 3259 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
ab6a046f 3260 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
48657158 3261
60f03a11 3262 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
48657158
MD		 3263 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3264 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3265 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3266 } elsif ($guest_agent->{type} eq 'isa') {
3267 push @$devices, '-device', "isa-serial,chardev=qga0";
3268 }
ab6a046f
AD		 3269 }
3270
2cf61f33
SR		 3271 my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3272 if ($rng && &$version_guard(4, 1, 2)) {
05853188
SR		 3273 check_rng_source($rng->{source});
3274
2cf61f33
SR		 3275 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3276 my $period = $rng->{period} // $rng_fmt->{period}->{default};
2cf61f33
SR		 3277 my $limiter_str = "";
3278 if ($max_bytes) {
3279 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3280 }
3281
2cf61f33 3282 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
2cf61f33
SR		 3283 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3284 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3285 }
3286
1d794448 3287 my $spice_port;
2fa3151e 3288
590e698c
DM		 3289 if ($qxlnum) {
3290 if ($qxlnum > 1) {
ac087616 3291 if ($winversion){
2f6f002c 3292 for (my $i = 1; $i < $qxlnum; $i++){
2ea5fb7e 3293 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
590e698c
DM		 3294 }
3295 } else {
3296 # assume other OS works like Linux
55655ebc
DC		 3297 my ($ram, $vram) = ("134217728", "67108864");
3298 if ($vga->{memory}) {
3299 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3300 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3301 }
3302 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3303 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
2fa3151e
AD		 3304 }
3305 }
3306
d559309f 3307 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
95a4b4a9 3308
af0eba7e 3309 my $pfamily = PVE::Tools::get_host_address_family($nodename);
91152441
WB		 3310 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3311 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4d316a63
AL		 3312
3313 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3314 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3315 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3316
91152441
WB		 3317 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3318 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
943340a6 3319
caab114a
TL		 3320 my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3321 if ($spice_enhancement->{foldersharing}) {
3322 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3323 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3324 }
c4df18db 3325
caab114a
TL		 3326 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3327 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3328 push @$devices, '-spice', "$spice_opts";
1011b570
DM		 3329 }
3330
8d9ae0d2
DM		 3331 # enable balloon by default, unless explicitly disabled
3332 if (!defined($conf->{balloon}) || $conf->{balloon}) {
d559309f 3333 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
8d9ae0d2
DM		 3334 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3335 }
1e3baf05 3336
0ea9541d
DM		 3337 if ($conf->{watchdog}) {
3338 my $wdopts = parse_watchdog($conf->{watchdog});
d559309f 3339 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
0a40e8ea 3340 my $watchdog = $wdopts->{model} || 'i6300esb';
5bdcf937
AD		 3341 push @$devices, '-device', "$watchdog$pciaddr";
3342 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
0ea9541d
DM		 3343 }
3344
1e3baf05 3345 my $vollist = [];
941e0c42 3346 my $scsicontroller = {};
26ee04b6 3347 my $ahcicontroller = {};
cdd20088 3348 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
1e3baf05 3349
5881b913
DM		 3350 # Add iscsi initiator name if available
3351 if (my $initiator = get_initiator_name()) {
3352 push @$devices, '-iscsi', "initiator-name=$initiator";
3353 }
3354
912792e2 3355 PVE::QemuConfig->foreach_volume($conf, sub {
1e3baf05
DM		 3356 my ($ds, $drive) = @_;
3357
ff1a2432 3358 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
1e3baf05 3359 push @$vollist, $drive->{file};
ff1a2432 3360 }
afdb31d5 3361
4dcce9ee
TL		 3362 # ignore efidisk here, already added in bios/fw handling code above
3363 return if $drive->{interface} eq 'efidisk';
3364
1e3baf05 3365 $use_virtio = 1 if $ds =~ m/^virtio/;
3b408e82
DM		 3366
3367 if (drive_is_cdrom ($drive)) {
3368 if ($bootindex_hash->{d}) {
3369 $drive->{bootindex} = $bootindex_hash->{d};
3370 $bootindex_hash->{d} += 1;
3371 }
3372 } else {
3373 if ($bootindex_hash->{c}) {
3374 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3375 $bootindex_hash->{c} += 1;
3376 }
3377 }
3378
2f6f002c 3379 if ($drive->{interface} eq 'virtio'){
51f492cd
AD		 3380 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3381 }
3382
2f6f002c 3383 if ($drive->{interface} eq 'scsi') {
cdd20088 3384
ee034f5c 3385 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
6731a4cf 3386
b8fb1c03
SR		 3387 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3388 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3389
d559309f 3390 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
a1b7d579 3391 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
fc8b40fd
AD		 3392
3393 my $iothread = '';
3394 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3395 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3396 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
e7a5104d
DC		 3397 } elsif ($drive->{iothread}) {
3398 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
fc8b40fd
AD		 3399 }
3400
6e11f143
AD		 3401 my $queues = '';
3402 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3403 $queues = ",num_queues=$drive->{queues}";
370b05e7 3404 }
6e11f143
AD		 3405
3406 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
cdd20088 3407 $scsicontroller->{$controller}=1;
2f6f002c 3408 }
3b408e82 3409
26ee04b6 3410 if ($drive->{interface} eq 'sata') {
2f6f002c
TL		 3411 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3412 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3413 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3414 $ahcicontroller->{$controller}=1;
26ee04b6 3415 }
46f58b5f 3416
776c5f50 3417 my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
4ef13a7f
FG		 3418 $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3419
15b21acc 3420 push @$devices, '-drive',$drive_cmd;
d559309f 3421 push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
1e3baf05
DM		 3422 });
3423
cc4d6182 3424 for (my $i = 0; $i < $MAX_NETS; $i++) {
2f6f002c
TL		 3425 next if !$conf->{"net$i"};
3426 my $d = parse_net($conf->{"net$i"});
3427 next if !$d;
1e3baf05 3428
2f6f002c 3429 $use_virtio = 1 if $d->{model} eq 'virtio';
1e3baf05 3430
2f6f002c
TL		 3431 if ($bootindex_hash->{n}) {
3432 $d->{bootindex} = $bootindex_hash->{n};
3433 $bootindex_hash->{n} += 1;
3434 }
1e3baf05 3435
2f6f002c
TL		 3436 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3437 push @$devices, '-netdev', $netdevfull;
5bdcf937 3438
2f6f002c
TL		 3439 my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3440 push @$devices, '-device', $netdevicefull;
5bdcf937 3441 }
1e3baf05 3442
6dbcb073
DC		 3443 if ($conf->{ivshmem}) {
3444 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
e3c27a6a 3445
6dbcb073
DC		 3446 my $bus;
3447 if ($q35) {
3448 $bus = print_pcie_addr("ivshmem");
3449 } else {
3450 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3451 }
e3c27a6a
TL		 3452
3453 my $ivshmem_name = $ivshmem->{name} // $vmid;
3454 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3455
6dbcb073
DC		 3456 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3457 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3458 }
3459
2513b862
DC		 3460 # pci.4 is nested in pci.1
3461 $bridges->{1} = 1 if $bridges->{4};
3462
db656e5f
DM		 3463 if (!$q35) {
3464 # add pci bridges
2ea5fb7e 3465 if (min_version($machine_version, 2, 3)) {
fc79e813
AD		 3466 $bridges->{1} = 1;
3467 $bridges->{2} = 1;
3468 }
3469
6731a4cf
AD		 3470 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3471
2513b862
DC		 3472 }
3473
3474 for my $k (sort {$b cmp $a} keys %$bridges) {
3475 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
13d68979
SR		 3476
3477 my $k_name = $k;
3478 if ($k == 2 && $legacy_igd) {
3479 $k_name = "$k-igd";
3480 }
3481 $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3482
2513b862
DC		 3483 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3484 if ($q35) {
3485 # add after -readconfig pve-q35.cfg
3486 splice @$devices, 2, 0, '-device', $devstr;
3487 } else {
3488 unshift @$devices, '-device', $devstr if $k > 0;
f8e83f05 3489 }
19672434
DM		 3490 }
3491
ac0077cc
SR		 3492 if (!$kvm) {
3493 push @$machineFlags, 'accel=tcg';
3494 }
3495
3496 my $machine_type_min = $machine_type;
3497 if ($add_pve_version) {
3498 $machine_type_min =~ s/\+pve\d+$//;
3499 $machine_type_min .= "+pve$required_pve_version";
3500 }
3501 push @$machineFlags, "type=${machine_type_min}";
3502
5bdcf937 3503 push @$cmd, @$devices;
2f6f002c
TL		 3504 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3505 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3506 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
8c559505 3507
7ceade4c
DC		 3508 if (my $vmstate = $conf->{vmstate}) {
3509 my $statepath = PVE::Storage::path($storecfg, $vmstate);
24d1f93a 3510 push @$vollist, $vmstate;
7ceade4c 3511 push @$cmd, '-loadstate', $statepath;
b85666cf 3512 print "activating and using '$vmstate' as vmstate\n";
7ceade4c
DC		 3513 }
3514
76350670
DC		 3515 # add custom args
3516 if ($conf->{args}) {
3517 my $aa = PVE::Tools::split_args($conf->{args});
3518 push @$cmd, @$aa;
3519 }
3520
1d794448 3521 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
1e3baf05 3522}
19672434 3523
05853188
SR		 3524sub check_rng_source {
3525 my ($source) = @_;
3526
3527 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3528 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3529 if ! -e $source;
3530
3531 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3532 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3533 # Needs to abort, otherwise QEMU crashes on first rng access.
3534 # Note that rng_current cannot be changed to 'none' manually, so
3535 # once the VM is past this point, it is no longer an issue.
3536 die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3537 . " exists, but '$rng_current' is set to 'none'. Ensure that"
3538 . " a compatible hardware-RNG is attached to the host.\n";
3539 }
3540}
3541
943340a6 3542sub spice_port {
1011b570 3543 my ($vmid) = @_;
943340a6 3544
0a13e08e 3545 my $res = mon_cmd($vmid, 'query-spice');
943340a6
DM		 3546
3547 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
1011b570
DM		 3548}
3549
86fdcfb2
DA		 3550sub vm_devices_list {
3551 my ($vmid) = @_;
3552
0a13e08e 3553 my $res = mon_cmd($vmid, 'query-pci');
f721624b 3554 my $devices_to_check = [];
ceea9078
DM		 3555 my $devices = {};
3556 foreach my $pcibus (@$res) {
f721624b
DC		 3557 push @$devices_to_check, @{$pcibus->{devices}},
3558 }
3559
3560 while (@$devices_to_check) {
3561 my $to_check = [];
3562 for my $d (@$devices_to_check) {
3563 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3564 next if !$d->{'pci_bridge'};
3565
3566 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3567 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
f78cc802 3568 }
f721624b 3569 $devices_to_check = $to_check;
f78cc802
AD		 3570 }
3571
0a13e08e 3572 my $resblock = mon_cmd($vmid, 'query-block');
f78cc802
AD		 3573 foreach my $block (@$resblock) {
3574 if($block->{device} =~ m/^drive-(\S+)/){
3575 $devices->{$1} = 1;
1dc4f496
DM		 3576 }
3577 }
86fdcfb2 3578
0a13e08e 3579 my $resmice = mon_cmd($vmid, 'query-mice');
3d7389fe
DM		 3580 foreach my $mice (@$resmice) {
3581 if ($mice->{name} eq 'QEMU HID Tablet') {
3582 $devices->{tablet} = 1;
3583 last;
3584 }
3585 }
3586
deb091c5
DC		 3587 # for usb devices there is no query-usb
3588 # but we can iterate over the entries in
3589 # qom-list path=/machine/peripheral
0a13e08e 3590 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
deb091c5
DC		 3591 foreach my $per (@$resperipheral) {
3592 if ($per->{name} =~ m/^usb\d+$/) {
3593 $devices->{$per->{name}} = 1;
3594 }
3595 }
3596
1dc4f496 3597 return $devices;
86fdcfb2
DA		 3598}
3599
ec21aa11 3600sub vm_deviceplug {
d559309f 3601 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
ae57f6b3 3602
3392d6ca 3603 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
db656e5f 3604
95d6343b
DA		 3605 my $devices_list = vm_devices_list($vmid);
3606 return 1 if defined($devices_list->{$deviceid});
3607
d559309f 3608 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
fee46675 3609
3d7389fe 3610 if ($deviceid eq 'tablet') {
fee46675 3611
d559309f
WB		 3612 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3613
3614 } elsif ($deviceid eq 'keyboard') {
3615
3616 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3d7389fe 3617
4eb68604
DC		 3618 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3619
f745762b
DC		 3620 die "usb hotplug currently not reliable\n";
3621 # since we can't reliably hot unplug all added usb devices
3622 # and usb passthrough disables live migration
3623 # we disable usb hotplugging for now
4eb68604
DC		 3624 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3625
fee46675 3626 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
40f28a9f 3627
22de899a
AD		 3628 qemu_iothread_add($vmid, $deviceid, $device);
3629
fee46675 3630 qemu_driveadd($storecfg, $vmid, $device);
d559309f 3631 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
fee46675 3632
5e5dcb73 3633 qemu_deviceadd($vmid, $devicefull);
fee46675
DM		 3634 eval { qemu_deviceaddverify($vmid, $deviceid); };
3635 if (my $err = $@) {
63c2da2f
DM		 3636 eval { qemu_drivedel($vmid, $deviceid); };
3637 warn $@ if $@;
fee46675 3638 die $err;
5e5dcb73 3639 }
cfc817c7 3640
2733141c 3641 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
fee46675 3642
fc8b40fd 3643
cdd20088 3644 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
d559309f 3645 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
a1b7d579 3646 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
2733141c
AD		 3647
3648 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
fee46675 3649
fc8b40fd
AD		 3650 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3651 qemu_iothread_add($vmid, $deviceid, $device);
3652 $devicefull .= ",iothread=iothread-$deviceid";
3653 }
3654
6e11f143
AD		 3655 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3656 $devicefull .= ",num_queues=$device->{queues}";
3657 }
3658
cfc817c7 3659 qemu_deviceadd($vmid, $devicefull);
fee46675 3660 qemu_deviceaddverify($vmid, $deviceid);
cfc817c7 3661
fee46675
DM		 3662 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3663
d559309f 3664 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
fee46675 3665 qemu_driveadd($storecfg, $vmid, $device);
a1b7d579 3666
d559309f 3667 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
fee46675
DM		 3668 eval { qemu_deviceadd($vmid, $devicefull); };
3669 if (my $err = $@) {
63c2da2f
DM		 3670 eval { qemu_drivedel($vmid, $deviceid); };
3671 warn $@ if $@;
fee46675 3672 die $err;
a4f091a0 3673 }
a4f091a0 3674
fee46675
DM		 3675 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3676
95d3be58 3677 return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
8718099c 3678
3392d6ca 3679 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
95d3be58
DC		 3680 my $use_old_bios_files = undef;
3681 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
8718099c 3682
95d3be58
DC		 3683 my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3684 qemu_deviceadd($vmid, $netdevicefull);
79046fd1
DC		 3685 eval {
3686 qemu_deviceaddverify($vmid, $deviceid);
3687 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3688 };
fee46675
DM		 3689 if (my $err = $@) {
3690 eval { qemu_netdevdel($vmid, $deviceid); };
3691 warn $@ if $@;
3692 die $err;
95d3be58 3693 }
2630d2a9 3694
fee46675 3695 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
b467f79a 3696
40f28a9f 3697 my $bridgeid = $2;
d559309f 3698 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
40f28a9f 3699 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
a1b7d579 3700
40f28a9f 3701 qemu_deviceadd($vmid, $devicefull);
fee46675
DM		 3702 qemu_deviceaddverify($vmid, $deviceid);
3703
3704 } else {
a1b7d579 3705 die "can't hotplug device '$deviceid'\n";
40f28a9f
AD		 3706 }
3707
5e5dcb73 3708 return 1;
a4dea331
DA		 3709}
3710
3eec5767 3711# fixme: this should raise exceptions on error!
ec21aa11 3712sub vm_deviceunplug {
f19d1c47 3713 my ($vmid, $conf, $deviceid) = @_;
873c2d69 3714
95d6343b
DA		 3715 my $devices_list = vm_devices_list($vmid);
3716 return 1 if !defined($devices_list->{$deviceid});
3717
63c2da2f
DM		 3718 die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3719
d559309f 3720 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
63c2da2f 3721
3d7389fe 3722 qemu_devicedel($vmid, $deviceid);
3d7389fe 3723
4eb68604
DC		 3724 } elsif ($deviceid =~ m/^usb\d+$/) {
3725
f745762b
DC		 3726 die "usb hotplug currently not reliable\n";
3727 # when unplugging usb devices this way,
3728 # there may be remaining usb controllers/hubs
3729 # so we disable it for now
4eb68604
DC		 3730 qemu_devicedel($vmid, $deviceid);
3731 qemu_devicedelverify($vmid, $deviceid);
3732
63c2da2f 3733 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
f19d1c47 3734
5e5dcb73 3735 qemu_devicedel($vmid, $deviceid);
63c2da2f
DM		 3736 qemu_devicedelverify($vmid, $deviceid);
3737 qemu_drivedel($vmid, $deviceid);
22de899a
AD		 3738 qemu_iothread_del($conf, $vmid, $deviceid);
3739
2733141c 3740 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
a1b7d579 3741
63c2da2f 3742 qemu_devicedel($vmid, $deviceid);
8ce30dde 3743 qemu_devicedelverify($vmid, $deviceid);
fc8b40fd 3744 qemu_iothread_del($conf, $vmid, $deviceid);
a1b7d579 3745
63c2da2f 3746 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
cfc817c7 3747
63c2da2f
DM		 3748 qemu_devicedel($vmid, $deviceid);
3749 qemu_drivedel($vmid, $deviceid);
a1b7d579 3750 qemu_deletescsihw($conf, $vmid, $deviceid);
8ce30dde 3751
63c2da2f 3752 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
a4f091a0 3753
2630d2a9 3754 qemu_devicedel($vmid, $deviceid);
63c2da2f
DM		 3755 qemu_devicedelverify($vmid, $deviceid);
3756 qemu_netdevdel($vmid, $deviceid);
3757
3758 } else {
3759 die "can't unplug device '$deviceid'\n";
2630d2a9
DA		 3760 }
3761
5e5dcb73
DA		 3762 return 1;
3763}
3764
3765sub qemu_deviceadd {
3766 my ($vmid, $devicefull) = @_;
873c2d69 3767
d695b5b7
AD		 3768 $devicefull = "driver=".$devicefull;
3769 my %options = split(/[=,]/, $devicefull);
f19d1c47 3770
0a13e08e 3771 mon_cmd($vmid, "device_add" , %options);
5e5dcb73 3772}
afdb31d5 3773
5e5dcb73 3774sub qemu_devicedel {
fee46675 3775 my ($vmid, $deviceid) = @_;
63c2da2f 3776
0a13e08e 3777 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
5e5dcb73
DA		 3778}
3779
22de899a
AD		 3780sub qemu_iothread_add {
3781 my($vmid, $deviceid, $device) = @_;
3782
3783 if ($device->{iothread}) {
3784 my $iothreads = vm_iothreads_list($vmid);
3785 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3786 }
3787}
3788
3789sub qemu_iothread_del {
3790 my($conf, $vmid, $deviceid) = @_;
3791
7820eae4
DC		 3792 my $confid = $deviceid;
3793 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3794 $confid = 'scsi' . $1;
3795 }
3796 my $device = parse_drive($confid, $conf->{$confid});
22de899a
AD		 3797 if ($device->{iothread}) {
3798 my $iothreads = vm_iothreads_list($vmid);
3799 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3800 }
3801}
3802
4d3f29ed
AD		 3803sub qemu_objectadd {
3804 my($vmid, $objectid, $qomtype) = @_;
3805
0a13e08e 3806 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4d3f29ed
AD		 3807
3808 return 1;
3809}
3810
3811sub qemu_objectdel {
3812 my($vmid, $objectid) = @_;
3813
0a13e08e 3814 mon_cmd($vmid, "object-del", id => $objectid);
4d3f29ed
AD		 3815
3816 return 1;
3817}
3818
5e5dcb73 3819sub qemu_driveadd {
fee46675 3820 my ($storecfg, $vmid, $device) = @_;
5e5dcb73 3821
776c5f50 3822 my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
7a69fc3c 3823 $drive =~ s/\\/\\\\/g;
0a13e08e 3824 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
fee46675 3825
5e5dcb73 3826 # If the command succeeds qemu prints: "OK"
fee46675
DM		 3827 return 1 if $ret =~ m/OK/s;
3828
3829 die "adding drive failed: $ret\n";
5e5dcb73 3830}
afdb31d5 3831
5e5dcb73
DA		 3832sub qemu_drivedel {
3833 my($vmid, $deviceid) = @_;
873c2d69 3834
0a13e08e 3835 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
5e5dcb73 3836 $ret =~ s/^\s+//;
a1b7d579 3837
63c2da2f 3838 return 1 if $ret eq "";
a1b7d579 3839
63c2da2f 3840 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
a1b7d579
DM		 3841 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3842
63c2da2f 3843 die "deleting drive $deviceid failed : $ret\n";
5e5dcb73 3844}
f19d1c47 3845
5e5dcb73 3846sub qemu_deviceaddverify {
fee46675 3847 my ($vmid, $deviceid) = @_;
873c2d69 3848
5e5dcb73
DA		 3849 for (my $i = 0; $i <= 5; $i++) {
3850 my $devices_list = vm_devices_list($vmid);
3851 return 1 if defined($devices_list->{$deviceid});
3852 sleep 1;
afdb31d5 3853 }
fee46675
DM		 3854
3855 die "error on hotplug device '$deviceid'\n";
5e5dcb73 3856}
afdb31d5 3857
5e5dcb73
DA		 3858
3859sub qemu_devicedelverify {
63c2da2f
DM		 3860 my ($vmid, $deviceid) = @_;
3861
a1b7d579 3862 # need to verify that the device is correctly removed as device_del
63c2da2f 3863 # is async and empty return is not reliable
5e5dcb73 3864
5e5dcb73
DA		 3865 for (my $i = 0; $i <= 5; $i++) {
3866 my $devices_list = vm_devices_list($vmid);
3867 return 1 if !defined($devices_list->{$deviceid});
3868 sleep 1;
afdb31d5 3869 }
63c2da2f
DM		 3870
3871 die "error on hot-unplugging device '$deviceid'\n";
873c2d69
DA		 3872}
3873
cdd20088 3874sub qemu_findorcreatescsihw {
d559309f 3875 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
cfc817c7 3876
ee034f5c 3877 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
2733141c
AD		 3878
3879 my $scsihwid="$controller_prefix$controller";
cfc817c7
DA		 3880 my $devices_list = vm_devices_list($vmid);
3881
cdd20088 3882 if(!defined($devices_list->{$scsihwid})) {
d559309f 3883 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
cfc817c7 3884 }
fee46675 3885
cfc817c7
DA		 3886 return 1;
3887}
3888
8ce30dde
AD		 3889sub qemu_deletescsihw {
3890 my ($conf, $vmid, $opt) = @_;
3891
3892 my $device = parse_drive($opt, $conf->{$opt});
3893
a1511b3c 3894 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
2733141c
AD		 3895 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3896 return 1;
3897 }
3898
ee034f5c 3899 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
8ce30dde
AD		 3900
3901 my $devices_list = vm_devices_list($vmid);
3902 foreach my $opt (keys %{$devices_list}) {
e0fd2b2f
FE		 3903 if (is_valid_drivename($opt)) {
3904 my $drive = parse_drive($opt, $conf->{$opt});
8ce30dde
AD		 3905 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3906 return 1;
3907 }
3908 }
3909 }
3910
3911 my $scsihwid="scsihw$controller";
3912
3913 vm_deviceunplug($vmid, $conf, $scsihwid);
3914
3915 return 1;
3916}
3917
281fedb3 3918sub qemu_add_pci_bridge {
d559309f 3919 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
40f28a9f
AD		 3920
3921 my $bridges = {};
281fedb3
DM		 3922
3923 my $bridgeid;
3924
d559309f 3925 print_pci_addr($device, $bridges, $arch, $machine_type);
40f28a9f
AD		 3926
3927 while (my ($k, $v) = each %$bridges) {
3928 $bridgeid = $k;
3929 }
fee46675 3930 return 1 if !defined($bridgeid) || $bridgeid < 1;
281fedb3 3931
40f28a9f
AD		 3932 my $bridge = "pci.$bridgeid";
3933 my $devices_list = vm_devices_list($vmid);
3934
281fedb3 3935 if (!defined($devices_list->{$bridge})) {
d559309f 3936 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
40f28a9f 3937 }
281fedb3 3938
40f28a9f
AD		 3939 return 1;
3940}
3941
25088687
DM		 3942sub qemu_set_link_status {
3943 my ($vmid, $device, $up) = @_;
3944
0a13e08e 3945 mon_cmd($vmid, "set_link", name => $device,
25088687
DM		 3946 up => $up ? JSON::true : JSON::false);
3947}
3948
2630d2a9 3949sub qemu_netdevadd {
d559309f 3950 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
2630d2a9 3951
d559309f 3952 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
73aa03b8 3953 my %options = split(/[=,]/, $netdev);
2630d2a9 3954
bf5aef9b
DC		 3955 if (defined(my $vhost = $options{vhost})) {
3956 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3957 }
3958
3959 if (defined(my $queues = $options{queues})) {
3960 $options{queues} = $queues + 0;
3961 }
3962
0a13e08e 3963 mon_cmd($vmid, "netdev_add", %options);
73aa03b8 3964 return 1;
2630d2a9
DA		 3965}
3966
3967sub qemu_netdevdel {
3968 my ($vmid, $deviceid) = @_;
3969
0a13e08e 3970 mon_cmd($vmid, "netdev_del", id => $deviceid);
2630d2a9
DA		 3971}
3972
16521d63 3973sub qemu_usb_hotplug {
d559309f 3974 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
16521d63
DC		 3975
3976 return if !$device;
3977
3978 # remove the old one first
3979 vm_deviceunplug($vmid, $conf, $deviceid);
3980
3981 # check if xhci controller is necessary and available
3982 if ($device->{usb3}) {
3983
3984 my $devicelist = vm_devices_list($vmid);
3985
3986 if (!$devicelist->{xhci}) {
d559309f 3987 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
16521d63
DC		 3988 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3989 }
3990 }
3991 my $d = parse_usb_device($device->{host});
3992 $d->{usb3} = $device->{usb3};
3993
3994 # add the new one
d559309f 3995 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
16521d63
DC		 3996}
3997
838776ab 3998sub qemu_cpu_hotplug {
8edc9c08 3999 my ($vmid, $conf, $vcpus) = @_;
838776ab 4000
3392d6ca 4001 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
1e881b75 4002
8edc9c08
AD		 4003 my $sockets = 1;
4004 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4005 $sockets = $conf->{sockets} if $conf->{sockets};
4006 my $cores = $conf->{cores} || 1;
4007 my $maxcpus = $sockets * $cores;
838776ab 4008
8edc9c08 4009 $vcpus = $maxcpus if !$vcpus;
3a11fadb 4010
8edc9c08
AD		 4011 die "you can't add more vcpus than maxcpus\n"
4012 if $vcpus > $maxcpus;
3a11fadb 4013
8edc9c08 4014 my $currentvcpus = $conf->{vcpus} || $maxcpus;
1e881b75 4015
eba3e64d 4016 if ($vcpus < $currentvcpus) {
1e881b75 4017
2ea5fb7e 4018 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
1e881b75
AD		 4019
4020 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4021 qemu_devicedel($vmid, "cpu$i");
4022 my $retry = 0;
4023 my $currentrunningvcpus = undef;
4024 while (1) {
65af8c31 4025 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
1e881b75 4026 last if scalar(@{$currentrunningvcpus}) == $i-1;
961af8a3 4027 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
1e881b75
AD		 4028 $retry++;
4029 sleep 1;
4030 }
4031 #update conf after each succesfull cpu unplug
4032 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4033 PVE::QemuConfig->write_config($vmid, $conf);
4034 }
4035 } else {
961af8a3 4036 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
1e881b75
AD		 4037 }
4038
4039 return;
4040 }
838776ab 4041
65af8c31 4042 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
961af8a3 4043 die "vcpus in running vm does not match its configuration\n"
8edc9c08 4044 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
838776ab 4045
2ea5fb7e 4046 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
eba3e64d
AD		 4047
4048 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4049 my $cpustr = print_cpu_device($conf, $i);
4050 qemu_deviceadd($vmid, $cpustr);
4051
4052 my $retry = 0;
4053 my $currentrunningvcpus = undef;
4054 while (1) {
65af8c31 4055 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
eba3e64d 4056 last if scalar(@{$currentrunningvcpus}) == $i;
961af8a3 4057 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
eba3e64d
AD		 4058 sleep 1;
4059 $retry++;
4060 }
4061 #update conf after each succesfull cpu hotplug
4062 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4063 PVE::QemuConfig->write_config($vmid, $conf);
4064 }
4065 } else {
4066
4067 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
0a13e08e 4068 mon_cmd($vmid, "cpu-add", id => int($i));
eba3e64d 4069 }
838776ab
AD		 4070 }
4071}
4072
affd2f88 4073sub qemu_block_set_io_throttle {
277ca170
WB		 4074 my ($vmid, $deviceid,
4075 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
9196a8ec
WB		 4076 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4077 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4078 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
affd2f88 4079
f3f323a3
AD		 4080 return if !check_running($vmid) ;
4081
0a13e08e 4082 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
277ca170
WB		 4083 bps => int($bps),
4084 bps_rd => int($bps_rd),
4085 bps_wr => int($bps_wr),
4086 iops => int($iops),
4087 iops_rd => int($iops_rd),
4088 iops_wr => int($iops_wr),
4089 bps_max => int($bps_max),
4090 bps_rd_max => int($bps_rd_max),
4091 bps_wr_max => int($bps_wr_max),
4092 iops_max => int($iops_max),
4093 iops_rd_max => int($iops_rd_max),
9196a8ec
WB		 4094 iops_wr_max => int($iops_wr_max),
4095 bps_max_length => int($bps_max_length),
4096 bps_rd_max_length => int($bps_rd_max_length),
4097 bps_wr_max_length => int($bps_wr_max_length),
4098 iops_max_length => int($iops_max_length),
4099 iops_rd_max_length => int($iops_rd_max_length),
4100 iops_wr_max_length => int($iops_wr_max_length),
277ca170 4101 );
f3f323a3 4102
affd2f88
AD		 4103}
4104
f5eb281a 4105# old code, only used to shutdown old VM after update
dab36e1e
DM		 4106sub __read_avail {
4107 my ($fh, $timeout) = @_;
4108
4109 my $sel = new IO::Select;
4110 $sel->add($fh);
4111
4112 my $res = '';
4113 my $buf;
4114
4115 my @ready;
4116 while (scalar (@ready = $sel->can_read($timeout))) {
4117 my $count;
4118 if ($count = $fh->sysread($buf, 8192)) {
4119 if ($buf =~ /^(.*)\(qemu\) $/s) {
4120 $res .= $1;
4121 last;
4122 } else {
4123 $res .= $buf;
4124 }
4125 } else {
4126 if (!defined($count)) {
4127 die "$!\n";
4128 }
4129 last;
4130 }
4131 }
4132
4133 die "monitor read timeout\n" if !scalar(@ready);
f5eb281a 4134
dab36e1e
DM		 4135 return $res;
4136}
4137
c1175c92
AD		 4138sub qemu_block_resize {
4139 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4140
ed221350 4141 my $running = check_running($vmid);
c1175c92 4142
7246e8f9 4143 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
c1175c92
AD		 4144
4145 return if !$running;
4146
375db731
FE		 4147 my $padding = (1024 - $size % 1024) % 1024;
4148 $size = $size + $padding;
4149
0a13e08e 4150 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
c1175c92
AD		 4151
4152}
4153
1ab0057c
AD		 4154sub qemu_volume_snapshot {
4155 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4156
ed221350 4157 my $running = check_running($vmid);
1ab0057c 4158
e5eaa028 4159 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
0a13e08e 4160 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
e5eaa028
WL		 4161 } else {
4162 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4163 }
1ab0057c
AD		 4164}
4165
fc46aff9
AD		 4166sub qemu_volume_snapshot_delete {
4167 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4168
ed221350 4169 my $running = check_running($vmid);
fc46aff9 4170
a2f1554b
AD		 4171 if($running) {
4172
4173 $running = undef;
4174 my $conf = PVE::QemuConfig->load_config($vmid);
912792e2 4175 PVE::QemuConfig->foreach_volume($conf, sub {
a2f1554b
AD		 4176 my ($ds, $drive) = @_;
4177 $running = 1 if $drive->{file} eq $volid;
4178 });
4179 }
4180
1ef7592f 4181 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
0a13e08e 4182 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
1ef7592f
AD		 4183 } else {
4184 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4185 }
fc46aff9
AD		 4186}
4187
264e519f
DM		 4188sub set_migration_caps {
4189 my ($vmid) = @_;
a89fded1 4190
8b8345f3 4191 my $cap_ref = [];
a89fded1
AD		 4192
4193 my $enabled_cap = {
8b8345f3 4194 "auto-converge" => 1,
0b0a47e8 4195 "xbzrle" => 1,
8b8345f3
DM		 4196 "x-rdma-pin-all" => 0,
4197 "zero-blocks" => 0,
b62532e4 4198 "compress" => 0
a89fded1
AD		 4199 };
4200
0a13e08e 4201 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
a89fded1 4202
8b8345f3 4203 for my $supported_capability (@$supported_capabilities) {
b463a3ce
SP		 4204 push @$cap_ref, {
4205 capability => $supported_capability->{capability},
22430fa2
DM		 4206 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4207 };
a89fded1
AD		 4208 }
4209
0a13e08e 4210 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
8b8345f3 4211}
a89fded1 4212
912792e2
FE		 4213sub foreach_volid {
4214 my ($conf, $func, @param) = @_;
4215
4216 my $volhash = {};
4217
4218 my $test_volid = sub {
ae180b8f 4219 my ($key, $drive, $snapname) = @_;
912792e2 4220
ae180b8f 4221 my $volid = $drive->{file};
912792e2
FE		 4222 return if !$volid;
4223
4224 $volhash->{$volid}->{cdrom} //= 1;
ae180b8f 4225 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
912792e2 4226
ae180b8f 4227 my $replicate = $drive->{replicate} // 1;
912792e2
FE		 4228 $volhash->{$volid}->{replicate} //= 0;
4229 $volhash->{$volid}->{replicate} = 1 if $replicate;
4230
4231 $volhash->{$volid}->{shared} //= 0;
ae180b8f 4232 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
912792e2
FE		 4233
4234 $volhash->{$volid}->{referenced_in_config} //= 0;
4235 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4236
4237 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4238 if defined($snapname);
ae180b8f
FE		 4239
4240 my $size = $drive->{size};
4241 $volhash->{$volid}->{size} //= $size if $size;
4242
4243 $volhash->{$volid}->{is_vmstate} //= 0;
4244 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4245
4246 $volhash->{$volid}->{is_unused} //= 0;
4247 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
912792e2
FE		 4248 };
4249
ae180b8f
FE		 4250 my $include_opts = {
4251 extra_keys => ['vmstate'],
4252 include_unused => 1,
4253 };
4254
0b953b8e 4255 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
912792e2
FE		 4256 foreach my $snapname (keys %{$conf->{snapshots}}) {
4257 my $snap = $conf->{snapshots}->{$snapname};
0b953b8e 4258 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
912792e2
FE		 4259 }
4260
4261 foreach my $volid (keys %$volhash) {
4262 &$func($volid, $volhash->{$volid}, @param);
4263 }
4264}
4265
81d95ae1 4266my $fast_plug_option = {
7498eb64 4267 'lock' => 1,
81d95ae1 4268 'name' => 1,
a1b7d579 4269 'onboot' => 1,
81d95ae1
DM		 4270 'shares' => 1,
4271 'startup' => 1,
b0ec896e 4272 'description' => 1,
ec647db4 4273 'protection' => 1,
8cad5e9b 4274 'vmstatestorage' => 1,
9e784b11 4275 'hookscript' => 1,
b8e7068a 4276 'tags' => 1,
81d95ae1
DM		 4277};
4278
3a11fadb
DM		 4279# hotplug changes in [PENDING]
4280# $selection hash can be used to only apply specified options, for
4281# example: { cores => 1 } (only apply changed 'cores')
4282# $errors ref is used to return error messages
c427973b 4283sub vmconfig_hotplug_pending {
3a11fadb 4284 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
c427973b 4285
8e90138a 4286 my $defaults = load_defaults();
045749f2
TL		 4287 my $arch = get_vm_arch($conf);
4288 my $machine_type = get_vm_machine($conf, undef, $arch);
c427973b
DM		 4289
4290 # commit values which do not have any impact on running VM first
3a11fadb
DM		 4291 # Note: those option cannot raise errors, we we do not care about
4292 # $selection and always apply them.
4293
4294 my $add_error = sub {
4295 my ($opt, $msg) = @_;
4296 $errors->{$opt} = "hotplug problem - $msg";
4297 };
c427973b
DM		 4298
4299 my $changes = 0;
4300 foreach my $opt (keys %{$conf->{pending}}) { # add/change
81d95ae1 4301 if ($fast_plug_option->{$opt}) {
c427973b
DM		 4302 $conf->{$opt} = $conf->{pending}->{$opt};
4303 delete $conf->{pending}->{$opt};
4304 $changes = 1;
4305 }
4306 }
4307
4308 if ($changes) {
ffda963f 4309 PVE::QemuConfig->write_config($vmid, $conf);
c427973b
DM		 4310 }
4311
b3c2bdd1 4312 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
c427973b 4313
98bc3aeb 4314 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
d321c4a9 4315 foreach my $opt (sort keys %$pending_delete_hash) {
3a11fadb 4316 next if $selection && !$selection->{$opt};
d321c4a9 4317 my $force = $pending_delete_hash->{$opt}->{force};
3a11fadb 4318 eval {
51a6f637
AD		 4319 if ($opt eq 'hotplug') {
4320 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4321 } elsif ($opt eq 'tablet') {
b3c2bdd1 4322 die "skip\n" if !$hotplug_features->{usb};
3a11fadb 4323 if ($defaults->{tablet}) {
d559309f
WB		 4324 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4325 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4326 if $arch eq 'aarch64';
3a11fadb 4327 } else {
d559309f
WB		 4328 vm_deviceunplug($vmid, $conf, 'tablet');
4329 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
3a11fadb 4330 }
4eb68604 4331 } elsif ($opt =~ m/^usb\d+/) {
f745762b
DC		 4332 die "skip\n";
4333 # since we cannot reliably hot unplug usb devices
4334 # we are disabling it
4eb68604
DC		 4335 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4336 vm_deviceunplug($vmid, $conf, $opt);
8edc9c08 4337 } elsif ($opt eq 'vcpus') {
b3c2bdd1 4338 die "skip\n" if !$hotplug_features->{cpu};
8edc9c08 4339 qemu_cpu_hotplug($vmid, $conf, undef);
9c2f7069 4340 } elsif ($opt eq 'balloon') {
81d95ae1 4341 # enable balloon device is not hotpluggable
75b51053
DC		 4342 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4343 # here we reset the ballooning value to memory
4344 my $balloon = $conf->{memory} || $defaults->{memory};
0a13e08e 4345 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
81d95ae1
DM		 4346 } elsif ($fast_plug_option->{$opt}) {
4347 # do nothing
3eec5767 4348 } elsif ($opt =~ m/^net(\d+)$/) {
b3c2bdd1 4349 die "skip\n" if !$hotplug_features->{network};
3eec5767 4350 vm_deviceunplug($vmid, $conf, $opt);
74479ee9 4351 } elsif (is_valid_drivename($opt)) {
b3c2bdd1 4352 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
19120f99 4353 vm_deviceunplug($vmid, $conf, $opt);
3dc38fbb 4354 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4d3f29ed
AD		 4355 } elsif ($opt =~ m/^memory$/) {
4356 die "skip\n" if !$hotplug_features->{memory};
6779f1ac 4357 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
c8effec3
AD		 4358 } elsif ($opt eq 'cpuunits') {
4359 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
58be00f1
AD		 4360 } elsif ($opt eq 'cpulimit') {
4361 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3d7389fe 4362 } else {
e56beeda 4363 die "skip\n";
3d7389fe 4364 }
3a11fadb
DM		 4365 };
4366 if (my $err = $@) {
e56beeda
DM		 4367 &$add_error($opt, $err) if $err ne "skip\n";
4368 } else {
3a11fadb 4369 delete $conf->{$opt};
98bc3aeb 4370 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
3d7389fe 4371 }
3d7389fe
DM		 4372 }
4373
e5a66e48 4374 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
9ed7a77c 4375 $apply_pending_cloudinit = sub {
e5a66e48
WB		 4376 return if $apply_pending_cloudinit_done; # once is enough
4377 $apply_pending_cloudinit_done = 1; # once is enough
4378
9ed7a77c 4379 my ($key, $value) = @_;
9ed7a77c
WB		 4380
4381 my @cloudinit_opts = keys %$confdesc_cloudinit;
4382 foreach my $opt (keys %{$conf->{pending}}) {
4383 next if !grep { $_ eq $opt } @cloudinit_opts;
4384 $conf->{$opt} = delete $conf->{pending}->{$opt};
4385 }
4386
4387 my $new_conf = { %$conf };
4388 $new_conf->{$key} = $value;
4389 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4390 };
4391
3d7389fe 4392 foreach my $opt (keys %{$conf->{pending}}) {
3a11fadb 4393 next if $selection && !$selection->{$opt};
3d7389fe 4394 my $value = $conf->{pending}->{$opt};
3a11fadb 4395 eval {
51a6f637
AD		 4396 if ($opt eq 'hotplug') {
4397 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4398 } elsif ($opt eq 'tablet') {
b3c2bdd1 4399 die "skip\n" if !$hotplug_features->{usb};
3a11fadb 4400 if ($value == 1) {
d559309f
WB		 4401 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4402 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4403 if $arch eq 'aarch64';
3a11fadb 4404 } elsif ($value == 0) {
d559309f
WB		 4405 vm_deviceunplug($vmid, $conf, 'tablet');
4406 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
3a11fadb 4407 }
4eb68604 4408 } elsif ($opt =~ m/^usb\d+$/) {
f745762b
DC		 4409 die "skip\n";
4410 # since we cannot reliably hot unplug usb devices
4411 # we are disabling it
4eb68604
DC		 4412 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4413 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4414 die "skip\n" if !$d;
d559309f 4415 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
8edc9c08 4416 } elsif ($opt eq 'vcpus') {
b3c2bdd1 4417 die "skip\n" if !$hotplug_features->{cpu};
3a11fadb
DM		 4418 qemu_cpu_hotplug($vmid, $conf, $value);
4419 } elsif ($opt eq 'balloon') {
81d95ae1 4420 # enable/disable balloning device is not hotpluggable
8fe689e7 4421 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
a1b7d579 4422 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
81d95ae1
DM		 4423 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4424
3a11fadb 4425 # allow manual ballooning if shares is set to zero
4cc1efa6 4426 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
9c2f7069 4427 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
0a13e08e 4428 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
9c2f7069 4429 }
a1b7d579 4430 } elsif ($opt =~ m/^net(\d+)$/) {
3eec5767 4431 # some changes can be done without hotplug
a1b7d579 4432 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
d559309f 4433 $vmid, $opt, $value, $arch, $machine_type);
74479ee9 4434 } elsif (is_valid_drivename($opt)) {
7a4bdb36 4435 die "skip\n" if $opt eq 'efidisk0';
a05cff86 4436 # some changes can be done without hotplug
9ed7a77c
WB		 4437 my $drive = parse_drive($opt, $value);
4438 if (drive_is_cloudinit($drive)) {
4439 &$apply_pending_cloudinit($opt, $value);
4440 }
b3c2bdd1 4441 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
9e7bce2c 4442 $vmid, $opt, $value, $arch, $machine_type);
4d3f29ed
AD		 4443 } elsif ($opt =~ m/^memory$/) { #dimms
4444 die "skip\n" if !$hotplug_features->{memory};
6779f1ac 4445 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
c8effec3
AD		 4446 } elsif ($opt eq 'cpuunits') {
4447 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
58be00f1 4448 } elsif ($opt eq 'cpulimit') {
c6f773b8 4449 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
58be00f1 4450 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3a11fadb 4451 } else {
e56beeda 4452 die "skip\n"; # skip non-hot-pluggable options
3d7389fe 4453 }
3a11fadb
DM		 4454 };
4455 if (my $err = $@) {
e56beeda
DM		 4456 &$add_error($opt, $err) if $err ne "skip\n";
4457 } else {
3a11fadb
DM		 4458 $conf->{$opt} = $value;
4459 delete $conf->{pending}->{$opt};
3d7389fe 4460 }
3d7389fe 4461 }
4df15a03
OB		 4462
4463 PVE::QemuConfig->write_config($vmid, $conf);
c427973b 4464}
055d554d 4465
3dc38fbb
WB		 4466sub try_deallocate_drive {
4467 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4468
4469 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4470 my $volid = $drive->{file};
4471 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4472 my $sid = PVE::Storage::parse_volume_id($volid);
4473 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
cee01bcb
WB		 4474
4475 # check if the disk is really unused
cee01bcb 4476 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
e0fd2b2f 4477 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
cee01bcb 4478 PVE::Storage::vdisk_free($storecfg, $volid);
3dc38fbb 4479 return 1;
40b977f3
WL		 4480 } else {
4481 # If vm is not owner of this disk remove from config
4482 return 1;
3dc38fbb
WB		 4483 }
4484 }
4485
4486 return undef;
4487}
4488
4489sub vmconfig_delete_or_detach_drive {
4490 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4491
4492 my $drive = parse_drive($opt, $conf->{$opt});
4493
4494 my $rpcenv = PVE::RPCEnvironment::get();
4495 my $authuser = $rpcenv->get_user();
4496
4497 if ($force) {
4498 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4499 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4500 } else {
4501 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4502 }
4503}
4504
98bc3aeb
OB		 4505
4506
055d554d 4507sub vmconfig_apply_pending {
eb5e482d
OB		 4508 my ($vmid, $conf, $storecfg, $errors) = @_;
4509
4510 my $add_apply_error = sub {
4511 my ($opt, $msg) = @_;
4512 my $err_msg = "unable to apply pending change $opt : $msg";
4513 $errors->{$opt} = $err_msg;
4514 warn $err_msg;
4515 };
c427973b
DM		 4516
4517 # cold plug
055d554d 4518
98bc3aeb 4519 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
d321c4a9 4520 foreach my $opt (sort keys %$pending_delete_hash) {
fb4d1ba2 4521 my $force = $pending_delete_hash->{$opt}->{force};
eb5e482d 4522 eval {
3d48b95a
OB		 4523 if ($opt =~ m/^unused/) {
4524 die "internal error";
4525 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
eb5e482d 4526 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
eb5e482d
OB		 4527 }
4528 };
4529 if (my $err = $@) {
4530 $add_apply_error->($opt, $err);
055d554d 4531 } else {
98bc3aeb 4532 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
055d554d 4533 delete $conf->{$opt};
055d554d
DM		 4534 }
4535 }
4536
3d48b95a 4537 PVE::QemuConfig->cleanup_pending($conf);
055d554d
DM		 4538
4539 foreach my $opt (keys %{$conf->{pending}}) { # add/change
3d48b95a 4540 next if $opt eq 'delete'; # just to be sure
eb5e482d 4541 eval {
3d48b95a 4542 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
eb5e482d 4543 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
eb5e482d
OB		 4544 }
4545 };
4546 if (my $err = $@) {
4547 $add_apply_error->($opt, $err);
055d554d 4548 } else {
eb5e482d 4549 $conf->{$opt} = delete $conf->{pending}->{$opt};
055d554d 4550 }
055d554d 4551 }
3d48b95a
OB		 4552
4553 # write all changes at once to avoid unnecessary i/o
4554 PVE::QemuConfig->write_config($vmid, $conf);
055d554d
DM		 4555}
4556
3eec5767 4557sub vmconfig_update_net {
d559309f 4558 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
3eec5767
DM		 4559
4560 my $newnet = parse_net($value);
4561
4562 if ($conf->{$opt}) {
4563 my $oldnet = parse_net($conf->{$opt});
4564
0f1af9e7
OB		 4565 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4566 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4567 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
3eec5767
DM		 4568 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4569
4570 # for non online change, we try to hot-unplug
7196b757 4571 die "skip\n" if !$hotplug;
3eec5767
DM		 4572 vm_deviceunplug($vmid, $conf, $opt);
4573 } else {
4574
4575 die "internal error" if $opt !~ m/net(\d+)/;
4576 my $iface = "tap${vmid}i$1";
a1b7d579 4577
0f1af9e7
OB		 4578 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4579 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4580 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4581 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
3eec5767 4582 PVE::Network::tap_unplug($iface);
28e129cc
AD		 4583
4584 if ($have_sdn) {
4585 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4586 } else {
4587 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4588 }
0f1af9e7 4589 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4f4fbeb0
WB		 4590 # Rate can be applied on its own but any change above needs to
4591 # include the rate in tap_plug since OVS resets everything.
4592 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
3eec5767 4593 }
38c590d9 4594
0f1af9e7 4595 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
25088687
DM		 4596 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4597 }
4598
38c590d9 4599 return 1;
3eec5767
DM		 4600 }
4601 }
a1b7d579 4602
7196b757 4603 if ($hotplug) {
d559309f 4604 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
38c590d9
DM		 4605 } else {
4606 die "skip\n";
4607 }
3eec5767
DM		 4608}
4609
a05cff86 4610sub vmconfig_update_disk {
9e7bce2c 4611 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
a05cff86
DM		 4612
4613 my $drive = parse_drive($opt, $value);
4614
4615 if ($conf->{$opt}) {
4616
4617 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4618
4619 my $media = $drive->{media} || 'disk';
4620 my $oldmedia = $old_drive->{media} || 'disk';
4621 die "unable to change media type\n" if $media ne $oldmedia;
4622
4623 if (!drive_is_cdrom($old_drive)) {
4624
a1b7d579 4625 if ($drive->{file} ne $old_drive->{file}) {
a05cff86 4626
7196b757 4627 die "skip\n" if !$hotplug;
a05cff86
DM		 4628
4629 # unplug and register as unused
4630 vm_deviceunplug($vmid, $conf, $opt);
4631 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
a1b7d579 4632
a05cff86
DM		 4633 } else {
4634 # update existing disk
4635
4636 # skip non hotpluggable value
0f1af9e7
OB		 4637 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4638 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4639 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4640 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4641 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
a05cff86
DM		 4642 die "skip\n";
4643 }
4644
4645 # apply throttle
0f1af9e7
OB		 4646 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4647 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4648 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4649 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4650 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4651 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4652 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4653 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4654 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4655 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4656 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4657 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4658 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4659 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4660 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4661 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4662 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4663 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
a1b7d579 4664
a05cff86
DM		 4665 qemu_block_set_io_throttle($vmid,"drive-$opt",
4666 ($drive->{mbps} || 0)*1024*1024,
4667 ($drive->{mbps_rd} || 0)*1024*1024,
4668 ($drive->{mbps_wr} || 0)*1024*1024,
4669 $drive->{iops} || 0,
4670 $drive->{iops_rd} || 0,
4671 $drive->{iops_wr} || 0,
4672 ($drive->{mbps_max} || 0)*1024*1024,
4673 ($drive->{mbps_rd_max} || 0)*1024*1024,
4674 ($drive->{mbps_wr_max} || 0)*1024*1024,
4675 $drive->{iops_max} || 0,
4676 $drive->{iops_rd_max} || 0,
9196a8ec
WB		 4677 $drive->{iops_wr_max} || 0,
4678 $drive->{bps_max_length} || 1,
4679 $drive->{bps_rd_max_length} || 1,
4680 $drive->{bps_wr_max_length} || 1,
4681 $drive->{iops_max_length} || 1,
4682 $drive->{iops_rd_max_length} || 1,
4683 $drive->{iops_wr_max_length} || 1);
a05cff86
DM		 4684
4685 }
a1b7d579 4686
a05cff86
DM		 4687 return 1;
4688 }
4de1bb25
DM		 4689
4690 } else { # cdrom
a1b7d579 4691
4de1bb25 4692 if ($drive->{file} eq 'none') {
922869f2 4693 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
2d9ddec5
WB		 4694 if (drive_is_cloudinit($old_drive)) {
4695 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4696 }
4de1bb25
DM		 4697 } else {
4698 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
ce9fce79
SR		 4699
4700 # force eject if locked
922869f2 4701 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
ce9fce79
SR		 4702
4703 if ($path) {
4704 mon_cmd($vmid, "blockdev-change-medium",
922869f2 4705 id => "$opt", filename => "$path");
ce9fce79 4706 }
4de1bb25 4707 }
a1b7d579 4708
34758d66 4709 return 1;
a05cff86
DM		 4710 }
4711 }
4712 }
4713
a1b7d579 4714 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4de1bb25 4715 # hotplug new disks
f7b4356f 4716 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
d559309f 4717 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
a05cff86
DM		 4718}
4719
13cfe3b7 4720# called in locked context by incoming migration
ba5396b5
FG		 4721sub vm_migrate_get_nbd_disks {
4722 my ($storecfg, $conf, $replicated_volumes) = @_;
13cfe3b7
FG		 4723
4724 my $local_volumes = {};
912792e2 4725 PVE::QemuConfig->foreach_volume($conf, sub {
13cfe3b7
FG		 4726 my ($ds, $drive) = @_;
4727
4728 return if drive_is_cdrom($drive);
4729
4730 my $volid = $drive->{file};
4731
4732 return if !$volid;
4733
4734 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4735
4736 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4737 return if $scfg->{shared};
ba5396b5
FG		 4738
4739 # replicated disks re-use existing state via bitmap
4740 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4741 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
13cfe3b7 4742 });
ba5396b5
FG		 4743 return $local_volumes;
4744}
4745
4746# called in locked context by incoming migration
4747sub vm_migrate_alloc_nbd_disks {
4748 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
13cfe3b7
FG		 4749
4750 my $format = undef;
4751
4752 my $nbd = {};
ba5396b5
FG		 4753 foreach my $opt (sort keys %$source_volumes) {
4754 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4755
4756 if ($use_existing) {
4757 $nbd->{$opt}->{drivestr} = print_drive($drive);
4758 $nbd->{$opt}->{volid} = $volid;
4759 $nbd->{$opt}->{replicated} = 1;
13cfe3b7
FG		 4760 next;
4761 }
13cfe3b7
FG		 4762
4763 # If a remote storage is specified and the format of the original
4764 # volume is not available there, fall back to the default format.
4765 # Otherwise use the same format as the original.
bf8fc5a3
FG		 4766 if (!$storagemap->{identity}) {
4767 $storeid = map_storage($storagemap, $storeid);
13cfe3b7
FG		 4768 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4769 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4770 my $fileFormat = qemu_img_format($scfg, $volname);
4771 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4772 } else {
4773 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4774 $format = qemu_img_format($scfg, $volname);
4775 }
4776
4777 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4778 my $newdrive = $drive;
4779 $newdrive->{format} = $format;
4780 $newdrive->{file} = $newvolid;
4781 my $drivestr = print_drive($newdrive);
ba5396b5
FG		 4782 $nbd->{$opt}->{drivestr} = $drivestr;
4783 $nbd->{$opt}->{volid} = $newvolid;
13cfe3b7
FG		 4784 }
4785
4786 return $nbd;
4787}
4788
4789# see vm_start_nolock for parameters, additionally:
4790# migrate_opts:
bf8fc5a3 4791# storagemap = parsed storage map for allocating NBD disks
3898a563
FG		 4792sub vm_start {
4793 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4794
84da8217 4795 return PVE::QemuConfig->lock_config($vmid, sub {
3898a563
FG		 4796 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4797
4ef13a7f
FG		 4798 die "you can't start a vm if it's a template\n"
4799 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
3898a563 4800
d544e0e0 4801 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
3898a563
FG		 4802
4803 PVE::QemuConfig->check_lock($conf)
d544e0e0
FE		 4804 if !($params->{skiplock} || $has_suspended_lock);
4805
4806 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
3898a563
FG		 4807
4808 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4809
ba5396b5
FG		 4810 if (my $storagemap = $migrate_opts->{storagemap}) {
4811 my $replicated = $migrate_opts->{replicated_volumes};
4812 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4813 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4814
4815 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4816 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4817 }
4818 }
13cfe3b7 4819
84da8217 4820 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
3898a563
FG		 4821 });
4822}
4823
4824
0c498cca
FG		 4825# params:
4826# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4827# skiplock => 0/1, skip checking for config lock
4ef13a7f 4828# skiptemplate => 0/1, skip checking whether VM is template
0c498cca 4829# forcemachine => to force Qemu machine (rollback/migration)
58c64ad5 4830# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
0c498cca
FG		 4831# timeout => in seconds
4832# paused => start VM in paused state (backup)
3898a563 4833# resume => resume from hibernation
0c498cca 4834# migrate_opts:
ba5396b5 4835# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
0c498cca
FG		 4836# migratedfrom => source node
4837# spice_ticket => used for spice migration, passed via tunnel/stdin
4838# network => CIDR of migration network
4839# type => secure/insecure - tunnel over encrypted connection or plain-text
0c498cca
FG		 4840# nbd_proto_version => int, 0 for TCP, 1 for UNIX
4841# replicated_volumes = which volids should be re-used with bitmaps for nbd migration
3898a563
FG		 4842sub vm_start_nolock {
4843 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
1e3baf05 4844
3898a563
FG		 4845 my $statefile = $params->{statefile};
4846 my $resume = $params->{resume};
3dcb98d5 4847
3898a563
FG		 4848 my $migratedfrom = $migrate_opts->{migratedfrom};
4849 my $migration_type = $migrate_opts->{type};
7ceade4c 4850
84da8217
FG		 4851 my $res = {};
4852
3898a563
FG		 4853 # clean up leftover reboot request files
4854 eval { clear_reboot_request($vmid); };
4855 warn $@ if $@;
1e3baf05 4856
3898a563
FG		 4857 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4858 vmconfig_apply_pending($vmid, $conf, $storecfg);
4859 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4860 }
64457ed4 4861
3898a563 4862 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
055d554d 4863
3898a563 4864 my $defaults = load_defaults();
0c9a7596 4865
3898a563
FG		 4866 # set environment variable useful inside network script
4867 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
6c47d546 4868
3898a563 4869 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
9e784b11 4870
3898a563 4871 my $forcemachine = $params->{forcemachine};
ea1c2110 4872 my $forcecpu = $params->{forcecpu};
3898a563 4873 if ($resume) {
ea1c2110 4874 # enforce machine and CPU type on suspended vm to ensure HW compatibility
3898a563 4875 $forcemachine = $conf->{runningmachine};
ea1c2110 4876 $forcecpu = $conf->{runningcpu};
3898a563
FG		 4877 print "Resuming suspended VM\n";
4878 }
7ceade4c 4879
2f6f002c
TL		 4880 my ($cmd, $vollist, $spice_port) =
4881 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6c47d546 4882
3898a563
FG		 4883 my $migration_ip;
4884 my $get_migration_ip = sub {
4885 my ($nodename) = @_;
b24e1ac2 4886
3898a563 4887 return $migration_ip if defined($migration_ip);
b24e1ac2 4888
3898a563 4889 my $cidr = $migrate_opts->{network};
0c498cca 4890
3898a563
FG		 4891 if (!defined($cidr)) {
4892 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4893 $cidr = $dc_conf->{migration}->{network};
4894 }
b24e1ac2 4895
3898a563
FG		 4896 if (defined($cidr)) {
4897 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
b24e1ac2 4898
3898a563
FG		 4899 die "could not get IP: no address configured on local " .
4900 "node for network '$cidr'\n" if scalar(@$ips) == 0;
b24e1ac2 4901
3898a563
FG		 4902 die "could not get IP: multiple addresses configured on local " .
4903 "node for network '$cidr'\n" if scalar(@$ips) > 1;
b24e1ac2 4904
3898a563
FG		 4905 $migration_ip = @$ips[0];
4906 }
b24e1ac2 4907
3898a563
FG		 4908 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4909 if !defined($migration_ip);
b24e1ac2 4910
3898a563
FG		 4911 return $migration_ip;
4912 };
b24e1ac2 4913
3898a563
FG		 4914 my $migrate_uri;
4915 if ($statefile) {
4916 if ($statefile eq 'tcp') {
4917 my $localip = "localhost";
4918 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4919 my $nodename = nodename();
2de2d6f7 4920
3898a563
FG		 4921 if (!defined($migration_type)) {
4922 if (defined($datacenterconf->{migration}->{type})) {
4923 $migration_type = $datacenterconf->{migration}->{type};
4924 } else {
4925 $migration_type = 'secure';
b7a5a225 4926 }
3898a563 4927 }
b7a5a225 4928
3898a563
FG		 4929 if ($migration_type eq 'insecure') {
4930 $localip = $get_migration_ip->($nodename);
4931 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4932 }
2de2d6f7 4933
3898a563
FG		 4934 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4935 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4936 $migrate_uri = "tcp:${localip}:${migrate_port}";
4937 push @$cmd, '-incoming', $migrate_uri;
4938 push @$cmd, '-S';
1c9d54bf 4939
3898a563
FG		 4940 } elsif ($statefile eq 'unix') {
4941 # should be default for secure migrations as a ssh TCP forward
4942 # tunnel is not deterministic reliable ready and fails regurarly
4943 # to set up in time, so use UNIX socket forwards
4944 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4945 unlink $socket_addr;
54323eed 4946
3898a563 4947 $migrate_uri = "unix:$socket_addr";
1c9d54bf 4948
3898a563
FG		 4949 push @$cmd, '-incoming', $migrate_uri;
4950 push @$cmd, '-S';
1c9d54bf 4951
3898a563
FG		 4952 } elsif (-e $statefile) {
4953 push @$cmd, '-loadstate', $statefile;
4954 } else {
4955 my $statepath = PVE::Storage::path($storecfg, $statefile);
4956 push @$vollist, $statefile;
4957 push @$cmd, '-loadstate', $statepath;
4958 }
4959 } elsif ($params->{paused}) {
4960 push @$cmd, '-S';
4961 }
4962
4963 # host pci devices
74c17b7a 4964 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
3898a563
FG		 4965 my $d = parse_hostpci($conf->{"hostpci$i"});
4966 next if !$d;
4967 my $pcidevices = $d->{pciid};
4968 foreach my $pcidevice (@$pcidevices) {
4969 my $pciid = $pcidevice->{id};
4970
4971 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4972 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4973 die "no pci device info for device '$pciid'\n" if !$info;
4974
4975 if ($d->{mdev}) {
4976 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4977 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5c1d42b7 4978 } else {
3898a563
FG		 4979 die "can't unbind/bind pci group to vfio '$pciid'\n"
4980 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4981 die "can't reset pci device '$pciid'\n"
4982 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
1e3baf05 4983 }
3898a563
FG		 4984 }
4985 }
1e3baf05 4986
3898a563 4987 PVE::Storage::activate_volumes($storecfg, $vollist);
1e3baf05 4988
3898a563
FG		 4989 eval {
4990 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4991 outfunc => sub {}, errfunc => sub {});
4992 };
4993 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4994 # timeout should be more than enough here...
4995 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4996
4997 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4998 : $defaults->{cpuunits};
4999
5000 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5001 my %run_params = (
5002 timeout => $statefile ? undef : $start_timeout,
5003 umask => 0077,
5004 noerr => 1,
5005 );
1e3baf05 5006
3898a563
FG		 5007 # when migrating, prefix QEMU output so other side can pick up any
5008 # errors that might occur and show the user
5009 if ($migratedfrom) {
5010 $run_params{quiet} = 1;
5011 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5012 }
8bf30c2a 5013
3898a563
FG		 5014 my %properties = (
5015 Slice => 'qemu.slice',
5016 KillMode => 'none',
5017 CPUShares => $cpuunits
5018 );
7023f3ea 5019
3898a563
FG		 5020 if (my $cpulimit = $conf->{cpulimit}) {
5021 $properties{CPUQuota} = int($cpulimit * 100);
5022 }
5023 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
7023f3ea 5024
3898a563
FG		 5025 my $run_qemu = sub {
5026 PVE::Tools::run_fork sub {
5027 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
6e0216d8 5028
3898a563
FG		 5029 my $exitcode = run_command($cmd, %run_params);
5030 die "QEMU exited with code $exitcode\n" if $exitcode;
503308ed 5031 };
3898a563 5032 };
503308ed 5033
3898a563 5034 if ($conf->{hugepages}) {
7023f3ea 5035
3898a563
FG		 5036 my $code = sub {
5037 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5038 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
7023f3ea 5039
3898a563
FG		 5040 PVE::QemuServer::Memory::hugepages_mount();
5041 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
7023f3ea 5042
503308ed 5043 eval { $run_qemu->() };
3898a563 5044 if (my $err = $@) {
f36e9894
SR		 5045 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5046 if !$conf->{keephugepages};
3898a563
FG		 5047 die $err;
5048 }
77cde36b 5049
f36e9894
SR		 5050 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5051 if !$conf->{keephugepages};
3898a563
FG		 5052 };
5053 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
1e3baf05 5054
3898a563
FG		 5055 } else {
5056 eval { $run_qemu->() };
5057 }
afdb31d5 5058
3898a563
FG		 5059 if (my $err = $@) {
5060 # deactivate volumes if start fails
5061 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5062 die "start failed: $err";
5063 }
62de2cbd 5064
3898a563 5065 print "migration listens on $migrate_uri\n" if $migrate_uri;
84da8217 5066 $res->{migrate_uri} = $migrate_uri;
eb8cddb5 5067
3898a563
FG		 5068 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5069 eval { mon_cmd($vmid, "cont"); };
5070 warn $@ if $@;
5071 }
2189246c 5072
3898a563 5073 #start nbd server for storage migration
13cfe3b7 5074 if (my $nbd = $migrate_opts->{nbd}) {
3898a563 5075 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
2189246c 5076
3898a563
FG		 5077 my $migrate_storage_uri;
5078 # nbd_protocol_version > 0 for unix socket support
5079 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5080 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5081 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5082 $migrate_storage_uri = "nbd:unix:$socket_path";
5083 } else {
5084 my $nodename = nodename();
5085 my $localip = $get_migration_ip->($nodename);
5086 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5087 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5088
5089 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5090 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5091 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
2189246c
AD		 5092 }
5093
84da8217
FG		 5094 $res->{migrate_storage_uri} = $migrate_storage_uri;
5095
13cfe3b7 5096 foreach my $opt (sort keys %$nbd) {
ba5396b5
FG		 5097 my $drivestr = $nbd->{$opt}->{drivestr};
5098 my $volid = $nbd->{$opt}->{volid};
3898a563 5099 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
84da8217
FG		 5100 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5101 print "storage migration listens on $nbd_uri volume:$drivestr\n";
ba5396b5
FG		 5102 print "re-using replicated volume: $opt - $volid\n"
5103 if $nbd->{$opt}->{replicated};
84da8217
FG		 5104
5105 $res->{drives}->{$opt} = $nbd->{$opt};
5106 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
3898a563
FG		 5107 }
5108 }
a89fded1 5109
3898a563
FG		 5110 if ($migratedfrom) {
5111 eval {
5112 set_migration_caps($vmid);
5113 };
5114 warn $@ if $@;
5115
5116 if ($spice_port) {
5117 print "spice listens on port $spice_port\n";
84da8217 5118 $res->{spice_port} = $spice_port;
3898a563
FG		 5119 if ($migrate_opts->{spice_ticket}) {
5120 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5121 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
95a4b4a9 5122 }
3898a563 5123 }
95a4b4a9 5124
3898a563
FG		 5125 } else {
5126 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5127 if !$statefile && $conf->{balloon};
25088687 5128
3898a563
FG		 5129 foreach my $opt (keys %$conf) {
5130 next if $opt !~ m/^net\d+$/;
5131 my $nicconf = parse_net($conf->{$opt});
5132 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
e18b0b99 5133 }
3898a563 5134 }
a1b7d579 5135
3898a563
FG		 5136 mon_cmd($vmid, 'qom-set',
5137 path => "machine/peripheral/balloon0",
5138 property => "guest-stats-polling-interval",
5139 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
eb065317 5140
3898a563
FG		 5141 if ($resume) {
5142 print "Resumed VM, removing state\n";
5143 if (my $vmstate = $conf->{vmstate}) {
5144 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5145 PVE::Storage::vdisk_free($storecfg, $vmstate);
7ceade4c 5146 }
ea1c2110 5147 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
3898a563
FG		 5148 PVE::QemuConfig->write_config($vmid, $conf);
5149 }
7ceade4c 5150
3898a563 5151 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
84da8217
FG		 5152
5153 return $res;
1e3baf05
DM		 5154}
5155
1e3baf05 5156sub vm_commandline {
b14477e7 5157 my ($storecfg, $vmid, $snapname) = @_;
1e3baf05 5158
ffda963f 5159 my $conf = PVE::QemuConfig->load_config($vmid);
092868c4 5160 my $forcemachine;
ea1c2110 5161 my $forcecpu;
1e3baf05 5162
b14477e7
RV		 5163 if ($snapname) {
5164 my $snapshot = $conf->{snapshots}->{$snapname};
87d92707
TL		 5165 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5166
ea1c2110
SR		 5167 # check for machine or CPU overrides in snapshot
5168 $forcemachine = $snapshot->{runningmachine};
5169 $forcecpu = $snapshot->{runningcpu};
092868c4 5170
87d92707 5171 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
b14477e7 5172
b14477e7
RV		 5173 $conf = $snapshot;
5174 }
5175
1e3baf05
DM		 5176 my $defaults = load_defaults();
5177
ea1c2110
SR		 5178 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5179 $forcemachine, $forcecpu);
1e3baf05 5180
5930c1ff 5181 return PVE::Tools::cmd2string($cmd);
1e3baf05
DM		 5182}
5183
5184sub vm_reset {
5185 my ($vmid, $skiplock) = @_;
5186
ffda963f 5187 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5188
ffda963f 5189 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5190
ffda963f 5191 PVE::QemuConfig->check_lock($conf) if !$skiplock;
1e3baf05 5192
0a13e08e 5193 mon_cmd($vmid, "system_reset");
ff1a2432
DM		 5194 });
5195}
5196
5197sub get_vm_volumes {
5198 my ($conf) = @_;
1e3baf05 5199
ff1a2432 5200 my $vollist = [];
d5769dc2 5201 foreach_volid($conf, sub {
392f8b5d 5202 my ($volid, $attr) = @_;
ff1a2432 5203
d5769dc2 5204 return if $volid =~ m|^/|;
ff1a2432 5205
d5769dc2
DM		 5206 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5207 return if !$sid;
ff1a2432
DM		 5208
5209 push @$vollist, $volid;
1e3baf05 5210 });
ff1a2432
DM		 5211
5212 return $vollist;
5213}
5214
5215sub vm_stop_cleanup {
70b04821 5216 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
ff1a2432 5217
745fed70 5218 eval {
ff1a2432 5219
254575e9
DM		 5220 if (!$keepActive) {
5221 my $vollist = get_vm_volumes($conf);
5222 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5223 }
a1b7d579 5224
ab6a046f 5225 foreach my $ext (qw(mon qmp pid vnc qga)) {
961bfcb2
DM		 5226 unlink "/var/run/qemu-server/${vmid}.$ext";
5227 }
a1b7d579 5228
6dbcb073
DC		 5229 if ($conf->{ivshmem}) {
5230 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4c5a6a24
TL		 5231 # just delete it for now, VMs which have this already open do not
5232 # are affected, but new VMs will get a separated one. If this
5233 # becomes an issue we either add some sort of ref-counting or just
5234 # add a "don't delete on stop" flag to the ivshmem format.
6dbcb073
DC		 5235 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5236 }
5237
6ab45bd7
DC		 5238 foreach my $key (keys %$conf) {
5239 next if $key !~ m/^hostpci(\d+)$/;
5240 my $hostpciindex = $1;
5241 my $d = parse_hostpci($conf->{$key});
5242 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5243
5244 foreach my $pci (@{$d->{pciid}}) {
2fd24788 5245 my $pciid = $pci->{id};
6ab45bd7
DC		 5246 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5247 }
5248 }
5249
70b04821 5250 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
745fed70
DM		 5251 };
5252 warn $@ if $@; # avoid errors - just warn
1e3baf05
DM		 5253}
5254
575d19da
DC		 5255# call only in locked context
5256sub _do_vm_stop {
5257 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
9269013a 5258
575d19da
DC		 5259 my $pid = check_running($vmid, $nocheck);
5260 return if !$pid;
1e3baf05 5261
575d19da
DC		 5262 my $conf;
5263 if (!$nocheck) {
5264 $conf = PVE::QemuConfig->load_config($vmid);
5265 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5266 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5267 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5268 $timeout = $opts->{down} if $opts->{down};
e6c3b671 5269 }
575d19da
DC		 5270 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5271 }
19672434 5272
575d19da
DC		 5273 eval {
5274 if ($shutdown) {
5275 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
0a13e08e 5276 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
9269013a 5277 } else {
0a13e08e 5278 mon_cmd($vmid, "system_powerdown");
1e3baf05
DM		 5279 }
5280 } else {
0a13e08e 5281 mon_cmd($vmid, "quit");
1e3baf05 5282 }
575d19da
DC		 5283 };
5284 my $err = $@;
1e3baf05 5285
575d19da
DC		 5286 if (!$err) {
5287 $timeout = 60 if !defined($timeout);
1e3baf05
DM		 5288
5289 my $count = 0;
e6c3b671 5290 while (($count < $timeout) && check_running($vmid, $nocheck)) {
1e3baf05
DM		 5291 $count++;
5292 sleep 1;
5293 }
5294
5295 if ($count >= $timeout) {
575d19da
DC		 5296 if ($force) {
5297 warn "VM still running - terminating now with SIGTERM\n";
5298 kill 15, $pid;
5299 } else {
5300 die "VM quit/powerdown failed - got timeout\n";
5301 }
5302 } else {
5303 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5304 return;
1e3baf05 5305 }
575d19da 5306 } else {
d60cbb97
TL		 5307 if (!check_running($vmid, $nocheck)) {
5308 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5309 return;
5310 }
5311 if ($force) {
575d19da
DC		 5312 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5313 kill 15, $pid;
5314 } else {
5315 die "VM quit/powerdown failed\n";
5316 }
5317 }
5318
5319 # wait again
5320 $timeout = 10;
5321
5322 my $count = 0;
5323 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5324 $count++;
5325 sleep 1;
5326 }
5327
5328 if ($count >= $timeout) {
5329 warn "VM still running - terminating now with SIGKILL\n";
5330 kill 9, $pid;
5331 sleep 1;
5332 }
1e3baf05 5333
575d19da
DC		 5334 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5335}
5336
5337# Note: use $nocheck to skip tests if VM configuration file exists.
5338# We need that when migration VMs to other nodes (files already moved)
5339# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5340sub vm_stop {
5341 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5342
5343 $force = 1 if !defined($force) && !$shutdown;
5344
5345 if ($migratedfrom){
5346 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5347 kill 15, $pid if $pid;
5348 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5349 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5350 return;
5351 }
5352
5353 PVE::QemuConfig->lock_config($vmid, sub {
5354 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
ff1a2432 5355 });
1e3baf05
DM		 5356}
5357
165411f0
DC		 5358sub vm_reboot {
5359 my ($vmid, $timeout) = @_;
5360
5361 PVE::QemuConfig->lock_config($vmid, sub {
66026117 5362 eval {
165411f0 5363
66026117
OB		 5364 # only reboot if running, as qmeventd starts it again on a stop event
5365 return if !check_running($vmid);
165411f0 5366
66026117 5367 create_reboot_request($vmid);
165411f0 5368
66026117
OB		 5369 my $storecfg = PVE::Storage::config();
5370 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
165411f0 5371
66026117
OB		 5372 };
5373 if (my $err = $@) {
3c1c3fe6 5374 # avoid that the next normal shutdown will be confused for a reboot
66026117
OB		 5375 clear_reboot_request($vmid);
5376 die $err;
5377 }
165411f0
DC		 5378 });
5379}
5380
75c24bba 5381# note: if using the statestorage parameter, the caller has to check privileges
1e3baf05 5382sub vm_suspend {
48b4cdc2 5383 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
159719e5
DC		 5384
5385 my $conf;
5386 my $path;
5387 my $storecfg;
5388 my $vmstate;
1e3baf05 5389
ffda963f 5390 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5391
159719e5 5392 $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5393
159719e5 5394 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
e79706d4 5395 PVE::QemuConfig->check_lock($conf)
159719e5
DC		 5396 if !($skiplock || $is_backing_up);
5397
5398 die "cannot suspend to disk during backup\n"
5399 if $is_backing_up && $includestate;
bcb7c9cf 5400
159719e5
DC		 5401 if ($includestate) {
5402 $conf->{lock} = 'suspending';
5403 my $date = strftime("%Y-%m-%d", localtime(time()));
5404 $storecfg = PVE::Storage::config();
75c24bba
DC		 5405 if (!$statestorage) {
5406 $statestorage = find_vmstate_storage($conf, $storecfg);
5407 # check permissions for the storage
5408 my $rpcenv = PVE::RPCEnvironment::get();
5409 if ($rpcenv->{type} ne 'cli') {
5410 my $authuser = $rpcenv->get_user();
5411 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5412 }
5413 }
5414
5415
48b4cdc2 5416 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
159719e5
DC		 5417 $path = PVE::Storage::path($storecfg, $vmstate);
5418 PVE::QemuConfig->write_config($vmid, $conf);
5419 } else {
0a13e08e 5420 mon_cmd($vmid, "stop");
159719e5 5421 }
1e3baf05 5422 });
159719e5
DC		 5423
5424 if ($includestate) {
5425 # save vm state
5426 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5427
5428 eval {
0a13e08e 5429 mon_cmd($vmid, "savevm-start", statefile => $path);
159719e5 5430 for(;;) {
0a13e08e 5431 my $state = mon_cmd($vmid, "query-savevm");
159719e5
DC		 5432 if (!$state->{status}) {
5433 die "savevm not active\n";
5434 } elsif ($state->{status} eq 'active') {
5435 sleep(1);
5436 next;
5437 } elsif ($state->{status} eq 'completed') {
b0a9a385 5438 print "State saved, quitting\n";
159719e5
DC		 5439 last;
5440 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5441 die "query-savevm failed with error '$state->{error}'\n"
5442 } else {
5443 die "query-savevm returned status '$state->{status}'\n";
5444 }
5445 }
5446 };
5447 my $err = $@;
5448
5449 PVE::QemuConfig->lock_config($vmid, sub {
5450 $conf = PVE::QemuConfig->load_config($vmid);
5451 if ($err) {
5452 # cleanup, but leave suspending lock, to indicate something went wrong
5453 eval {
0a13e08e 5454 mon_cmd($vmid, "savevm-end");
159719e5
DC		 5455 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5456 PVE::Storage::vdisk_free($storecfg, $vmstate);
ea1c2110 5457 delete $conf->@{qw(vmstate runningmachine runningcpu)};
159719e5
DC		 5458 PVE::QemuConfig->write_config($vmid, $conf);
5459 };
5460 warn $@ if $@;
5461 die $err;
5462 }
5463
5464 die "lock changed unexpectedly\n"
5465 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5466
0a13e08e 5467 mon_cmd($vmid, "quit");
159719e5
DC		 5468 $conf->{lock} = 'suspended';
5469 PVE::QemuConfig->write_config($vmid, $conf);
5470 });
5471 }
1e3baf05
DM		 5472}
5473
5474sub vm_resume {
289e0b85 5475 my ($vmid, $skiplock, $nocheck) = @_;
1e3baf05 5476
ffda963f 5477 PVE::QemuConfig->lock_config($vmid, sub {
0a13e08e 5478 my $res = mon_cmd($vmid, 'query-status');
c2786bed
DC		 5479 my $resume_cmd = 'cont';
5480
5481 if ($res->{status} && $res->{status} eq 'suspended') {
5482 $resume_cmd = 'system_wakeup';
5483 }
5484
289e0b85 5485 if (!$nocheck) {
1e3baf05 5486
ffda963f 5487 my $conf = PVE::QemuConfig->load_config($vmid);
1e3baf05 5488
e79706d4
FG		 5489 PVE::QemuConfig->check_lock($conf)
5490 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
289e0b85 5491 }
3e24733b 5492
0a13e08e 5493 mon_cmd($vmid, $resume_cmd);
1e3baf05
DM		 5494 });
5495}
5496
5fdbe4f0
DM		 5497sub vm_sendkey {
5498 my ($vmid, $skiplock, $key) = @_;
1e3baf05 5499
ffda963f 5500 PVE::QemuConfig->lock_config($vmid, sub {
1e3baf05 5501
ffda963f 5502 my $conf = PVE::QemuConfig->load_config($vmid);
f5eb281a 5503
7b7c6d1b 5504 # there is no qmp command, so we use the human monitor command
0a13e08e 5505 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
d30820d6 5506 die $res if $res ne '';
1e3baf05
DM		 5507 });
5508}
5509
3e16d5fc
DM		 5510# vzdump restore implementaion
5511
ed221350 5512sub tar_archive_read_firstfile {
3e16d5fc 5513 my $archive = shift;
afdb31d5 5514
3e16d5fc
DM		 5515 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5516
5517 # try to detect archive type first
387ba257 5518 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
3e16d5fc 5519 die "unable to open file '$archive'\n";
387ba257 5520 my $firstfile = <$fh>;
3e16d5fc 5521 kill 15, $pid;
387ba257 5522 close $fh;
3e16d5fc
DM		 5523
5524 die "ERROR: archive contaions no data\n" if !$firstfile;
5525 chomp $firstfile;
5526
5527 return $firstfile;
5528}
5529
ed221350
DM		 5530sub tar_restore_cleanup {
5531 my ($storecfg, $statfile) = @_;
3e16d5fc
DM		 5532
5533 print STDERR "starting cleanup\n";
5534
5535 if (my $fd = IO::File->new($statfile, "r")) {
5536 while (defined(my $line = <$fd>)) {
5537 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5538 my $volid = $2;
5539 eval {
5540 if ($volid =~ m|^/|) {
5541 unlink $volid || die 'unlink failed\n';
5542 } else {
ed221350 5543 PVE::Storage::vdisk_free($storecfg, $volid);
3e16d5fc 5544 }
afdb31d5 5545 print STDERR "temporary volume '$volid' sucessfuly removed\n";
3e16d5fc
DM		 5546 };
5547 print STDERR "unable to cleanup '$volid' - $@" if $@;
5548 } else {
5549 print STDERR "unable to parse line in statfile - $line";
afdb31d5 5550 }
3e16d5fc
DM		 5551 }
5552 $fd->close();
5553 }
5554}
5555
d1e92cf6 5556sub restore_file_archive {
a0d1b1a2 5557 my ($archive, $vmid, $user, $opts) = @_;
3e16d5fc 5558
a2ec5a67
FG		 5559 return restore_vma_archive($archive, $vmid, $user, $opts)
5560 if $archive eq '-';
5561
c6d51783
AA		 5562 my $info = PVE::Storage::archive_info($archive);
5563 my $format = $opts->{format} // $info->{format};
5564 my $comp = $info->{compression};
91bd6c90
DM		 5565
5566 # try to detect archive format
5567 if ($format eq 'tar') {
5568 return restore_tar_archive($archive, $vmid, $user, $opts);
5569 } else {
5570 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5571 }
5572}
5573
d1e92cf6
DM		 5574# hepler to remove disks that will not be used after restore
5575my $restore_cleanup_oldconf = sub {
5576 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5577
912792e2 5578 PVE::QemuConfig->foreach_volume($oldconf, sub {
d1e92cf6
DM		 5579 my ($ds, $drive) = @_;
5580
5581 return if drive_is_cdrom($drive, 1);
5582
5583 my $volid = $drive->{file};
5584 return if !$volid || $volid =~ m|^/|;
5585
5586 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5587 return if !$path || !$owner || ($owner != $vmid);
5588
5589 # Note: only delete disk we want to restore
5590 # other volumes will become unused
5591 if ($virtdev_hash->{$ds}) {
5592 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5593 if (my $err = $@) {
5594 warn $err;
5595 }
5596 }
5597 });
5598
5599 # delete vmstate files, after the restore we have no snapshots anymore
5600 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5601 my $snap = $oldconf->{snapshots}->{$snapname};
5602 if ($snap->{vmstate}) {
5603 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5604 if (my $err = $@) {
5605 warn $err;
5606 }
5607 }
5608 }
5609};
5610
9f3d73bc
DM		 5611# Helper to parse vzdump backup device hints
5612#
5613# $rpcenv: Environment, used to ckeck storage permissions
5614# $user: User ID, to check storage permissions
5615# $storecfg: Storage configuration
5616# $fh: the file handle for reading the configuration
5617# $devinfo: should contain device sizes for all backu-up'ed devices
5618# $options: backup options (pool, default storage)
5619#
5620# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5621my $parse_backup_hints = sub {
5622 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
d1e92cf6 5623
9f3d73bc 5624 my $virtdev_hash = {};
d1e92cf6 5625
9f3d73bc
DM		 5626 while (defined(my $line = <$fh>)) {
5627 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5628 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5629 die "archive does not contain data for drive '$virtdev'\n"
5630 if !$devinfo->{$devname};
5631
5632 if (defined($options->{storage})) {
5633 $storeid = $options->{storage} || 'local';
5634 } elsif (!$storeid) {
5635 $storeid = 'local';
d1e92cf6 5636 }
9f3d73bc
DM		 5637 $format = 'raw' if !$format;
5638 $devinfo->{$devname}->{devname} = $devname;
5639 $devinfo->{$devname}->{virtdev} = $virtdev;
5640 $devinfo->{$devname}->{format} = $format;
5641 $devinfo->{$devname}->{storeid} = $storeid;
5642
5643 # check permission on storage
5644 my $pool = $options->{pool}; # todo: do we need that?
5645 if ($user ne 'root@pam') {
5646 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
d1e92cf6 5647 }
d1e92cf6 5648
9f3d73bc
DM		 5649 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5650 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5651 my $virtdev = $1;
5652 my $drive = parse_drive($virtdev, $2);
5653 if (drive_is_cloudinit($drive)) {
5654 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5364990d
TL		 5655 $storeid = $options->{storage} if defined ($options->{storage});
5656 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
9f3d73bc 5657 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
d1e92cf6 5658
9f3d73bc
DM		 5659 $virtdev_hash->{$virtdev} = {
5660 format => $format,
5364990d 5661 storeid => $storeid,
9f3d73bc
DM		 5662 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5663 is_cloudinit => 1,
5664 };
d1e92cf6 5665 }
9f3d73bc
DM		 5666 }
5667 }
d1e92cf6 5668
9f3d73bc
DM		 5669 return $virtdev_hash;
5670};
d1e92cf6 5671
9f3d73bc
DM		 5672# Helper to allocate and activate all volumes required for a restore
5673#
5674# $storecfg: Storage configuration
5675# $virtdev_hash: as returned by parse_backup_hints()
5676#
5677# Returns: { $virtdev => $volid }
5678my $restore_allocate_devices = sub {
5679 my ($storecfg, $virtdev_hash, $vmid) = @_;
d1e92cf6 5680
9f3d73bc
DM		 5681 my $map = {};
5682 foreach my $virtdev (sort keys %$virtdev_hash) {
5683 my $d = $virtdev_hash->{$virtdev};
5684 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5685 my $storeid = $d->{storeid};
5686 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
d1e92cf6 5687
9f3d73bc
DM		 5688 # test if requested format is supported
5689 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5690 my $supported = grep { $_ eq $d->{format} } @$validFormats;
5691 $d->{format} = $defFormat if !$supported;
d1e92cf6 5692
9f3d73bc
DM		 5693 my $name;
5694 if ($d->{is_cloudinit}) {
5695 $name = "vm-$vmid-cloudinit";
5696 $name .= ".$d->{format}" if $d->{format} ne 'raw';
d1e92cf6
DM		 5697 }
5698
9f3d73bc 5699 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
d1e92cf6 5700
9f3d73bc
DM		 5701 print STDERR "new volume ID is '$volid'\n";
5702 $d->{volid} = $volid;
d1e92cf6 5703
9f3d73bc 5704 PVE::Storage::activate_volumes($storecfg, [$volid]);
d1e92cf6 5705
9f3d73bc 5706 $map->{$virtdev} = $volid;
d1e92cf6
DM		 5707 }
5708
9f3d73bc
DM		 5709 return $map;
5710};
d1e92cf6 5711
9f3d73bc 5712my $restore_update_config_line = sub {
91bd6c90
DM		 5713 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5714
5715 return if $line =~ m/^\#qmdump\#/;
5716 return if $line =~ m/^\#vzdump\#/;
5717 return if $line =~ m/^lock:/;
5718 return if $line =~ m/^unused\d+:/;
5719 return if $line =~ m/^parent:/;
5720
b5b99790 5721 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
91bd6c90
DM		 5722 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5723 # try to convert old 1.X settings
5724 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5725 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5726 my ($model, $macaddr) = split(/\=/, $devconfig);
b5b99790 5727 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
91bd6c90
DM		 5728 my $net = {
5729 model => $model,
5730 bridge => "vmbr$ind",
5731 macaddr => $macaddr,
5732 };
5733 my $netstr = print_net($net);
5734
5735 print $outfd "net$cookie->{netcount}: $netstr\n";
5736 $cookie->{netcount}++;
5737 }
5738 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5739 my ($id, $netstr) = ($1, $2);
5740 my $net = parse_net($netstr);
b5b99790 5741 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
91bd6c90
DM		 5742 $netstr = print_net($net);
5743 print $outfd "$id: $netstr\n";
6470743f 5744 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
91bd6c90 5745 my $virtdev = $1;
907ea891 5746 my $value = $3;
d9faf790
WB		 5747 my $di = parse_drive($virtdev, $value);
5748 if (defined($di->{backup}) && !$di->{backup}) {
91bd6c90 5749 print $outfd "#$line";
c0f7406e 5750 } elsif ($map->{$virtdev}) {
8fd57431 5751 delete $di->{format}; # format can change on restore
91bd6c90 5752 $di->{file} = $map->{$virtdev};
71c58bb7 5753 $value = print_drive($di);
91bd6c90
DM		 5754 print $outfd "$virtdev: $value\n";
5755 } else {
5756 print $outfd $line;
5757 }
1a0c2f03 5758 } elsif (($line =~ m/^vmgenid: (.*)/)) {
babecffe 5759 my $vmgenid = $1;
6ee499ff 5760 if ($vmgenid ne '0') {
1a0c2f03 5761 # always generate a new vmgenid if there was a valid one setup
6ee499ff
DC		 5762 $vmgenid = generate_uuid();
5763 }
1a0c2f03 5764 print $outfd "vmgenid: $vmgenid\n";
19a5dd55
WL		 5765 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5766 my ($uuid, $uuid_str);
5767 UUID::generate($uuid);
5768 UUID::unparse($uuid, $uuid_str);
5769 my $smbios1 = parse_smbios1($2);
5770 $smbios1->{uuid} = $uuid_str;
5771 print $outfd $1.print_smbios1($smbios1)."\n";
91bd6c90
DM		 5772 } else {
5773 print $outfd $line;
5774 }
9f3d73bc
DM		 5775};
5776
5777my $restore_deactivate_volumes = sub {
5778 my ($storecfg, $devinfo) = @_;
5779
5780 my $vollist = [];
5781 foreach my $devname (keys %$devinfo) {
5782 my $volid = $devinfo->{$devname}->{volid};
5783 push @$vollist, $volid if $volid;
5784 }
5785
5786 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5787};
5788
5789my $restore_destroy_volumes = sub {
5790 my ($storecfg, $devinfo) = @_;
5791
5792 foreach my $devname (keys %$devinfo) {
5793 my $volid = $devinfo->{$devname}->{volid};
5794 next if !$volid;
5795 eval {
5796 if ($volid =~ m|^/|) {
5797 unlink $volid || die 'unlink failed\n';
5798 } else {
5799 PVE::Storage::vdisk_free($storecfg, $volid);
5800 }
5801 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5802 };
5803 print STDERR "unable to cleanup '$volid' - $@" if $@;
5804 }
5805};
91bd6c90
DM		 5806
5807sub scan_volids {
5808 my ($cfg, $vmid) = @_;
5809
5810 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5811
5812 my $volid_hash = {};
5813 foreach my $storeid (keys %$info) {
5814 foreach my $item (@{$info->{$storeid}}) {
5815 next if !($item->{volid} && $item->{size});
5996a936 5816 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
91bd6c90
DM		 5817 $volid_hash->{$item->{volid}} = $item;
5818 }
5819 }
5820
5821 return $volid_hash;
5822}
5823
68b108ee 5824sub update_disk_config {
91bd6c90 5825 my ($vmid, $conf, $volid_hash) = @_;
be190583 5826
91bd6c90 5827 my $changes;
9b29cbd0 5828 my $prefix = "VM $vmid";
91bd6c90 5829
c449137a
DC		 5830 # used and unused disks
5831 my $referenced = {};
91bd6c90 5832
5996a936
DM		 5833 # Note: it is allowed to define multiple storages with same path (alias), so
5834 # we need to check both 'volid' and real 'path' (two different volid can point
5835 # to the same path).
5836
c449137a 5837 my $referencedpath = {};
be190583 5838
91bd6c90 5839 # update size info
0c4fef3f 5840 PVE::QemuConfig->foreach_volume($conf, sub {
ca04977c 5841 my ($opt, $drive) = @_;
91bd6c90 5842
ca04977c
FE		 5843 my $volid = $drive->{file};
5844 return if !$volid;
91bd6c90 5845
ca04977c
FE		 5846 # mark volid as "in-use" for next step
5847 $referenced->{$volid} = 1;
5848 if ($volid_hash->{$volid} &&
5849 (my $path = $volid_hash->{$volid}->{path})) {
5850 $referencedpath->{$path} = 1;
91bd6c90 5851 }
ca04977c
FE		 5852
5853 return if drive_is_cdrom($drive);
5854 return if !$volid_hash->{$volid};
5855
9b29cbd0 5856 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size});
ca04977c
FE		 5857 if (defined($updated)) {
5858 $changes = 1;
5859 $conf->{$opt} = print_drive($updated);
9b29cbd0 5860 print "$prefix ($opt): $msg\n";
ca04977c
FE		 5861 }
5862 });
91bd6c90 5863
5996a936 5864 # remove 'unusedX' entry if volume is used
ca04977c
FE		 5865 PVE::QemuConfig->foreach_unused_volume($conf, sub {
5866 my ($opt, $drive) = @_;
5867
5868 my $volid = $drive->{file};
5869 return if !$volid;
5870
5996a936 5871 my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
c449137a 5872 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
68b108ee 5873 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5996a936
DM		 5874 $changes = 1;
5875 delete $conf->{$opt};
5876 }
c449137a
DC		 5877
5878 $referenced->{$volid} = 1;
5879 $referencedpath->{$path} = 1 if $path;
ca04977c 5880 });
5996a936 5881
91bd6c90
DM		 5882 foreach my $volid (sort keys %$volid_hash) {
5883 next if $volid =~ m/vm-$vmid-state-/;
c449137a 5884 next if $referenced->{$volid};
5996a936
DM		 5885 my $path = $volid_hash->{$volid}->{path};
5886 next if !$path; # just to be sure
c449137a 5887 next if $referencedpath->{$path};
91bd6c90 5888 $changes = 1;
53b81297 5889 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
68b108ee 5890 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
c449137a 5891 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
91bd6c90
DM		 5892 }
5893
5894 return $changes;
5895}
5896
5897sub rescan {
9224dcee 5898 my ($vmid, $nolock, $dryrun) = @_;
91bd6c90 5899
20519efc 5900 my $cfg = PVE::Storage::config();
91bd6c90 5901
b9a1a3ab
TL		 5902 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5903 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
4771526a
AA		 5904 foreach my $stor (keys %{$cfg->{ids}}) {
5905 delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5906 }
5907
53b81297 5908 print "rescan volumes...\n";
91bd6c90
DM		 5909 my $volid_hash = scan_volids($cfg, $vmid);
5910
5911 my $updatefn = sub {
5912 my ($vmid) = @_;
5913
ffda963f 5914 my $conf = PVE::QemuConfig->load_config($vmid);
be190583 5915
ffda963f 5916 PVE::QemuConfig->check_lock($conf);
91bd6c90 5917
03da3f0d
DM		 5918 my $vm_volids = {};
5919 foreach my $volid (keys %$volid_hash) {
5920 my $info = $volid_hash->{$volid};
5921 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5922 }
5923
68b108ee 5924 my $changes = update_disk_config($vmid, $conf, $vm_volids);
91bd6c90 5925
9224dcee 5926 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
91bd6c90
DM		 5927 };
5928
5929 if (defined($vmid)) {
5930 if ($nolock) {
5931 &$updatefn($vmid);
5932 } else {
ffda963f 5933 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
91bd6c90
DM		 5934 }
5935 } else {
5936 my $vmlist = config_list();
5937 foreach my $vmid (keys %$vmlist) {
5938 if ($nolock) {
5939 &$updatefn($vmid);
5940 } else {
ffda963f 5941 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
be190583 5942 }
91bd6c90
DM		 5943 }
5944 }
5945}
5946
9f3d73bc
DM		 5947sub restore_proxmox_backup_archive {
5948 my ($archive, $vmid, $user, $options) = @_;
5949
5950 my $storecfg = PVE::Storage::config();
5951
5952 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5953 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5954
5955 my $server = $scfg->{server};
5956 my $datastore = $scfg->{datastore};
5957 my $username = $scfg->{username} // 'root@pam';
5958 my $fingerprint = $scfg->{fingerprint};
503e96f8 5959 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
9f3d73bc
DM		 5960
5961 my $repo = "$username\@$server:$datastore";
dea4b04c
WB		 5962
5963 # This is only used for `pbs-restore`!
9f3d73bc
DM		 5964 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5965 local $ENV{PBS_PASSWORD} = $password;
5966 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5967
5968 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5969 PVE::Storage::parse_volname($storecfg, $archive);
5970
5971 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5972
5973 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5974
5975 my $tmpdir = "/var/tmp/vzdumptmp$$";
5976 rmtree $tmpdir;
5977 mkpath $tmpdir;
5978
5979 my $conffile = PVE::QemuConfig->config_file($vmid);
5980 my $tmpfn = "$conffile.$$.tmp";
5981 # disable interrupts (always do cleanups)
5982 local $SIG{INT} =
5983 local $SIG{TERM} =
5984 local $SIG{QUIT} =
5985 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5986
5987 # Note: $oldconf is undef if VM does not exists
5988 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5989 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5990
5991 my $rpcenv = PVE::RPCEnvironment::get();
5992 my $devinfo = {};
5993
5994 eval {
5995 # enable interrupts
5996 local $SIG{INT} =
5997 local $SIG{TERM} =
5998 local $SIG{QUIT} =
5999 local $SIG{HUP} =
6000 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6001
6002 my $cfgfn = "$tmpdir/qemu-server.conf";
6003 my $firewall_config_fn = "$tmpdir/fw.conf";
6004 my $index_fn = "$tmpdir/index.json";
6005
6006 my $cmd = "restore";
6007
6008 my $param = [$pbs_backup_name, "index.json", $index_fn];
6009 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6010 my $index = PVE::Tools::file_get_contents($index_fn);
6011 $index = decode_json($index);
6012
6013 # print Dumper($index);
6014 foreach my $info (@{$index->{files}}) {
6015 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6016 my $devname = $1;
6017 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6018 $devinfo->{$devname}->{size} = $1;
6019 } else {
6020 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6021 }
6022 }
6023 }
6024
6025 my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6026 if (!$is_qemu_server_backup) {
6027 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6028 }
6029 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6030
6031 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6032 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6033
6034 if ($has_firewall_config) {
6035 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6036 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6037
6038 my $pve_firewall_dir = '/etc/pve/firewall';
6039 mkdir $pve_firewall_dir; # make sure the dir exists
6040 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6041 }
6042
6043 my $fh = IO::File->new($cfgfn, "r") ||
a1cbe55c 6044 die "unable to read qemu-server.conf - $!\n";
9f3d73bc
DM		 6045
6046 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6047
6048 # fixme: rate limit?
6049
6050 # create empty/temp config
6051 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6052
6053 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6054
6055 # allocate volumes
6056 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6057
6058 foreach my $virtdev (sort keys %$virtdev_hash) {
6059 my $d = $virtdev_hash->{$virtdev};
6060 next if $d->{is_cloudinit}; # no need to restore cloudinit
6061
6062 my $volid = $d->{volid};
6063
6064 my $path = PVE::Storage::path($storecfg, $volid);
9f3d73bc 6065
dea4b04c 6066 # This is the ONLY user of the PBS_ env vars set on top of this function!
9f3d73bc 6067 my $pbs_restore_cmd = [
e400f5e5 6068 '/usr/bin/pbs-restore',
9f3d73bc
DM		 6069 '--repository', $repo,
6070 $pbs_backup_name,
e400f5e5
DM		 6071 "$d->{devname}.img.fidx",
6072 $path,
9f3d73bc
DM		 6073 '--verbose',
6074 ];
6075
55fb78aa 6076 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
503e96f8 6077 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
55fb78aa 6078
e400f5e5
DM		 6079 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6080 push @$pbs_restore_cmd, '--skip-zero';
6081 }
9f3d73bc 6082
e400f5e5 6083 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
9f3d73bc 6084 print "restore proxmox backup image: $dbg_cmdstring\n";
e400f5e5 6085 run_command($pbs_restore_cmd);
9f3d73bc
DM		 6086 }
6087
6088 $fh->seek(0, 0) || die "seek failed - $!\n";
6089
6090 my $outfd = new IO::File ($tmpfn, "w") ||
6091 die "unable to write config for VM $vmid\n";
6092
6093 my $cookie = { netcount => 0 };
6094 while (defined(my $line = <$fh>)) {
6095 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6096 }
6097
6098 $fh->close();
6099 $outfd->close();
6100 };
6101 my $err = $@;
6102
6103 $restore_deactivate_volumes->($storecfg, $devinfo);
6104
6105 rmtree $tmpdir;
6106
6107 if ($err) {
6108 unlink $tmpfn;
6109 $restore_destroy_volumes->($storecfg, $devinfo);
6110 die $err;
6111 }
6112
6113 rename($tmpfn, $conffile) ||
6114 die "unable to commit configuration file '$conffile'\n";
6115
6116 PVE::Cluster::cfs_update(); # make sure we read new file
6117
6118 eval { rescan($vmid, 1); };
6119 warn $@ if $@;
6120}
6121
91bd6c90
DM		 6122sub restore_vma_archive {
6123 my ($archive, $vmid, $user, $opts, $comp) = @_;
6124
91bd6c90
DM		 6125 my $readfrom = $archive;
6126
7c536e11
WB		 6127 my $cfg = PVE::Storage::config();
6128 my $commands = [];
6129 my $bwlimit = $opts->{bwlimit};
6130
6131 my $dbg_cmdstring = '';
6132 my $add_pipe = sub {
6133 my ($cmd) = @_;
6134 push @$commands, $cmd;
6135 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6136 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
91bd6c90 6137 $readfrom = '-';
7c536e11
WB		 6138 };
6139
6140 my $input = undef;
6141 if ($archive eq '-') {
6142 $input = '<&STDIN';
6143 } else {
6144 # If we use a backup from a PVE defined storage we also consider that
6145 # storage's rate limit:
6146 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6147 if (defined($volid)) {
6148 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6149 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6150 if ($readlimit) {
6151 print STDERR "applying read rate limit: $readlimit\n";
9444c6e4 6152 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7c536e11
WB		 6153 $add_pipe->($cstream);
6154 }
6155 }
6156 }
6157
6158 if ($comp) {
c6d51783
AA		 6159 my $info = PVE::Storage::decompressor_info('vma', $comp);
6160 my $cmd = $info->{decompressor};
6161 push @$cmd, $readfrom;
7c536e11 6162 $add_pipe->($cmd);
91bd6c90
DM		 6163 }
6164
6165 my $tmpdir = "/var/tmp/vzdumptmp$$";
6166 rmtree $tmpdir;
6167
6168 # disable interrupts (always do cleanups)
5b97ef24
TL		 6169 local $SIG{INT} =
6170 local $SIG{TERM} =
6171 local $SIG{QUIT} =
6172 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
91bd6c90
DM		 6173
6174 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6175 POSIX::mkfifo($mapfifo, 0600);
6176 my $fifofh;
6177
6178 my $openfifo = sub {
6179 open($fifofh, '>', $mapfifo) || die $!;
6180 };
6181
7c536e11 6182 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
91bd6c90
DM		 6183
6184 my $oldtimeout;
6185 my $timeout = 5;
6186
6187 my $devinfo = {};
6188
6189 my $rpcenv = PVE::RPCEnvironment::get();
6190
ffda963f 6191 my $conffile = PVE::QemuConfig->config_file($vmid);
91bd6c90
DM		 6192 my $tmpfn = "$conffile.$$.tmp";
6193
ae200950 6194 # Note: $oldconf is undef if VM does not exist
ffda963f
FG		 6195 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6196 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
ed221350 6197
7c536e11
WB		 6198 my %storage_limits;
6199
91bd6c90 6200 my $print_devmap = sub {
91bd6c90
DM		 6201 my $cfgfn = "$tmpdir/qemu-server.conf";
6202
6203 # we can read the config - that is already extracted
6204 my $fh = IO::File->new($cfgfn, "r") ||
a1cbe55c 6205 die "unable to read qemu-server.conf - $!\n";
91bd6c90 6206
6738ab9c 6207 my $fwcfgfn = "$tmpdir/qemu-server.fw";
3457d090
WL		 6208 if (-f $fwcfgfn) {
6209 my $pve_firewall_dir = '/etc/pve/firewall';
6210 mkdir $pve_firewall_dir; # make sure the dir exists
6211 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6212 }
6738ab9c 6213
9f3d73bc 6214 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
91bd6c90 6215
7c536e11
WB		 6216 foreach my $key (keys %storage_limits) {
6217 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6218 next if !$limit;
6219 print STDERR "rate limit for storage $key: $limit KiB/s\n";
6220 $storage_limits{$key} = $limit * 1024;
6221 }
6222
91bd6c90 6223 foreach my $devname (keys %$devinfo) {
be190583
DM		 6224 die "found no device mapping information for device '$devname'\n"
6225 if !$devinfo->{$devname}->{virtdev};
91bd6c90
DM		 6226 }
6227
ed221350 6228 # create empty/temp config
be190583 6229 if ($oldconf) {
ed221350 6230 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
d1e92cf6 6231 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
ed221350
DM		 6232 }
6233
9f3d73bc
DM		 6234 # allocate volumes
6235 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6236
6237 # print restore information to $fifofh
91bd6c90
DM		 6238 foreach my $virtdev (sort keys %$virtdev_hash) {
6239 my $d = $virtdev_hash->{$virtdev};
9f3d73bc
DM		 6240 next if $d->{is_cloudinit}; # no need to restore cloudinit
6241
7c536e11 6242 my $storeid = $d->{storeid};
9f3d73bc 6243 my $volid = $d->{volid};
7c536e11
WB		 6244
6245 my $map_opts = '';
6246 if (my $limit = $storage_limits{$storeid}) {
6247 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6248 }
8fd57431 6249
91bd6c90 6250 my $write_zeros = 1;
88240a83 6251 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
91bd6c90
DM		 6252 $write_zeros = 0;
6253 }
6254
9f3d73bc 6255 my $path = PVE::Storage::path($cfg, $volid);
87056e18 6256
9f3d73bc 6257 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
91bd6c90 6258
9f3d73bc 6259 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
91bd6c90
DM		 6260 }
6261
6262 $fh->seek(0, 0) || die "seek failed - $!\n";
6263
6264 my $outfd = new IO::File ($tmpfn, "w") ||
6265 die "unable to write config for VM $vmid\n";
6266
6267 my $cookie = { netcount => 0 };
6268 while (defined(my $line = <$fh>)) {
9f3d73bc 6269 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
91bd6c90
DM		 6270 }
6271
6272 $fh->close();
6273 $outfd->close();
6274 };
6275
6276 eval {
6277 # enable interrupts
6cb0144a
EK		 6278 local $SIG{INT} =
6279 local $SIG{TERM} =
6280 local $SIG{QUIT} =
6281 local $SIG{HUP} =
6282 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
91bd6c90
DM		 6283 local $SIG{ALRM} = sub { die "got timeout\n"; };
6284
6285 $oldtimeout = alarm($timeout);
6286
6287 my $parser = sub {
6288 my $line = shift;
6289
6290 print "$line\n";
6291
6292 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6293 my ($dev_id, $size, $devname) = ($1, $2, $3);
6294 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6295 } elsif ($line =~ m/^CTIME: /) {
46f58b5f 6296 # we correctly received the vma config, so we can disable
3cf90d7a
DM		 6297 # the timeout now for disk allocation (set to 10 minutes, so
6298 # that we always timeout if something goes wrong)
6299 alarm(600);
91bd6c90
DM		 6300 &$print_devmap();
6301 print $fifofh "done\n";
6302 my $tmp = $oldtimeout || 0;
6303 $oldtimeout = undef;
6304 alarm($tmp);
6305 close($fifofh);
6306 }
6307 };
be190583 6308
7c536e11
WB		 6309 print "restore vma archive: $dbg_cmdstring\n";
6310 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
91bd6c90
DM		 6311 };
6312 my $err = $@;
6313
6314 alarm($oldtimeout) if $oldtimeout;
6315
9f3d73bc 6316 $restore_deactivate_volumes->($cfg, $devinfo);
5f96f4df 6317
91bd6c90 6318 unlink $mapfifo;
9f3d73bc 6319 rmtree $tmpdir;
91bd6c90
DM		 6320
6321 if ($err) {
91bd6c90 6322 unlink $tmpfn;
9f3d73bc 6323 $restore_destroy_volumes->($cfg, $devinfo);
91bd6c90
DM		 6324 die $err;
6325 }
6326
ed221350 6327 rename($tmpfn, $conffile) ||
91bd6c90
DM		 6328 die "unable to commit configuration file '$conffile'\n";
6329
ed221350
DM		 6330 PVE::Cluster::cfs_update(); # make sure we read new file
6331
91bd6c90
DM		 6332 eval { rescan($vmid, 1); };
6333 warn $@ if $@;
6334}
6335
6336sub restore_tar_archive {
6337 my ($archive, $vmid, $user, $opts) = @_;
6338
9c502e26 6339 if ($archive ne '-') {
ed221350 6340 my $firstfile = tar_archive_read_firstfile($archive);
9c502e26
DM		 6341 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6342 if $firstfile ne 'qemu-server.conf';
6343 }
3e16d5fc 6344
20519efc 6345 my $storecfg = PVE::Storage::config();
ebb55558 6346
4b026937
TL		 6347 # avoid zombie disks when restoring over an existing VM -> cleanup first
6348 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6349 # skiplock=1 because qmrestore has set the 'create' lock itself already
ffda963f 6350 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
b04ea584 6351 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
ed221350 6352
3e16d5fc
DM		 6353 my $tocmd = "/usr/lib/qemu-server/qmextract";
6354
2415a446 6355 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
a0d1b1a2 6356 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
3e16d5fc
DM		 6357 $tocmd .= ' --prealloc' if $opts->{prealloc};
6358 $tocmd .= ' --info' if $opts->{info};
6359
a0d1b1a2 6360 # tar option "xf" does not autodetect compression when read from STDIN,
9c502e26 6361 # so we pipe to zcat
2415a446
DM		 6362 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6363 PVE::Tools::shellquote("--to-command=$tocmd");
3e16d5fc
DM		 6364
6365 my $tmpdir = "/var/tmp/vzdumptmp$$";
6366 mkpath $tmpdir;
6367
6368 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6369 local $ENV{VZDUMP_VMID} = $vmid;
a0d1b1a2 6370 local $ENV{VZDUMP_USER} = $user;
3e16d5fc 6371
ffda963f 6372 my $conffile = PVE::QemuConfig->config_file($vmid);
3e16d5fc
DM		 6373 my $tmpfn = "$conffile.$$.tmp";
6374
6375 # disable interrupts (always do cleanups)
6cb0144a
EK		 6376 local $SIG{INT} =
6377 local $SIG{TERM} =
6378 local $SIG{QUIT} =
6379 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
3e16d5fc 6380
afdb31d5 6381 eval {
3e16d5fc 6382 # enable interrupts
6cb0144a
EK		 6383 local $SIG{INT} =
6384 local $SIG{TERM} =
6385 local $SIG{QUIT} =
6386 local $SIG{HUP} =
6387 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
3e16d5fc 6388
9c502e26
DM		 6389 if ($archive eq '-') {
6390 print "extracting archive from STDIN\n";
6391 run_command($cmd, input => "<&STDIN");
6392 } else {
6393 print "extracting archive '$archive'\n";
6394 run_command($cmd);
6395 }
3e16d5fc
DM		 6396
6397 return if $opts->{info};
6398
6399 # read new mapping
6400 my $map = {};
6401 my $statfile = "$tmpdir/qmrestore.stat";
6402 if (my $fd = IO::File->new($statfile, "r")) {
6403 while (defined (my $line = <$fd>)) {
6404 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6405 $map->{$1} = $2 if $1;
6406 } else {
6407 print STDERR "unable to parse line in statfile - $line\n";
6408 }
6409 }
6410 $fd->close();
6411 }
6412
6413 my $confsrc = "$tmpdir/qemu-server.conf";
6414
6415 my $srcfd = new IO::File($confsrc, "r") ||
6416 die "unable to open file '$confsrc'\n";
6417
6418 my $outfd = new IO::File ($tmpfn, "w") ||
6419 die "unable to write config for VM $vmid\n";
6420
91bd6c90 6421 my $cookie = { netcount => 0 };
3e16d5fc 6422 while (defined (my $line = <$srcfd>)) {
9f3d73bc 6423 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
3e16d5fc
DM		 6424 }
6425
6426 $srcfd->close();
6427 $outfd->close();
6428 };
7dc7f315 6429 if (my $err = $@) {
3e16d5fc 6430 unlink $tmpfn;
ed221350 6431 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
3e16d5fc 6432 die $err;
afdb31d5 6433 }
3e16d5fc
DM		 6434
6435 rmtree $tmpdir;
6436
6437 rename $tmpfn, $conffile ||
6438 die "unable to commit configuration file '$conffile'\n";
91bd6c90 6439
ed221350
DM		 6440 PVE::Cluster::cfs_update(); # make sure we read new file
6441
91bd6c90
DM		 6442 eval { rescan($vmid, 1); };
6443 warn $@ if $@;
3e16d5fc
DM		 6444};
6445
65a5ce88 6446sub foreach_storage_used_by_vm {
18bfb361
DM		 6447 my ($conf, $func) = @_;
6448
6449 my $sidhash = {};
6450
912792e2 6451 PVE::QemuConfig->foreach_volume($conf, sub {
8ddbcf8b
FG		 6452 my ($ds, $drive) = @_;
6453 return if drive_is_cdrom($drive);
18bfb361
DM		 6454
6455 my $volid = $drive->{file};
6456
6457 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
be190583 6458 $sidhash->{$sid} = $sid if $sid;
8ddbcf8b 6459 });
18bfb361
DM		 6460
6461 foreach my $sid (sort keys %$sidhash) {
6462 &$func($sid);
6463 }
6464}
6465
6c9f59c1
TL		 6466my $qemu_snap_storage = {
6467 rbd => 1,
6468};
e5eaa028
WL		 6469sub do_snapshots_with_qemu {
6470 my ($storecfg, $volid) = @_;
6471
6472 my $storage_name = PVE::Storage::parse_volume_id($volid);
8aa2ed7c 6473 my $scfg = $storecfg->{ids}->{$storage_name};
e5eaa028 6474
8aa2ed7c 6475 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
e5eaa028
WL		 6476 return 1;
6477 }
6478
6479 if ($volid =~ m/\.(qcow2|qed)$/){
6480 return 1;
6481 }
6482
6483 return undef;
6484}
6485
4dcc780c 6486sub qga_check_running {
a4938c72 6487 my ($vmid, $nowarn) = @_;
4dcc780c 6488
0a13e08e 6489 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
4dcc780c 6490 if ($@) {
a4938c72 6491 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
4dcc780c
WL		 6492 return 0;
6493 }
6494 return 1;
6495}
6496
04a69bb4
AD		 6497sub template_create {
6498 my ($vmid, $conf, $disk) = @_;
6499
04a69bb4 6500 my $storecfg = PVE::Storage::config();
04a69bb4 6501
912792e2 6502 PVE::QemuConfig->foreach_volume($conf, sub {
9cd07842
DM		 6503 my ($ds, $drive) = @_;
6504
6505 return if drive_is_cdrom($drive);
6506 return if $disk && $ds ne $disk;
6507
6508 my $volid = $drive->{file};
bbd56097 6509 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
9cd07842 6510
04a69bb4
AD		 6511 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6512 $drive->{file} = $voliddst;
71c58bb7 6513 $conf->{$ds} = print_drive($drive);
ffda963f 6514 PVE::QemuConfig->write_config($vmid, $conf);
04a69bb4 6515 });
04a69bb4
AD		 6516}
6517
92bdc3f0
DC		 6518sub convert_iscsi_path {
6519 my ($path) = @_;
6520
6521 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6522 my $portal = $1;
6523 my $target = $2;
6524 my $lun = $3;
6525
6526 my $initiator_name = get_initiator_name();
6527
6528 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6529 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6530 }
6531
6532 die "cannot convert iscsi path '$path', unkown format\n";
6533}
6534
5133de42 6535sub qemu_img_convert {
988e2714 6536 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5133de42
AD		 6537
6538 my $storecfg = PVE::Storage::config();
6539 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6540 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6541
af1f1ec0 6542 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6bb91c17 6543
af1f1ec0
DC		 6544 my $cachemode;
6545 my $src_path;
6546 my $src_is_iscsi = 0;
bdd1feef 6547 my $src_format;
6bb91c17 6548
af1f1ec0
DC		 6549 if ($src_storeid) {
6550 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5133de42 6551 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
af1f1ec0
DC		 6552 $src_format = qemu_img_format($src_scfg, $src_volname);
6553 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6554 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6555 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6556 } elsif (-f $src_volid) {
6557 $src_path = $src_volid;
e0fd2b2f 6558 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
af1f1ec0
DC		 6559 $src_format = $1;
6560 }
6561 }
5133de42 6562
af1f1ec0 6563 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
5133de42 6564
af1f1ec0
DC		 6565 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6566 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6567 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6568 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
5133de42 6569
af1f1ec0
DC		 6570 my $cmd = [];
6571 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
bdd1feef
TL		 6572 push @$cmd, '-l', "snapshot.name=$snapname"
6573 if $snapname && $src_format && $src_format eq "qcow2";
af1f1ec0
DC		 6574 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6575 push @$cmd, '-T', $cachemode if defined($cachemode);
6576
6577 if ($src_is_iscsi) {
6578 push @$cmd, '--image-opts';
6579 $src_path = convert_iscsi_path($src_path);
bdd1feef 6580 } elsif ($src_format) {
af1f1ec0
DC		 6581 push @$cmd, '-f', $src_format;
6582 }
92bdc3f0 6583
af1f1ec0
DC		 6584 if ($dst_is_iscsi) {
6585 push @$cmd, '--target-image-opts';
6586 $dst_path = convert_iscsi_path($dst_path);
6587 } else {
6588 push @$cmd, '-O', $dst_format;
6589 }
92bdc3f0 6590
af1f1ec0 6591 push @$cmd, $src_path;
92bdc3f0 6592
af1f1ec0
DC		 6593 if (!$dst_is_iscsi && $is_zero_initialized) {
6594 push @$cmd, "zeroinit:$dst_path";
6595 } else {
6596 push @$cmd, $dst_path;
6597 }
92bdc3f0 6598
af1f1ec0
DC		 6599 my $parser = sub {
6600 my $line = shift;
6601 if($line =~ m/\((\S+)\/100\%\)/){
6602 my $percent = $1;
6603 my $transferred = int($size * $percent / 100);
6604 my $remaining = $size - $transferred;
92bdc3f0 6605
af1f1ec0 6606 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
988e2714 6607 }
5133de42 6608
af1f1ec0 6609 };
5133de42 6610
af1f1ec0
DC		 6611 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6612 my $err = $@;
6613 die "copy failed: $err" if $err;
5133de42
AD		 6614}
6615
6616sub qemu_img_format {
6617 my ($scfg, $volname) = @_;
6618
e0fd2b2f 6619 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
5133de42 6620 return $1;
be190583 6621 } else {
5133de42 6622 return "raw";
5133de42
AD		 6623 }
6624}
6625
cfad42af 6626sub qemu_drive_mirror {
bc6c8231 6627 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
cfad42af 6628
5a345967
AD		 6629 $jobs = {} if !$jobs;
6630
6631 my $qemu_target;
6632 my $format;
35e4ab04 6633 $jobs->{"drive-$drive"} = {};
152fe752 6634
1e5143de 6635 if ($dst_volid =~ /^nbd:/) {
87955688 6636 $qemu_target = $dst_volid;
5a345967 6637 $format = "nbd";
5a345967 6638 } else {
5a345967
AD		 6639 my $storecfg = PVE::Storage::config();
6640 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6641
6642 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
cfad42af 6643
5a345967 6644 $format = qemu_img_format($dst_scfg, $dst_volname);
21ccdb50 6645
5a345967 6646 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
21ccdb50 6647
5a345967
AD		 6648 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6649 }
988e2714
WB		 6650
6651 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
88383920
DM		 6652 $opts->{format} = $format if $format;
6653
bc6c8231
FG		 6654 if (defined($src_bitmap)) {
6655 $opts->{sync} = 'incremental';
6656 $opts->{bitmap} = $src_bitmap;
6657 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6658 }
6659
9fa05d31 6660 if (defined($bwlimit)) {
f6409f61
TL		 6661 $opts->{speed} = $bwlimit * 1024;
6662 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
9fa05d31
SI		 6663 } else {
6664 print "drive mirror is starting for drive-$drive\n";
6665 }
21ccdb50 6666
6dde5ea2 6667 # if a job already runs for this device we get an error, catch it for cleanup
0a13e08e 6668 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
5a345967
AD		 6669 if (my $err = $@) {
6670 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6dde5ea2
TL		 6671 warn "$@\n" if $@;
6672 die "mirroring error: $err\n";
5a345967
AD		 6673 }
6674
e02fb126 6675 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
5a345967
AD		 6676}
6677
db1f8b39
FG		 6678# $completion can be either
6679# 'complete': wait until all jobs are ready, block-job-complete them (default)
6680# 'cancel': wait until all jobs are ready, block-job-cancel them
6681# 'skip': wait until all jobs are ready, return with block jobs in ready state
5a345967 6682sub qemu_drive_mirror_monitor {
e02fb126
ML		 6683 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6684
db1f8b39 6685 $completion //= 'complete';
2e953867 6686
08ac653f 6687 eval {
5a345967
AD		 6688 my $err_complete = 0;
6689
08ac653f 6690 while (1) {
5a345967
AD		 6691 die "storage migration timed out\n" if $err_complete > 300;
6692
0a13e08e 6693 my $stats = mon_cmd($vmid, "query-block-jobs");
08ac653f 6694
5a345967
AD		 6695 my $running_mirror_jobs = {};
6696 foreach my $stat (@$stats) {
6697 next if $stat->{type} ne 'mirror';
6698 $running_mirror_jobs->{$stat->{device}} = $stat;
6699 }
08ac653f 6700
5a345967 6701 my $readycounter = 0;
67fb9de6 6702
5a345967
AD		 6703 foreach my $job (keys %$jobs) {
6704
6705 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6706 print "$job : finished\n";
6707 delete $jobs->{$job};
6708 next;
6709 }
6710
bd2d5fe6 6711 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
f34ebd52 6712
5a345967
AD		 6713 my $busy = $running_mirror_jobs->{$job}->{busy};
6714 my $ready = $running_mirror_jobs->{$job}->{ready};
6715 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6716 my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6717 my $remaining = $total - $transferred;
6718 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
08ac653f 6719
5a345967
AD		 6720 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6721 }
f34ebd52 6722
d1782eba 6723 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
5a345967 6724 }
b467f79a 6725
5a345967
AD		 6726 last if scalar(keys %$jobs) == 0;
6727
6728 if ($readycounter == scalar(keys %$jobs)) {
6729 print "all mirroring jobs are ready \n";
e02fb126 6730 last if $completion eq 'skip'; #do the complete later
5a345967
AD		 6731
6732 if ($vmiddst && $vmiddst != $vmid) {
1a988fd2
DC		 6733 my $agent_running = $qga && qga_check_running($vmid);
6734 if ($agent_running) {
5619e74a 6735 print "freeze filesystem\n";
0a13e08e 6736 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5619e74a
AD		 6737 } else {
6738 print "suspend vm\n";
6739 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6740 }
6741
5a345967
AD		 6742 # if we clone a disk for a new target vm, we don't switch the disk
6743 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
5619e74a 6744
1a988fd2 6745 if ($agent_running) {
5619e74a 6746 print "unfreeze filesystem\n";
0a13e08e 6747 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5619e74a
AD		 6748 } else {
6749 print "resume vm\n";
6750 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
6751 }
6752
2e953867 6753 last;
5a345967
AD		 6754 } else {
6755
6756 foreach my $job (keys %$jobs) {
6757 # try to switch the disk if source and destination are on the same guest
bd2d5fe6 6758 print "$job: Completing block job...\n";
5a345967 6759
e02fb126 6760 my $op;
db1f8b39 6761 if ($completion eq 'complete') {
e02fb126 6762 $op = 'block-job-complete';
db1f8b39 6763 } elsif ($completion eq 'cancel') {
e02fb126
ML		 6764 $op = 'block-job-cancel';
6765 } else {
6766 die "invalid completion value: $completion\n";
6767 }
6768 eval { mon_cmd($vmid, $op, device => $job) };
5a345967 6769 if ($@ =~ m/cannot be completed/) {
bd2d5fe6 6770 print "$job: Block job cannot be completed, try again.\n";
5a345967
AD		 6771 $err_complete++;
6772 }else {
bd2d5fe6 6773 print "$job: Completed successfully.\n";
5a345967
AD		 6774 $jobs->{$job}->{complete} = 1;
6775 }
6776 }
2e953867 6777 }
08ac653f 6778 }
08ac653f 6779 sleep 1;
cfad42af 6780 }
08ac653f 6781 };
88383920 6782 my $err = $@;
08ac653f 6783
88383920 6784 if ($err) {
5a345967 6785 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
88383920
DM		 6786 die "mirroring error: $err";
6787 }
6788
5a345967
AD		 6789}
6790
6791sub qemu_blockjobs_cancel {
6792 my ($vmid, $jobs) = @_;
6793
6794 foreach my $job (keys %$jobs) {
bd2d5fe6 6795 print "$job: Cancelling block job\n";
0a13e08e 6796 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
5a345967
AD		 6797 $jobs->{$job}->{cancel} = 1;
6798 }
6799
6800 while (1) {
0a13e08e 6801 my $stats = mon_cmd($vmid, "query-block-jobs");
5a345967
AD		 6802
6803 my $running_jobs = {};
6804 foreach my $stat (@$stats) {
6805 $running_jobs->{$stat->{device}} = $stat;
6806 }
6807
6808 foreach my $job (keys %$jobs) {
6809
bd2d5fe6
WB		 6810 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6811 print "$job: Done.\n";
5a345967
AD		 6812 delete $jobs->{$job};
6813 }
6814 }
6815
6816 last if scalar(keys %$jobs) == 0;
6817
6818 sleep 1;
cfad42af
AD		 6819 }
6820}
6821
152fe752 6822sub clone_disk {
be190583 6823 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
818ce80e 6824 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
152fe752
DM		 6825
6826 my $newvolid;
6827
6828 if (!$full) {
6829 print "create linked clone of drive $drivename ($drive->{file})\n";
258e646c 6830 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
152fe752
DM		 6831 push @$newvollist, $newvolid;
6832 } else {
5a345967 6833
152fe752
DM		 6834 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6835 $storeid = $storage if $storage;
6836
44549149 6837 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
152fe752
DM		 6838 my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6839
6840 print "create full clone of drive $drivename ($drive->{file})\n";
931432bd 6841 my $name = undef;
7fe8b44c
TL		 6842 if (drive_is_cloudinit($drive)) {
6843 $name = "vm-$newvmid-cloudinit";
6844 $name .= ".$dst_format" if $dst_format ne 'raw';
6845 $snapname = undef;
6846 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
818ce80e
DC		 6847 } elsif ($drivename eq 'efidisk0') {
6848 $size = get_efivars_size($conf);
7fe8b44c 6849 }
931432bd 6850 $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
152fe752
DM		 6851 push @$newvollist, $newvolid;
6852
3999f370 6853 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
1dbd6d30 6854
7fe8b44c
TL		 6855 if (drive_is_cloudinit($drive)) {
6856 goto no_data_clone;
6857 }
6858
988e2714 6859 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
152fe752 6860 if (!$running || $snapname) {
d189e590 6861 # TODO: handle bwlimits
818ce80e
DC		 6862 if ($drivename eq 'efidisk0') {
6863 # the relevant data on the efidisk may be smaller than the source
6864 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6865 # that is given by the OVMF_VARS.fd
6866 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6867 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6868 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6869 } else {
6870 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6871 }
152fe752 6872 } else {
2e541679
AD		 6873
6874 my $kvmver = get_running_qemu_version ($vmid);
2ea5fb7e 6875 if (!min_version($kvmver, 2, 7)) {
961af8a3
WB		 6876 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6877 if $drive->{iothread};
2e541679 6878 }
2af848a2 6879
db1f8b39 6880 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
be190583 6881 }
152fe752
DM		 6882 }
6883
7fe8b44c 6884no_data_clone:
152fe752
DM		 6885 my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6886
6887 my $disk = $drive;
6888 $disk->{format} = undef;
6889 $disk->{file} = $newvolid;
6890 $disk->{size} = $size;
6891
6892 return $disk;
6893}
6894
98cfd8b6
AD		 6895sub get_running_qemu_version {
6896 my ($vmid) = @_;
0a13e08e 6897 my $res = mon_cmd($vmid, "query-version");
98cfd8b6
AD		 6898 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6899}
6900
249c4a6c
AD		 6901sub qemu_use_old_bios_files {
6902 my ($machine_type) = @_;
6903
6904 return if !$machine_type;
6905
6906 my $use_old_bios_files = undef;
6907
6908 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6909 $machine_type = $1;
6910 $use_old_bios_files = 1;
6911 } else {
9471e48b 6912 my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
249c4a6c
AD		 6913 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6914 # load new efi bios files on migration. So this hack is required to allow
6915 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6916 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
2ea5fb7e 6917 $use_old_bios_files = !min_version($version, 2, 4);
249c4a6c
AD		 6918 }
6919
6920 return ($use_old_bios_files, $machine_type);
6921}
6922
818ce80e
DC		 6923sub get_efivars_size {
6924 my ($conf) = @_;
6925 my $arch = get_vm_arch($conf);
6926 my (undef, $ovmf_vars) = get_ovmf_files($arch);
6927 die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6928 return -s $ovmf_vars;
6929}
6930
6931sub update_efidisk_size {
6932 my ($conf) = @_;
6933
6934 return if !defined($conf->{efidisk0});
6935
6936 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6937 $disk->{size} = get_efivars_size($conf);
6938 $conf->{efidisk0} = print_drive($disk);
6939
6940 return;
6941}
6942
96ed3574
WB		 6943sub create_efidisk($$$$$) {
6944 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
3e1f1122 6945
96ed3574
WB		 6946 my (undef, $ovmf_vars) = get_ovmf_files($arch);
6947 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
3e1f1122 6948
af1f1ec0
DC		 6949 my $vars_size_b = -s $ovmf_vars;
6950 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
3e1f1122
TL		 6951 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6952 PVE::Storage::activate_volumes($storecfg, [$volid]);
6953
af1f1ec0 6954 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
340dbcf7 6955 my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
3e1f1122 6956
340dbcf7 6957 return ($volid, $size/1024);
3e1f1122
TL		 6958}
6959
22de899a
AD		 6960sub vm_iothreads_list {
6961 my ($vmid) = @_;
6962
0a13e08e 6963 my $res = mon_cmd($vmid, 'query-iothreads');
22de899a
AD		 6964
6965 my $iothreads = {};
6966 foreach my $iothread (@$res) {
6967 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6968 }
6969
6970 return $iothreads;
6971}
6972
ee034f5c
AD		 6973sub scsihw_infos {
6974 my ($conf, $drive) = @_;
6975
6976 my $maxdev = 0;
6977
7fe1b688 6978 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
ee034f5c 6979 $maxdev = 7;
a1511b3c 6980 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
ee034f5c
AD		 6981 $maxdev = 1;
6982 } else {
6983 $maxdev = 256;
6984 }
6985
6986 my $controller = int($drive->{index} / $maxdev);
a1511b3c 6987 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
ee034f5c
AD		 6988
6989 return ($maxdev, $controller, $controller_prefix);
6990}
a1511b3c 6991
4317f69f
AD		 6992sub windows_version {
6993 my ($ostype) = @_;
6994
6995 return 0 if !$ostype;
6996
6997 my $winversion = 0;
6998
6999 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7000 $winversion = 5;
7001 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7002 $winversion = 6;
7003 } elsif ($ostype =~ m/^win(\d+)$/) {
7004 $winversion = $1;
7005 }
7006
7007 return $winversion;
7008}
7009
44549149
EK		 7010sub resolve_dst_disk_format {
7011 my ($storecfg, $storeid, $src_volname, $format) = @_;
7012 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7013
7014 if (!$format) {
7015 # if no target format is specified, use the source disk format as hint
7016 if ($src_volname) {
7017 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7018 $format = qemu_img_format($scfg, $src_volname);
7019 } else {
7020 return $defFormat;
7021 }
7022 }
7023
7024 # test if requested format is supported - else use default
7025 my $supported = grep { $_ eq $format } @$validFormats;
7026 $format = $defFormat if !$supported;
7027 return $format;
7028}
7029
66cebc46
DC		 7030# NOTE: if this logic changes, please update docs & possibly gui logic
7031sub find_vmstate_storage {
7032 my ($conf, $storecfg) = @_;
7033
7034 # first, return storage from conf if set
7035 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7036
7037 my ($target, $shared, $local);
7038
7039 foreach_storage_used_by_vm($conf, sub {
7040 my ($sid) = @_;
7041 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7042 my $dst = $scfg->{shared} ? \$shared : \$local;
7043 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7044 });
7045
7046 # second, use shared storage where VM has at least one disk
7047 # third, use local storage where VM has at least one disk
7048 # fall back to local storage
7049 $target = $shared // $local // 'local';
7050
7051 return $target;
7052}
7053
6ee499ff 7054sub generate_uuid {
ae2fcb3b
EK		 7055 my ($uuid, $uuid_str);
7056 UUID::generate($uuid);
7057 UUID::unparse($uuid, $uuid_str);
6ee499ff
DC		 7058 return $uuid_str;
7059}
7060
7061sub generate_smbios1_uuid {
7062 return "uuid=".generate_uuid();
ae2fcb3b
EK		 7063}
7064
9c152e87
TL		 7065sub nbd_stop {
7066 my ($vmid) = @_;
7067
0a13e08e 7068 mon_cmd($vmid, 'nbd-server-stop');
9c152e87
TL		 7069}
7070
dae98db9
DC		 7071sub create_reboot_request {
7072 my ($vmid) = @_;
7073 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7074 or die "failed to create reboot trigger file: $!\n";
7075 close($fh);
7076}
7077
7078sub clear_reboot_request {
7079 my ($vmid) = @_;
7080 my $path = "/run/qemu-server/$vmid.reboot";
7081 my $res = 0;
7082
7083 $res = unlink($path);
7084 die "could not remove reboot request for $vmid: $!"
7085 if !$res && $! != POSIX::ENOENT;
7086
7087 return $res;
7088}
7089
65e866e5
DM		 7090# bash completion helper
7091
7092sub complete_backup_archives {
7093 my ($cmdname, $pname, $cvalue) = @_;
7094
7095 my $cfg = PVE::Storage::config();
7096
7097 my $storeid;
7098
7099 if ($cvalue =~ m/^([^:]+):/) {
7100 $storeid = $1;
7101 }
7102
7103 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7104
7105 my $res = [];
7106 foreach my $id (keys %$data) {
7107 foreach my $item (@{$data->{$id}}) {
f43a4f12 7108 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
65e866e5
DM		 7109 push @$res, $item->{volid} if defined($item->{volid});
7110 }
7111 }
7112
7113 return $res;
7114}
7115
7116my $complete_vmid_full = sub {
7117 my ($running) = @_;
7118
7119 my $idlist = vmstatus();
7120
7121 my $res = [];
7122
7123 foreach my $id (keys %$idlist) {
7124 my $d = $idlist->{$id};
7125 if (defined($running)) {
7126 next if $d->{template};
7127 next if $running && $d->{status} ne 'running';
7128 next if !$running && $d->{status} eq 'running';
7129 }
7130 push @$res, $id;
7131
7132 }
7133 return $res;
7134};
7135
7136sub complete_vmid {
7137 return &$complete_vmid_full();
7138}
7139
7140sub complete_vmid_stopped {
7141 return &$complete_vmid_full(0);
7142}
7143
7144sub complete_vmid_running {
7145 return &$complete_vmid_full(1);
7146}
7147
335af808
DM		 7148sub complete_storage {
7149
7150 my $cfg = PVE::Storage::config();
7151 my $ids = $cfg->{ids};
7152
7153 my $res = [];
7154 foreach my $sid (keys %$ids) {
7155 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
c4c844ef 7156 next if !$ids->{$sid}->{content}->{images};
335af808
DM		 7157 push @$res, $sid;
7158 }
7159
7160 return $res;
7161}
7162
255e9c54
AL		 7163sub complete_migration_storage {
7164 my ($cmd, $param, $current_value, $all_args) = @_;
7165
7166 my $targetnode = @$all_args[1];
7167
7168 my $cfg = PVE::Storage::config();
7169 my $ids = $cfg->{ids};
7170
7171 my $res = [];
7172 foreach my $sid (keys %$ids) {
7173 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7174 next if !$ids->{$sid}->{content}->{images};
7175 push @$res, $sid;
7176 }
7177
7178 return $res;
7179}
7180
1e3baf05 71811;
Virtual Machine Manager