]> git.proxmox.com Git - qemu-server.git/blob - PVE/QemuServer.pm
fix #5440: hmp helpers: drive{add, del}: increase timeout
[qemu-server.git] / PVE / QemuServer.pm
1 package PVE::QemuServer;
2
3 use strict;
4 use warnings;
5
6 use Cwd 'abs_path';
7 use Digest::SHA;
8 use Fcntl ':flock';
9 use Fcntl;
10 use File::Basename;
11 use File::Copy qw(copy);
12 use File::Path;
13 use File::stat;
14 use Getopt::Long;
15 use IO::Dir;
16 use IO::File;
17 use IO::Handle;
18 use IO::Select;
19 use IO::Socket::UNIX;
20 use IPC::Open3;
21 use JSON;
22 use MIME::Base64;
23 use POSIX;
24 use Storable qw(dclone);
25 use Time::HiRes qw(gettimeofday usleep);
26 use URI::Escape;
27 use UUID;
28
29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
30 use PVE::CGroup;
31 use PVE::DataCenterConfig;
32 use PVE::Exception qw(raise raise_param_exc);
33 use PVE::Format qw(render_duration render_bytes);
34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
35 use PVE::INotify;
36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
37 use PVE::ProcFSTools;
38 use PVE::PBSClient;
39 use PVE::RPCEnvironment;
40 use PVE::Storage;
41 use PVE::SysFSTools;
42 use PVE::Systemd;
43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44
45 use PVE::QMPClient;
46 use PVE::QemuConfig;
47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
48 use PVE::QemuServer::Cloudinit;
49 use PVE::QemuServer::CGroup;
50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52 use PVE::QemuServer::Machine;
53 use PVE::QemuServer::Memory;
54 use PVE::QemuServer::Monitor qw(mon_cmd);
55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE::QemuServer::USB qw(parse_usb_device);
57
58 my $have_sdn;
59 eval {
60 require PVE::Network::SDN::Zones;
61 $have_sdn = 1;
62 };
63
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
65 my $OVMF = {
66 x86_64 => {
67 '4m-no-smm' => [
68 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
69 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
70 ],
71 '4m-no-smm-ms' => [
72 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
73 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
74 ],
75 '4m' => [
76 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
78 ],
79 '4m-ms' => [
80 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
82 ],
83 default => [
84 "$EDK2_FW_BASE/OVMF_CODE.fd",
85 "$EDK2_FW_BASE/OVMF_VARS.fd",
86 ],
87 },
88 aarch64 => {
89 default => [
90 "$EDK2_FW_BASE/AAVMF_CODE.fd",
91 "$EDK2_FW_BASE/AAVMF_VARS.fd",
92 ],
93 },
94 };
95
96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
97
98 # Note about locking: we use flock on the config file protect against concurent actions.
99 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
101 # But you can ignore this kind of lock with the --skiplock flag.
102
103 cfs_register_file('/qemu-server/',
104 \&parse_vm_config,
105 \&write_vm_config);
106
107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
108 description => "Some command save/restore state from this location.",
109 type => 'string',
110 maxLength => 128,
111 optional => 1,
112 });
113
114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
115 description => "Specifies the Qemu machine type.",
116 type => 'string',
117 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
118 maxLength => 40,
119 optional => 1,
120 });
121
122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
123 description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
124 type => 'string',
125 format => 'storage-pair-list',
126 optional => 1,
127 });
128
129 #no warnings 'redefine';
130
131 my $nodename_cache;
132 sub nodename {
133 $nodename_cache //= PVE::INotify::nodename();
134 return $nodename_cache;
135 }
136
137 my $watchdog_fmt = {
138 model => {
139 default_key => 1,
140 type => 'string',
141 enum => [qw(i6300esb ib700)],
142 description => "Watchdog type to emulate.",
143 default => 'i6300esb',
144 optional => 1,
145 },
146 action => {
147 type => 'string',
148 enum => [qw(reset shutdown poweroff pause debug none)],
149 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 optional => 1,
151 },
152 };
153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
154
155 my $agent_fmt = {
156 enabled => {
157 description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
158 type => 'boolean',
159 default => 0,
160 default_key => 1,
161 },
162 fstrim_cloned_disks => {
163 description => "Run fstrim after moving a disk or migrating the VM.",
164 type => 'boolean',
165 optional => 1,
166 default => 0
167 },
168 type => {
169 description => "Select the agent type",
170 type => 'string',
171 default => 'virtio',
172 optional => 1,
173 enum => [qw(virtio isa)],
174 },
175 };
176
177 my $vga_fmt = {
178 type => {
179 description => "Select the VGA type.",
180 type => 'string',
181 default => 'std',
182 optional => 1,
183 default_key => 1,
184 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
185 },
186 memory => {
187 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
188 type => 'integer',
189 optional => 1,
190 minimum => 4,
191 maximum => 512,
192 },
193 };
194
195 my $ivshmem_fmt = {
196 size => {
197 type => 'integer',
198 minimum => 1,
199 description => "The size of the file in MB.",
200 },
201 name => {
202 type => 'string',
203 pattern => '[a-zA-Z0-9\-]+',
204 optional => 1,
205 format_description => 'string',
206 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
207 },
208 };
209
210 my $audio_fmt = {
211 device => {
212 type => 'string',
213 enum => [qw(ich9-intel-hda intel-hda AC97)],
214 description => "Configure an audio device."
215 },
216 driver => {
217 type => 'string',
218 enum => ['spice', 'none'],
219 default => 'spice',
220 optional => 1,
221 description => "Driver backend for the audio device."
222 },
223 };
224
225 my $spice_enhancements_fmt = {
226 foldersharing => {
227 type => 'boolean',
228 optional => 1,
229 default => '0',
230 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
231 },
232 videostreaming => {
233 type => 'string',
234 enum => ['off', 'all', 'filter'],
235 default => 'off',
236 optional => 1,
237 description => "Enable video streaming. Uses compression for detected video streams."
238 },
239 };
240
241 my $rng_fmt = {
242 source => {
243 type => 'string',
244 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
245 default_key => 1,
246 description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
247 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
248 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
249 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
250 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
251 ." a hardware RNG from the host.",
252 },
253 max_bytes => {
254 type => 'integer',
255 description => "Maximum bytes of entropy allowed to get injected into the guest every"
256 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
257 ." `0` to disable limiting (potentially dangerous!).",
258 optional => 1,
259
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
261 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
262 # reading from /dev/urandom
263 default => 1024,
264 },
265 period => {
266 type => 'integer',
267 description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
268 ." the guest to retrieve another 'max_bytes' of entropy.",
269 optional => 1,
270 default => 1000,
271 },
272 };
273
274 my $meta_info_fmt = {
275 'ctime' => {
276 type => 'integer',
277 description => "The guest creation timestamp as UNIX epoch time",
278 minimum => 0,
279 optional => 1,
280 },
281 'creation-qemu' => {
282 type => 'string',
283 description => "The QEMU (machine) version from the time this VM was created.",
284 pattern => '\d+(\.\d+)+',
285 optional => 1,
286 },
287 };
288
289 my $confdesc = {
290 onboot => {
291 optional => 1,
292 type => 'boolean',
293 description => "Specifies whether a VM will be started during system bootup.",
294 default => 0,
295 },
296 autostart => {
297 optional => 1,
298 type => 'boolean',
299 description => "Automatic restart after crash (currently ignored).",
300 default => 0,
301 },
302 hotplug => {
303 optional => 1,
304 type => 'string', format => 'pve-hotplug-features',
305 description => "Selectively enable hotplug features. This is a comma separated list of"
306 ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
307 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
308 default => 'network,disk,usb',
309 },
310 reboot => {
311 optional => 1,
312 type => 'boolean',
313 description => "Allow reboot. If set to '0' the VM exit on reboot.",
314 default => 1,
315 },
316 lock => {
317 optional => 1,
318 type => 'string',
319 description => "Lock/unlock the VM.",
320 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
321 },
322 cpulimit => {
323 optional => 1,
324 type => 'number',
325 description => "Limit of CPU usage.",
326 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
327 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
328 minimum => 0,
329 maximum => 128,
330 default => 0,
331 },
332 cpuunits => {
333 optional => 1,
334 type => 'integer',
335 description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
336 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
337 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
338 ." weights of all the other running VMs.",
339 minimum => 1,
340 maximum => 262144,
341 default => 'cgroup v1: 1024, cgroup v2: 100',
342 },
343 memory => {
344 optional => 1,
345 type => 'integer',
346 description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
347 ." you use the balloon device.",
348 minimum => 16,
349 default => 512,
350 },
351 balloon => {
352 optional => 1,
353 type => 'integer',
354 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
355 minimum => 0,
356 },
357 shares => {
358 optional => 1,
359 type => 'integer',
360 description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
361 ." more memory this VM gets. Number is relative to weights of all other running VMs."
362 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
363 minimum => 0,
364 maximum => 50000,
365 default => 1000,
366 },
367 keyboard => {
368 optional => 1,
369 type => 'string',
370 description => "Keyboard layout for VNC server. This option is generally not required and"
371 ." is often better handled from within the guest OS.",
372 enum => PVE::Tools::kvmkeymaplist(),
373 default => undef,
374 },
375 name => {
376 optional => 1,
377 type => 'string', format => 'dns-name',
378 description => "Set a name for the VM. Only used on the configuration web interface.",
379 },
380 scsihw => {
381 optional => 1,
382 type => 'string',
383 description => "SCSI controller model",
384 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
385 default => 'lsi',
386 },
387 description => {
388 optional => 1,
389 type => 'string',
390 description => "Description for the VM. Shown in the web-interface VM's summary."
391 ." This is saved as comment inside the configuration file.",
392 maxLength => 1024 * 8,
393 },
394 ostype => {
395 optional => 1,
396 type => 'string',
397 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
398 description => "Specify guest operating system.",
399 verbose_description => <<EODESC,
400 Specify guest operating system. This is used to enable special
401 optimization/features for specific operating systems:
402
403 [horizontal]
404 other;; unspecified OS
405 wxp;; Microsoft Windows XP
406 w2k;; Microsoft Windows 2000
407 w2k3;; Microsoft Windows 2003
408 w2k8;; Microsoft Windows 2008
409 wvista;; Microsoft Windows Vista
410 win7;; Microsoft Windows 7
411 win8;; Microsoft Windows 8/2012/2012r2
412 win10;; Microsoft Windows 10/2016/2019
413 win11;; Microsoft Windows 11/2022
414 l24;; Linux 2.4 Kernel
415 l26;; Linux 2.6 - 5.X Kernel
416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
417 EODESC
418 },
419 boot => {
420 optional => 1,
421 type => 'string', format => 'pve-qm-boot',
422 description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
423 ." key or 'legacy=' is deprecated.",
424 },
425 bootdisk => {
426 optional => 1,
427 type => 'string', format => 'pve-qm-bootdisk',
428 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
429 pattern => '(ide|sata|scsi|virtio)\d+',
430 },
431 smp => {
432 optional => 1,
433 type => 'integer',
434 description => "The number of CPUs. Please use option -sockets instead.",
435 minimum => 1,
436 default => 1,
437 },
438 sockets => {
439 optional => 1,
440 type => 'integer',
441 description => "The number of CPU sockets.",
442 minimum => 1,
443 default => 1,
444 },
445 cores => {
446 optional => 1,
447 type => 'integer',
448 description => "The number of cores per socket.",
449 minimum => 1,
450 default => 1,
451 },
452 numa => {
453 optional => 1,
454 type => 'boolean',
455 description => "Enable/disable NUMA.",
456 default => 0,
457 },
458 hugepages => {
459 optional => 1,
460 type => 'string',
461 description => "Enable/disable hugepages memory.",
462 enum => [qw(any 2 1024)],
463 },
464 keephugepages => {
465 optional => 1,
466 type => 'boolean',
467 default => 0,
468 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
469 ." after VM shutdown and can be used for subsequent starts.",
470 },
471 vcpus => {
472 optional => 1,
473 type => 'integer',
474 description => "Number of hotplugged vcpus.",
475 minimum => 1,
476 default => 0,
477 },
478 acpi => {
479 optional => 1,
480 type => 'boolean',
481 description => "Enable/disable ACPI.",
482 default => 1,
483 },
484 agent => {
485 optional => 1,
486 description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
487 type => 'string',
488 format => $agent_fmt,
489 },
490 kvm => {
491 optional => 1,
492 type => 'boolean',
493 description => "Enable/disable KVM hardware virtualization.",
494 default => 1,
495 },
496 tdf => {
497 optional => 1,
498 type => 'boolean',
499 description => "Enable/disable time drift fix.",
500 default => 0,
501 },
502 localtime => {
503 optional => 1,
504 type => 'boolean',
505 description => "Set the real time clock (RTC) to local time. This is enabled by default if"
506 ." the `ostype` indicates a Microsoft Windows OS.",
507 },
508 freeze => {
509 optional => 1,
510 type => 'boolean',
511 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
512 },
513 vga => {
514 optional => 1,
515 type => 'string', format => $vga_fmt,
516 description => "Configure the VGA hardware.",
517 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
518 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
519 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
520 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
521 ." display server. For win* OS you can select how many independent displays you want,"
522 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
523 ." using a serial device as terminal.",
524 },
525 watchdog => {
526 optional => 1,
527 type => 'string', format => 'pve-qm-watchdog',
528 description => "Create a virtual hardware watchdog device.",
529 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
530 ." action), the watchdog must be periodically polled by an agent inside the guest or"
531 ." else the watchdog will reset the guest (or execute the respective action specified)",
532 },
533 startdate => {
534 optional => 1,
535 type => 'string',
536 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
537 description => "Set the initial date of the real time clock. Valid format for date are:"
538 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
539 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
540 default => 'now',
541 },
542 startup => get_standard_option('pve-startup-order'),
543 template => {
544 optional => 1,
545 type => 'boolean',
546 description => "Enable/disable Template.",
547 default => 0,
548 },
549 args => {
550 optional => 1,
551 type => 'string',
552 description => "Arbitrary arguments passed to kvm.",
553 verbose_description => <<EODESCR,
554 Arbitrary arguments passed to kvm, for example:
555
556 args: -no-reboot -no-hpet
557
558 NOTE: this option is for experts only.
559 EODESCR
560 },
561 tablet => {
562 optional => 1,
563 type => 'boolean',
564 default => 1,
565 description => "Enable/disable the USB tablet device.",
566 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
567 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
568 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
569 ." may consider disabling this to save some context switches. This is turned off by"
570 ." default if you use spice (`qm set <vmid> --vga qxl`).",
571 },
572 migrate_speed => {
573 optional => 1,
574 type => 'integer',
575 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
576 minimum => 0,
577 default => 0,
578 },
579 migrate_downtime => {
580 optional => 1,
581 type => 'number',
582 description => "Set maximum tolerated downtime (in seconds) for migrations.",
583 minimum => 0,
584 default => 0.1,
585 },
586 cdrom => {
587 optional => 1,
588 type => 'string', format => 'pve-qm-ide',
589 typetext => '<volume>',
590 description => "This is an alias for option -ide2",
591 },
592 cpu => {
593 optional => 1,
594 description => "Emulated CPU type.",
595 type => 'string',
596 format => 'pve-vm-cpu-conf',
597 },
598 parent => get_standard_option('pve-snapshot-name', {
599 optional => 1,
600 description => "Parent snapshot name. This is used internally, and should not be modified.",
601 }),
602 snaptime => {
603 optional => 1,
604 description => "Timestamp for snapshots.",
605 type => 'integer',
606 minimum => 0,
607 },
608 vmstate => {
609 optional => 1,
610 type => 'string', format => 'pve-volume-id',
611 description => "Reference to a volume which stores the VM state. This is used internally"
612 ." for snapshots.",
613 },
614 vmstatestorage => get_standard_option('pve-storage-id', {
615 description => "Default storage for VM state volumes/files.",
616 optional => 1,
617 }),
618 runningmachine => get_standard_option('pve-qemu-machine', {
619 description => "Specifies the QEMU machine type of the running vm. This is used internally"
620 ." for snapshots.",
621 }),
622 runningcpu => {
623 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
624 ." internally for snapshots.",
625 optional => 1,
626 type => 'string',
627 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
628 format_description => 'QEMU -cpu parameter'
629 },
630 machine => get_standard_option('pve-qemu-machine'),
631 arch => {
632 description => "Virtual processor architecture. Defaults to the host.",
633 optional => 1,
634 type => 'string',
635 enum => [qw(x86_64 aarch64)],
636 },
637 smbios1 => {
638 description => "Specify SMBIOS type 1 fields.",
639 type => 'string', format => 'pve-qm-smbios1',
640 maxLength => 512,
641 optional => 1,
642 },
643 protection => {
644 optional => 1,
645 type => 'boolean',
646 description => "Sets the protection flag of the VM. This will disable the remove VM and"
647 ." remove disk operations.",
648 default => 0,
649 },
650 bios => {
651 optional => 1,
652 type => 'string',
653 enum => [ qw(seabios ovmf) ],
654 description => "Select BIOS implementation.",
655 default => 'seabios',
656 },
657 vmgenid => {
658 type => 'string',
659 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
660 format_description => 'UUID',
661 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
662 ." to disable explicitly.",
663 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
664 ." value identifier to the guest OS. This allows to notify the guest operating system"
665 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
666 ." execution or creation from a template). The guest operating system notices the"
667 ." change, and is then able to react as appropriate by marking its copies of"
668 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
669 ."Note that auto-creation only works when done through API/CLI create or update methods"
670 .", but not when manually editing the config file.",
671 default => "1 (autogenerated)",
672 optional => 1,
673 },
674 hookscript => {
675 type => 'string',
676 format => 'pve-volume-id',
677 optional => 1,
678 description => "Script that will be executed during various steps in the vms lifetime.",
679 },
680 ivshmem => {
681 type => 'string',
682 format => $ivshmem_fmt,
683 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
684 ." the host.",
685 optional => 1,
686 },
687 audio0 => {
688 type => 'string',
689 format => $audio_fmt,
690 description => "Configure a audio device, useful in combination with QXL/Spice.",
691 optional => 1
692 },
693 spice_enhancements => {
694 type => 'string',
695 format => $spice_enhancements_fmt,
696 description => "Configure additional enhancements for SPICE.",
697 optional => 1
698 },
699 tags => {
700 type => 'string', format => 'pve-tag-list',
701 description => 'Tags of the VM. This is only meta information.',
702 optional => 1,
703 },
704 rng0 => {
705 type => 'string',
706 format => $rng_fmt,
707 description => "Configure a VirtIO-based Random Number Generator.",
708 optional => 1,
709 },
710 meta => {
711 type => 'string',
712 format => $meta_info_fmt,
713 description => "Some (read-only) meta-information about this guest.",
714 optional => 1,
715 },
716 };
717
718 my $cicustom_fmt = {
719 meta => {
720 type => 'string',
721 optional => 1,
722 description => 'Specify a custom file containing all meta data passed to the VM via"
723 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
724 format => 'pve-volume-id',
725 format_description => 'volume',
726 },
727 network => {
728 type => 'string',
729 optional => 1,
730 description => 'Specify a custom file containing all network data passed to the VM via'
731 .' cloud-init.',
732 format => 'pve-volume-id',
733 format_description => 'volume',
734 },
735 user => {
736 type => 'string',
737 optional => 1,
738 description => 'Specify a custom file containing all user data passed to the VM via'
739 .' cloud-init.',
740 format => 'pve-volume-id',
741 format_description => 'volume',
742 },
743 vendor => {
744 type => 'string',
745 optional => 1,
746 description => 'Specify a custom file containing all vendor data passed to the VM via'
747 .' cloud-init.',
748 format => 'pve-volume-id',
749 format_description => 'volume',
750 },
751 };
752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
753
754 my $confdesc_cloudinit = {
755 citype => {
756 optional => 1,
757 type => 'string',
758 description => 'Specifies the cloud-init configuration format. The default depends on the'
759 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
760 .' and `configdrive2` for windows.',
761 enum => ['configdrive2', 'nocloud', 'opennebula'],
762 },
763 ciuser => {
764 optional => 1,
765 type => 'string',
766 description => "cloud-init: User name to change ssh keys and password for instead of the"
767 ." image's configured default user.",
768 },
769 cipassword => {
770 optional => 1,
771 type => 'string',
772 description => 'cloud-init: Password to assign the user. Using this is generally not'
773 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
774 .' support hashed passwords.',
775 },
776 cicustom => {
777 optional => 1,
778 type => 'string',
779 description => 'cloud-init: Specify custom files to replace the automatically generated'
780 .' ones at start.',
781 format => 'pve-qm-cicustom',
782 },
783 searchdomain => {
784 optional => 1,
785 type => 'string',
786 description => 'cloud-init: Sets DNS search domains for a container. Create will'
787 .' automatically use the setting from the host if neither searchdomain nor nameserver'
788 .' are set.',
789 },
790 nameserver => {
791 optional => 1,
792 type => 'string', format => 'address-list',
793 description => 'cloud-init: Sets DNS server IP address for a container. Create will'
794 .' automatically use the setting from the host if neither searchdomain nor nameserver'
795 .' are set.',
796 },
797 sshkeys => {
798 optional => 1,
799 type => 'string',
800 format => 'urlencoded',
801 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
802 },
803 };
804
805 # what about other qemu settings ?
806 #cpu => 'string',
807 #machine => 'string',
808 #fda => 'file',
809 #fdb => 'file',
810 #mtdblock => 'file',
811 #sd => 'file',
812 #pflash => 'file',
813 #snapshot => 'bool',
814 #bootp => 'file',
815 ##tftp => 'dir',
816 ##smb => 'dir',
817 #kernel => 'file',
818 #append => 'string',
819 #initrd => 'file',
820 ##soundhw => 'string',
821
822 while (my ($k, $v) = each %$confdesc) {
823 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
824 }
825
826 my $MAX_USB_DEVICES = 5;
827 my $MAX_NETS = 32;
828 my $MAX_SERIAL_PORTS = 4;
829 my $MAX_PARALLEL_PORTS = 3;
830 my $MAX_NUMA = 8;
831
832 my $numa_fmt = {
833 cpus => {
834 type => "string",
835 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
836 description => "CPUs accessing this NUMA node.",
837 format_description => "id[-id];...",
838 },
839 memory => {
840 type => "number",
841 description => "Amount of memory this NUMA node provides.",
842 optional => 1,
843 },
844 hostnodes => {
845 type => "string",
846 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
847 description => "Host NUMA nodes to use.",
848 format_description => "id[-id];...",
849 optional => 1,
850 },
851 policy => {
852 type => 'string',
853 enum => [qw(preferred bind interleave)],
854 description => "NUMA allocation policy.",
855 optional => 1,
856 },
857 };
858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
859 my $numadesc = {
860 optional => 1,
861 type => 'string', format => $numa_fmt,
862 description => "NUMA topology.",
863 };
864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
865
866 for (my $i = 0; $i < $MAX_NUMA; $i++) {
867 $confdesc->{"numa$i"} = $numadesc;
868 }
869
870 my $nic_model_list = [
871 'e1000',
872 'e1000-82540em',
873 'e1000-82544gc',
874 'e1000-82545em',
875 'e1000e',
876 'i82551',
877 'i82557b',
878 'i82559er',
879 'ne2k_isa',
880 'ne2k_pci',
881 'pcnet',
882 'rtl8139',
883 'virtio',
884 'vmxnet3',
885 ];
886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
887
888 my $net_fmt_bridge_descr = <<__EOD__;
889 Bridge to attach the network device to. The Proxmox VE standard bridge
890 is called 'vmbr0'.
891
892 If you do not specify a bridge, we create a kvm user (NATed) network
893 device, which provides DHCP and DNS services. The following addresses
894 are used:
895
896 10.0.2.2 Gateway
897 10.0.2.3 DNS Server
898 10.0.2.4 SMB Server
899
900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
901 __EOD__
902
903 my $net_fmt = {
904 macaddr => get_standard_option('mac-addr', {
905 description => "MAC address. That address must be unique withing your network. This is"
906 ." automatically generated if not specified.",
907 }),
908 model => {
909 type => 'string',
910 description => "Network Card Model. The 'virtio' model provides the best performance with"
911 ." very low CPU overhead. If your guest does not support this driver, it is usually"
912 ." best to use 'e1000'.",
913 enum => $nic_model_list,
914 default_key => 1,
915 },
916 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
917 bridge => get_standard_option('pve-bridge-id', {
918 description => $net_fmt_bridge_descr,
919 optional => 1,
920 }),
921 queues => {
922 type => 'integer',
923 minimum => 0, maximum => 16,
924 description => 'Number of packet queues to be used on the device.',
925 optional => 1,
926 },
927 rate => {
928 type => 'number',
929 minimum => 0,
930 description => "Rate limit in mbps (megabytes per second) as floating point number.",
931 optional => 1,
932 },
933 tag => {
934 type => 'integer',
935 minimum => 1, maximum => 4094,
936 description => 'VLAN tag to apply to packets on this interface.',
937 optional => 1,
938 },
939 trunks => {
940 type => 'string',
941 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
942 description => 'VLAN trunks to pass through this interface.',
943 format_description => 'vlanid[;vlanid...]',
944 optional => 1,
945 },
946 firewall => {
947 type => 'boolean',
948 description => 'Whether this interface should be protected by the firewall.',
949 optional => 1,
950 },
951 link_down => {
952 type => 'boolean',
953 description => 'Whether this interface should be disconnected (like pulling the plug).',
954 optional => 1,
955 },
956 mtu => {
957 type => 'integer',
958 minimum => 1, maximum => 65520,
959 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
960 optional => 1,
961 },
962 };
963
964 my $netdesc = {
965 optional => 1,
966 type => 'string', format => $net_fmt,
967 description => "Specify network devices.",
968 };
969
970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
971
972 my $ipconfig_fmt = {
973 ip => {
974 type => 'string',
975 format => 'pve-ipv4-config',
976 format_description => 'IPv4Format/CIDR',
977 description => 'IPv4 address in CIDR format.',
978 optional => 1,
979 default => 'dhcp',
980 },
981 gw => {
982 type => 'string',
983 format => 'ipv4',
984 format_description => 'GatewayIPv4',
985 description => 'Default gateway for IPv4 traffic.',
986 optional => 1,
987 requires => 'ip',
988 },
989 ip6 => {
990 type => 'string',
991 format => 'pve-ipv6-config',
992 format_description => 'IPv6Format/CIDR',
993 description => 'IPv6 address in CIDR format.',
994 optional => 1,
995 default => 'dhcp',
996 },
997 gw6 => {
998 type => 'string',
999 format => 'ipv6',
1000 format_description => 'GatewayIPv6',
1001 description => 'Default gateway for IPv6 traffic.',
1002 optional => 1,
1003 requires => 'ip6',
1004 },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008 optional => 1,
1009 type => 'string', format => 'pve-qm-ipconfig',
1010 description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++) {
1027 $confdesc->{"net$i"} = $netdesc;
1028 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037 my ($volid, $noerr) = @_;
1038
1039 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041 return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046 my ($volid, $noerr) = @_;
1047
1048 return $volid if $volid =~ m|^/|;
1049
1050 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051 if ($@) {
1052 return if $noerr;
1053 die $@;
1054 }
1055 return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059 host => {
1060 default_key => 1,
1061 type => 'string', format => 'pve-qm-usb-device',
1062 format_description => 'HOSTUSBDEVICE|spice',
1063 description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066 'bus-port(.port)*' (decimal numbers) or
1067 'vendor_id:product_id' (hexadeciaml numbers) or
1068 'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077 },
1078 usb3 => {
1079 optional => 1,
1080 type => 'boolean',
1081 description => "Specifies whether if given host option is a USB3 device or port.",
1082 default => 0,
1083 },
1084 };
1085
1086 my $usbdesc = {
1087 optional => 1,
1088 type => 'string', format => $usb_fmt,
1089 description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094 optional => 1,
1095 type => 'string',
1096 pattern => '(/dev/.+|socket)',
1097 description => "Create a serial device inside the VM (n is 0 to 3)",
1098 verbose_description => <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111 optional => 1,
1112 type => 'string',
1113 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114 description => "Map host parallel devices (n is 0 to 2).",
1115 verbose_description => <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1126 $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1130 $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1134 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1142 $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146 legacy => {
1147 optional => 1,
1148 default_key => 1,
1149 type => 'string',
1150 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151 . " Deprecated, use 'order=' instead.",
1152 pattern => '[acdn]{1,4}',
1153 format_description => "[acdn]{1,4}",
1154
1155 # note: this is also the fallback if boot: is not given at all
1156 default => 'cdn',
1157 },
1158 order => {
1159 optional => 1,
1160 type => 'string',
1161 format => 'pve-qm-bootdev-list',
1162 format_description => "device[;device...]",
1163 description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176 },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182 my ($dev, $noerr) = @_;
1183
1184 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187 my $check = sub {
1188 my ($base) = @_;
1189 return 0 if $dev !~ m/^$base\d+$/;
1190 return 0 if !$confdesc->{$dev};
1191 return 1;
1192 };
1193
1194 return $dev if $check->("net");
1195 return $dev if $check->("usb");
1196 return $dev if $check->("hostpci");
1197
1198 return if $noerr;
1199 die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203 my ($devs) = @_;
1204 return "" if !@$devs;
1205 my $data = { order => join(';', @$devs) };
1206 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212 return $kvm_api_version if $kvm_api_version;
1213
1214 open my $fh, '<', '/dev/kvm' or return;
1215
1216 # 0xae00 => KVM_GET_API_VERSION
1217 $kvm_api_version = ioctl($fh, 0xae00, 0);
1218 close($fh);
1219
1220 return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227 my ($binary) = @_;
1228
1229 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230 my $st = stat($binary);
1231
1232 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234 $cachedmtime == $st->mtime;
1235
1236 $kvm_user_version->{$binary} = 'unknown';
1237 $kvm_mtime->{$binary} = $st->mtime;
1238
1239 my $code = sub {
1240 my $line = shift;
1241 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242 $kvm_user_version->{$binary} = $2;
1243 }
1244 };
1245
1246 eval { run_command([$binary, '--version'], outfunc => $code); };
1247 warn $@ if $@;
1248
1249 return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253 my ($machine_type, $version) = @_;
1254 $version = kvm_user_version() if !defined($version);
1255 return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259 return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263 my $key = shift;
1264 return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270 return $cdrom_path if $cdrom_path;
1271
1272 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278 my ($storecfg, $vmid, $cdrom) = @_;
1279
1280 if ($cdrom eq 'cdrom') {
1281 return get_cdrom_path();
1282 } elsif ($cdrom eq 'none') {
1283 return '';
1284 } elsif ($cdrom =~ m|^/|) {
1285 return $cdrom;
1286 } else {
1287 return PVE::Storage::path($storecfg, $cdrom);
1288 }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293 my ($vmid, $file, $media) = @_;
1294
1295 if (!($file eq 'none' || $file eq 'cdrom' ||
1296 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298 return if $file =~ m|/|;
1299
1300 if ($media && $media eq 'cdrom') {
1301 $file = "local:iso/$file";
1302 } else {
1303 $file = "local:$vmid/$file";
1304 }
1305 }
1306
1307 return $file;
1308 }
1309
1310 sub verify_media_type {
1311 my ($opt, $vtype, $media) = @_;
1312
1313 return if !$media;
1314
1315 my $etype;
1316 if ($media eq 'disk') {
1317 $etype = 'images';
1318 } elsif ($media eq 'cdrom') {
1319 $etype = 'iso';
1320 } else {
1321 die "internal error";
1322 }
1323
1324 return if ($vtype eq $etype);
1325
1326 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330 my ($opt, $storecfg, $drive) = @_;
1331
1332 # try to convert filesystem paths to volume IDs
1333
1334 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335 ($drive->{file} !~ m|^/dev/.+|) &&
1336 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337 ($drive->{file} !~ m/^\d+$/)) {
1338 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340 if !$vtype;
1341 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342 verify_media_type($opt, $vtype, $drive->{media});
1343 $drive->{file} = $volid;
1344 }
1345
1346 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350 my ($data) = @_;
1351
1352 my $res = {};
1353
1354 return $res if $data eq '0';
1355
1356 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358 foreach my $feature (PVE::Tools::split_list($data)) {
1359 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360 $res->{$1} = 1;
1361 } else {
1362 die "invalid hotplug feature '$feature'\n";
1363 }
1364 }
1365 return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370 my ($value, $noerr) = @_;
1371
1372 return $value if parse_hotplug_features($value);
1373
1374 return if $noerr;
1375
1376 die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380 my($fh, $noerr) = @_;
1381
1382 my $SG_IO = 0x2285;
1383 my $SG_GET_VERSION_NUM = 0x2282;
1384
1385 my $versionbuf = "\x00" x 8;
1386 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387 if (!$ret) {
1388 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389 return;
1390 }
1391 my $version = unpack("I", $versionbuf);
1392 if ($version < 30000) {
1393 die "scsi generic interface too old\n" if !$noerr;
1394 return;
1395 }
1396
1397 my $buf = "\x00" x 36;
1398 my $sensebuf = "\x00" x 8;
1399 my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401 # see /usr/include/scsi/sg.h
1402 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404 my $packet = pack(
1405 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406 );
1407
1408 $ret = ioctl($fh, $SG_IO, $packet);
1409 if (!$ret) {
1410 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411 return;
1412 }
1413
1414 my @res = unpack($sg_io_hdr_t, $packet);
1415 if ($res[17] || $res[18]) {
1416 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417 return;
1418 }
1419
1420 my $res = {};
1421 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423 $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424 $res->{type} &= 0x1F;
1425
1426 return $res;
1427 }
1428
1429 sub path_is_scsi {
1430 my ($path) = @_;
1431
1432 my $fh = IO::File->new("+<$path") || return;
1433 my $res = scsi_inquiry($fh, 1);
1434 close($fh);
1435
1436 return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440 my ($conf, $arch) = @_;
1441
1442 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444 # we use uhci for old VMs because tablet driver was buggy in older qemu
1445 my $usbbus;
1446 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447 $usbbus = 'ehci';
1448 } else {
1449 $usbbus = 'uhci';
1450 }
1451
1452 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456 my ($conf, $arch) = @_;
1457
1458 return if $arch ne 'aarch64';
1459
1460 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464 my ($drive) = @_;
1465 return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471 my $device = '';
1472 my $maxdev = 0;
1473
1474 my $drive_id = get_drive_id($drive);
1475 if ($drive->{interface} eq 'virtio') {
1476 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479 } elsif ($drive->{interface} eq 'scsi') {
1480
1481 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482 my $unit = $drive->{index} % $maxdev;
1483 my $devicetype = 'hd';
1484 my $path = '';
1485 if (drive_is_cdrom($drive)) {
1486 $devicetype = 'cd';
1487 } else {
1488 if ($drive->{file} =~ m|^/|) {
1489 $path = $drive->{file};
1490 if (my $info = path_is_scsi($path)) {
1491 if ($info->{type} == 0 && $drive->{scsiblock}) {
1492 $devicetype = 'block';
1493 } elsif ($info->{type} == 1) { # tape
1494 $devicetype = 'generic';
1495 }
1496 }
1497 } else {
1498 $path = PVE::Storage::path($storecfg, $drive->{file});
1499 }
1500
1501 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502 my $version = extract_version($machine_type, kvm_user_version());
1503 if ($path =~ m/^iscsi\:\/\// &&
1504 !min_version($version, 4, 1)) {
1505 $devicetype = 'generic';
1506 }
1507 }
1508
1509 if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511 } else {
1512 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513 .",lun=$drive->{index}";
1514 }
1515 $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518 $device .= ",rotation_rate=1";
1519 }
1520 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524 my $controller = int($drive->{index} / $maxdev);
1525 my $unit = $drive->{index} % $maxdev;
1526 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528 $device = "ide-$devicetype";
1529 if ($drive->{interface} eq 'ide') {
1530 $device .= ",bus=ide.$controller,unit=$unit";
1531 } else {
1532 $device .= ",bus=ahci$controller.$unit";
1533 }
1534 $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536 if ($devicetype eq 'hd') {
1537 if (my $model = $drive->{model}) {
1538 $model = URI::Escape::uri_unescape($model);
1539 $device .= ",model=$model";
1540 }
1541 if ($drive->{ssd}) {
1542 $device .= ",rotation_rate=1";
1543 }
1544 }
1545 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546 } elsif ($drive->{interface} eq 'usb') {
1547 die "implement me";
1548 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549 } else {
1550 die "unsupported interface type";
1551 }
1552
1553 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555 if (my $serial = $drive->{serial}) {
1556 $serial = URI::Escape::uri_unescape($serial);
1557 $device .= ",serial=$serial";
1558 }
1559
1560
1561 return $device;
1562 }
1563
1564 sub get_initiator_name {
1565 my $initiator;
1566
1567 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568 while (defined(my $line = <$fh>)) {
1569 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570 $initiator = $1;
1571 last;
1572 }
1573 $fh->close();
1574
1575 return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581 my $path;
1582 my $volid = $drive->{file};
1583 my $format = $drive->{format};
1584 my $drive_id = get_drive_id($drive);
1585
1586 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587 my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589 if (drive_is_cdrom($drive)) {
1590 $path = get_iso_path($storecfg, $vmid, $volid);
1591 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592 } else {
1593 if ($storeid) {
1594 $path = PVE::Storage::path($storecfg, $volid);
1595 $format //= qemu_img_format($scfg, $volname);
1596 } else {
1597 $path = $volid;
1598 $format //= "raw";
1599 }
1600 }
1601
1602 my $is_rbd = $path =~ m/^rbd:/;
1603
1604 my $opts = '';
1605 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606 foreach my $o (@qemu_drive_options) {
1607 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608 }
1609
1610 # snapshot only accepts on|off
1611 if (defined($drive->{snapshot})) {
1612 my $v = $drive->{snapshot} ? 'on' : 'off';
1613 $opts .= ",snapshot=$v";
1614 }
1615
1616 if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617 $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618 }
1619
1620 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621 my ($dir, $qmpname) = @$type;
1622 if (my $v = $drive->{"mbps$dir"}) {
1623 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624 }
1625 if (my $v = $drive->{"mbps${dir}_max"}) {
1626 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627 }
1628 if (my $v = $drive->{"bps${dir}_max_length"}) {
1629 $opts .= ",throttling.bps$qmpname-max-length=$v";
1630 }
1631 if (my $v = $drive->{"iops${dir}"}) {
1632 $opts .= ",throttling.iops$qmpname=$v";
1633 }
1634 if (my $v = $drive->{"iops${dir}_max"}) {
1635 $opts .= ",throttling.iops$qmpname-max=$v";
1636 }
1637 if (my $v = $drive->{"iops${dir}_max_length"}) {
1638 $opts .= ",throttling.iops$qmpname-max-length=$v";
1639 }
1640 }
1641
1642 if ($pbs_name) {
1643 $format = "rbd" if $is_rbd;
1644 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645 if !$format;
1646 $opts .= ",format=alloc-track,file.driver=$format";
1647 } elsif ($format) {
1648 $opts .= ",format=$format";
1649 }
1650
1651 my $cache_direct = 0;
1652
1653 if (my $cache = $drive->{cache}) {
1654 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655 } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656 $opts .= ",cache=none";
1657 $cache_direct = 1;
1658 }
1659
1660 # io_uring with cache mode writeback or writethrough on krbd will hang...
1661 my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664 # sometimes, just plain disable...
1665 my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667 if (!$drive->{aio}) {
1668 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669 # io_uring supports all cache modes
1670 $opts .= ",aio=io_uring";
1671 } else {
1672 # aio native works only with O_DIRECT
1673 if($cache_direct) {
1674 $opts .= ",aio=native";
1675 } else {
1676 $opts .= ",aio=threads";
1677 }
1678 }
1679 }
1680
1681 if (!drive_is_cdrom($drive)) {
1682 my $detectzeroes;
1683 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684 $detectzeroes = 'off';
1685 } elsif ($drive->{discard}) {
1686 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687 } else {
1688 # This used to be our default with discard not being specified:
1689 $detectzeroes = 'on';
1690 }
1691
1692 # note: 'detect-zeroes' works per blockdev and we want it to persist
1693 # after the alloc-track is removed, so put it on 'file' directly
1694 my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696 }
1697
1698 if ($pbs_name) {
1699 $opts .= ",backing=$pbs_name";
1700 $opts .= ",auto-remove=on";
1701 }
1702
1703 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704 my $file_param = "file";
1705 if ($pbs_name) {
1706 # non-rbd drivers require the underlying file to be a seperate block
1707 # node, so add a second .file indirection
1708 $file_param .= ".file" if !$is_rbd;
1709 $file_param .= ".filename";
1710 }
1711 my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717 my ($pbs_conf, $pbs_name) = @_;
1718 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719 $blockdev .= ",repository=$pbs_conf->{repository}";
1720 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1721 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1722 $blockdev .= ",archive=$pbs_conf->{archive}";
1723 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1724 return $blockdev;
1725 }
1726
1727 sub print_netdevice_full {
1728 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1729
1730 my $device = $net->{model};
1731 if ($net->{model} eq 'virtio') {
1732 $device = 'virtio-net-pci';
1733 };
1734
1735 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1736 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1737 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1738 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1739 # and out of each queue plus one config interrupt and control vector queue
1740 my $vectors = $net->{queues} * 2 + 2;
1741 $tmpstr .= ",vectors=$vectors,mq=on";
1742 }
1743 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1744
1745 if (my $mtu = $net->{mtu}) {
1746 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1747 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1748 if ($mtu == 1) {
1749 $mtu = $bridge_mtu;
1750 } elsif ($mtu < 576) {
1751 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1752 } elsif ($mtu > $bridge_mtu) {
1753 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1754 }
1755 $tmpstr .= ",host_mtu=$mtu";
1756 } else {
1757 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1758 }
1759 }
1760
1761 if ($use_old_bios_files) {
1762 my $romfile;
1763 if ($device eq 'virtio-net-pci') {
1764 $romfile = 'pxe-virtio.rom';
1765 } elsif ($device eq 'e1000') {
1766 $romfile = 'pxe-e1000.rom';
1767 } elsif ($device eq 'e1000e') {
1768 $romfile = 'pxe-e1000e.rom';
1769 } elsif ($device eq 'ne2k') {
1770 $romfile = 'pxe-ne2k_pci.rom';
1771 } elsif ($device eq 'pcnet') {
1772 $romfile = 'pxe-pcnet.rom';
1773 } elsif ($device eq 'rtl8139') {
1774 $romfile = 'pxe-rtl8139.rom';
1775 }
1776 $tmpstr .= ",romfile=$romfile" if $romfile;
1777 }
1778
1779 return $tmpstr;
1780 }
1781
1782 sub print_netdev_full {
1783 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1784
1785 my $i = '';
1786 if ($netid =~ m/^net(\d+)$/) {
1787 $i = int($1);
1788 }
1789
1790 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1791
1792 my $ifname = "tap${vmid}i$i";
1793
1794 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1795 die "interface name '$ifname' is too long (max 15 character)\n"
1796 if length($ifname) >= 16;
1797
1798 my $vhostparam = '';
1799 if (is_native($arch)) {
1800 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1801 }
1802
1803 my $vmname = $conf->{name} || "vm$vmid";
1804
1805 my $netdev = "";
1806 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1807
1808 if ($net->{bridge}) {
1809 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1810 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1811 } else {
1812 $netdev = "type=user,id=$netid,hostname=$vmname";
1813 }
1814
1815 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1816
1817 return $netdev;
1818 }
1819
1820 my $vga_map = {
1821 'cirrus' => 'cirrus-vga',
1822 'std' => 'VGA',
1823 'vmware' => 'vmware-svga',
1824 'virtio' => 'virtio-vga',
1825 'virtio-gl' => 'virtio-vga-gl',
1826 };
1827
1828 sub print_vga_device {
1829 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1830
1831 my $type = $vga_map->{$vga->{type}};
1832 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1833 $type = 'virtio-gpu';
1834 }
1835 my $vgamem_mb = $vga->{memory};
1836
1837 my $max_outputs = '';
1838 if ($qxlnum) {
1839 $type = $id ? 'qxl' : 'qxl-vga';
1840
1841 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1842 # set max outputs so linux can have up to 4 qxl displays with one device
1843 if (min_version($machine_version, 4, 1)) {
1844 $max_outputs = ",max_outputs=4";
1845 }
1846 }
1847 }
1848
1849 die "no devicetype for $vga->{type}\n" if !$type;
1850
1851 my $memory = "";
1852 if ($vgamem_mb) {
1853 if ($vga->{type} =~ /^virtio/) {
1854 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1855 $memory = ",max_hostmem=$bytes";
1856 } elsif ($qxlnum) {
1857 # from https://www.spice-space.org/multiple-monitors.html
1858 $memory = ",vgamem_mb=$vga->{memory}";
1859 my $ram = $vgamem_mb * 4;
1860 my $vram = $vgamem_mb * 2;
1861 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1862 } else {
1863 $memory = ",vgamem_mb=$vga->{memory}";
1864 }
1865 } elsif ($qxlnum && $id) {
1866 $memory = ",ram_size=67108864,vram_size=33554432";
1867 }
1868
1869 my $edidoff = "";
1870 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1871 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1872 }
1873
1874 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1875 my $vgaid = "vga" . ($id // '');
1876 my $pciaddr;
1877 if ($q35 && $vgaid eq 'vga') {
1878 # the first display uses pcie.0 bus on q35 machines
1879 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1880 } else {
1881 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1882 }
1883
1884 if ($vga->{type} eq 'virtio-gl') {
1885 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1886 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1887 if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1888
1889 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1890 if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1891 }
1892
1893 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1894 }
1895
1896 sub parse_number_sets {
1897 my ($set) = @_;
1898 my $res = [];
1899 foreach my $part (split(/;/, $set)) {
1900 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1901 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1902 push @$res, [ $1, $2 ];
1903 } else {
1904 die "invalid range: $part\n";
1905 }
1906 }
1907 return $res;
1908 }
1909
1910 sub parse_numa {
1911 my ($data) = @_;
1912
1913 my $res = parse_property_string($numa_fmt, $data);
1914 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1915 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1916 return $res;
1917 }
1918
1919 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1920 sub parse_net {
1921 my ($data) = @_;
1922
1923 my $res = eval { parse_property_string($net_fmt, $data) };
1924 if ($@) {
1925 warn $@;
1926 return;
1927 }
1928 if (!defined($res->{macaddr})) {
1929 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1930 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1931 }
1932 return $res;
1933 }
1934
1935 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1936 sub parse_ipconfig {
1937 my ($data) = @_;
1938
1939 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1940 if ($@) {
1941 warn $@;
1942 return;
1943 }
1944
1945 if ($res->{gw} && !$res->{ip}) {
1946 warn 'gateway specified without specifying an IP address';
1947 return;
1948 }
1949 if ($res->{gw6} && !$res->{ip6}) {
1950 warn 'IPv6 gateway specified without specifying an IPv6 address';
1951 return;
1952 }
1953 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1954 warn 'gateway specified together with DHCP';
1955 return;
1956 }
1957 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1958 # gw6 + auto/dhcp
1959 warn "IPv6 gateway specified together with $res->{ip6} address";
1960 return;
1961 }
1962
1963 if (!$res->{ip} && !$res->{ip6}) {
1964 return { ip => 'dhcp', ip6 => 'dhcp' };
1965 }
1966
1967 return $res;
1968 }
1969
1970 sub print_net {
1971 my $net = shift;
1972
1973 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1974 }
1975
1976 sub add_random_macs {
1977 my ($settings) = @_;
1978
1979 foreach my $opt (keys %$settings) {
1980 next if $opt !~ m/^net(\d+)$/;
1981 my $net = parse_net($settings->{$opt});
1982 next if !$net;
1983 $settings->{$opt} = print_net($net);
1984 }
1985 }
1986
1987 sub vm_is_volid_owner {
1988 my ($storecfg, $vmid, $volid) = @_;
1989
1990 if ($volid !~ m|^/|) {
1991 my ($path, $owner);
1992 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1993 if ($owner && ($owner == $vmid)) {
1994 return 1;
1995 }
1996 }
1997
1998 return;
1999 }
2000
2001 sub vmconfig_register_unused_drive {
2002 my ($storecfg, $vmid, $conf, $drive) = @_;
2003
2004 if (drive_is_cloudinit($drive)) {
2005 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2006 warn $@ if $@;
2007 } elsif (!drive_is_cdrom($drive)) {
2008 my $volid = $drive->{file};
2009 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2010 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2011 }
2012 }
2013 }
2014
2015 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2016 my $smbios1_fmt = {
2017 uuid => {
2018 type => 'string',
2019 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2020 format_description => 'UUID',
2021 description => "Set SMBIOS1 UUID.",
2022 optional => 1,
2023 },
2024 version => {
2025 type => 'string',
2026 pattern => '[A-Za-z0-9+\/]+={0,2}',
2027 format_description => 'Base64 encoded string',
2028 description => "Set SMBIOS1 version.",
2029 optional => 1,
2030 },
2031 serial => {
2032 type => 'string',
2033 pattern => '[A-Za-z0-9+\/]+={0,2}',
2034 format_description => 'Base64 encoded string',
2035 description => "Set SMBIOS1 serial number.",
2036 optional => 1,
2037 },
2038 manufacturer => {
2039 type => 'string',
2040 pattern => '[A-Za-z0-9+\/]+={0,2}',
2041 format_description => 'Base64 encoded string',
2042 description => "Set SMBIOS1 manufacturer.",
2043 optional => 1,
2044 },
2045 product => {
2046 type => 'string',
2047 pattern => '[A-Za-z0-9+\/]+={0,2}',
2048 format_description => 'Base64 encoded string',
2049 description => "Set SMBIOS1 product ID.",
2050 optional => 1,
2051 },
2052 sku => {
2053 type => 'string',
2054 pattern => '[A-Za-z0-9+\/]+={0,2}',
2055 format_description => 'Base64 encoded string',
2056 description => "Set SMBIOS1 SKU string.",
2057 optional => 1,
2058 },
2059 family => {
2060 type => 'string',
2061 pattern => '[A-Za-z0-9+\/]+={0,2}',
2062 format_description => 'Base64 encoded string',
2063 description => "Set SMBIOS1 family string.",
2064 optional => 1,
2065 },
2066 base64 => {
2067 type => 'boolean',
2068 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2069 optional => 1,
2070 },
2071 };
2072
2073 sub parse_smbios1 {
2074 my ($data) = @_;
2075
2076 my $res = eval { parse_property_string($smbios1_fmt, $data) };
2077 warn $@ if $@;
2078 return $res;
2079 }
2080
2081 sub print_smbios1 {
2082 my ($smbios1) = @_;
2083 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2084 }
2085
2086 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2087
2088 sub parse_watchdog {
2089 my ($value) = @_;
2090
2091 return if !$value;
2092
2093 my $res = eval { parse_property_string($watchdog_fmt, $value) };
2094 warn $@ if $@;
2095 return $res;
2096 }
2097
2098 sub parse_guest_agent {
2099 my ($conf) = @_;
2100
2101 return {} if !defined($conf->{agent});
2102
2103 my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2104 warn $@ if $@;
2105
2106 # if the agent is disabled ignore the other potentially set properties
2107 return {} if !$res->{enabled};
2108 return $res;
2109 }
2110
2111 sub get_qga_key {
2112 my ($conf, $key) = @_;
2113 return undef if !defined($conf->{agent});
2114
2115 my $agent = parse_guest_agent($conf);
2116 return $agent->{$key};
2117 }
2118
2119 sub parse_vga {
2120 my ($value) = @_;
2121
2122 return {} if !$value;
2123 my $res = eval { parse_property_string($vga_fmt, $value) };
2124 warn $@ if $@;
2125 return $res;
2126 }
2127
2128 sub parse_rng {
2129 my ($value) = @_;
2130
2131 return if !$value;
2132
2133 my $res = eval { parse_property_string($rng_fmt, $value) };
2134 warn $@ if $@;
2135 return $res;
2136 }
2137
2138 sub parse_meta_info {
2139 my ($value) = @_;
2140
2141 return if !$value;
2142
2143 my $res = eval { parse_property_string($meta_info_fmt, $value) };
2144 warn $@ if $@;
2145 return $res;
2146 }
2147
2148 sub new_meta_info_string {
2149 my () = @_; # for now do not allow to override any value
2150
2151 return PVE::JSONSchema::print_property_string(
2152 {
2153 'creation-qemu' => kvm_user_version(),
2154 ctime => "". int(time()),
2155 },
2156 $meta_info_fmt
2157 );
2158 }
2159
2160 sub qemu_created_version_fixups {
2161 my ($conf, $forcemachine, $kvmver) = @_;
2162
2163 my $meta = parse_meta_info($conf->{meta}) // {};
2164 my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2165
2166 # check if we need to apply some handling for VMs that always use the latest machine version but
2167 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2168 # would be required (we do not want to pin machine version for non-windows OS type)
2169 if (
2170 (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2171 && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2172 && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2173 && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2174 ) {
2175 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2176 if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2177 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2178 # and thus with the predictable interface naming of systemd
2179 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2180 }
2181 }
2182 return;
2183 }
2184
2185 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2186 sub verify_usb_device {
2187 my ($value, $noerr) = @_;
2188
2189 return $value if parse_usb_device($value);
2190
2191 return if $noerr;
2192
2193 die "unable to parse usb device\n";
2194 }
2195
2196 # add JSON properties for create and set function
2197 sub json_config_properties {
2198 my ($prop, $with_disk_alloc) = @_;
2199
2200 my $skip_json_config_opts = {
2201 parent => 1,
2202 snaptime => 1,
2203 vmstate => 1,
2204 runningmachine => 1,
2205 runningcpu => 1,
2206 meta => 1,
2207 };
2208
2209 foreach my $opt (keys %$confdesc) {
2210 next if $skip_json_config_opts->{$opt};
2211
2212 if ($with_disk_alloc && is_valid_drivename($opt)) {
2213 $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2214 } else {
2215 $prop->{$opt} = $confdesc->{$opt};
2216 }
2217 }
2218
2219 return $prop;
2220 }
2221
2222 # Properties that we can read from an OVF file
2223 sub json_ovf_properties {
2224 my $prop = {};
2225
2226 for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2227 $prop->{$device} = {
2228 type => 'string',
2229 format => 'pve-volume-id-or-absolute-path',
2230 description => "Disk image that gets imported to $device",
2231 optional => 1,
2232 };
2233 }
2234
2235 $prop->{cores} = {
2236 type => 'integer',
2237 description => "The number of CPU cores.",
2238 optional => 1,
2239 };
2240 $prop->{memory} = {
2241 type => 'integer',
2242 description => "Amount of RAM for the VM in MB.",
2243 optional => 1,
2244 };
2245 $prop->{name} = {
2246 type => 'string',
2247 description => "Name of the VM.",
2248 optional => 1,
2249 };
2250
2251 return $prop;
2252 }
2253
2254 # return copy of $confdesc_cloudinit to generate documentation
2255 sub cloudinit_config_properties {
2256
2257 return dclone($confdesc_cloudinit);
2258 }
2259
2260 sub check_type {
2261 my ($key, $value) = @_;
2262
2263 die "unknown setting '$key'\n" if !$confdesc->{$key};
2264
2265 my $type = $confdesc->{$key}->{type};
2266
2267 if (!defined($value)) {
2268 die "got undefined value\n";
2269 }
2270
2271 if ($value =~ m/[\n\r]/) {
2272 die "property contains a line feed\n";
2273 }
2274
2275 if ($type eq 'boolean') {
2276 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2277 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2278 die "type check ('boolean') failed - got '$value'\n";
2279 } elsif ($type eq 'integer') {
2280 return int($1) if $value =~ m/^(\d+)$/;
2281 die "type check ('integer') failed - got '$value'\n";
2282 } elsif ($type eq 'number') {
2283 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2284 die "type check ('number') failed - got '$value'\n";
2285 } elsif ($type eq 'string') {
2286 if (my $fmt = $confdesc->{$key}->{format}) {
2287 PVE::JSONSchema::check_format($fmt, $value);
2288 return $value;
2289 }
2290 $value =~ s/^\"(.*)\"$/$1/;
2291 return $value;
2292 } else {
2293 die "internal error"
2294 }
2295 }
2296
2297 sub destroy_vm {
2298 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2299
2300 my $conf = PVE::QemuConfig->load_config($vmid);
2301
2302 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2303
2304 if ($conf->{template}) {
2305 # check if any base image is still used by a linked clone
2306 PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2307 my ($ds, $drive) = @_;
2308 return if drive_is_cdrom($drive);
2309
2310 my $volid = $drive->{file};
2311 return if !$volid || $volid =~ m|^/|;
2312
2313 die "base volume '$volid' is still in use by linked cloned\n"
2314 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2315
2316 });
2317 }
2318
2319 my $volids = {};
2320 my $remove_owned_drive = sub {
2321 my ($ds, $drive) = @_;
2322 return if drive_is_cdrom($drive, 1);
2323
2324 my $volid = $drive->{file};
2325 return if !$volid || $volid =~ m|^/|;
2326 return if $volids->{$volid};
2327
2328 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2329 return if !$path || !$owner || ($owner != $vmid);
2330
2331 $volids->{$volid} = 1;
2332 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2333 warn "Could not remove disk '$volid', check manually: $@" if $@;
2334 };
2335
2336 # only remove disks owned by this VM (referenced in the config)
2337 my $include_opts = {
2338 include_unused => 1,
2339 extra_keys => ['vmstate'],
2340 };
2341 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2342
2343 for my $snap (values %{$conf->{snapshots}}) {
2344 next if !defined($snap->{vmstate});
2345 my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2346 next if !defined($drive);
2347 $remove_owned_drive->('vmstate', $drive);
2348 }
2349
2350 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2351
2352 if ($purge_unreferenced) { # also remove unreferenced disk
2353 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2354 PVE::Storage::foreach_volid($vmdisks, sub {
2355 my ($volid, $sid, $volname, $d) = @_;
2356 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2357 warn $@ if $@;
2358 });
2359 }
2360
2361 if (defined $replacement_conf) {
2362 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2363 } else {
2364 PVE::QemuConfig->destroy_config($vmid);
2365 }
2366 }
2367
2368 sub parse_vm_config {
2369 my ($filename, $raw, $strict) = @_;
2370
2371 return if !defined($raw);
2372
2373 my $res = {
2374 digest => Digest::SHA::sha1_hex($raw),
2375 snapshots => {},
2376 pending => {},
2377 };
2378
2379 my $handle_error = sub {
2380 my ($msg) = @_;
2381
2382 if ($strict) {
2383 die $msg;
2384 } else {
2385 warn $msg;
2386 }
2387 };
2388
2389 $filename =~ m|/qemu-server/(\d+)\.conf$|
2390 || die "got strange filename '$filename'";
2391
2392 my $vmid = $1;
2393
2394 my $conf = $res;
2395 my $descr;
2396 my $section = '';
2397
2398 my @lines = split(/\n/, $raw);
2399 foreach my $line (@lines) {
2400 next if $line =~ m/^\s*$/;
2401
2402 if ($line =~ m/^\[PENDING\]\s*$/i) {
2403 $section = 'pending';
2404 if (defined($descr)) {
2405 $descr =~ s/\s+$//;
2406 $conf->{description} = $descr;
2407 }
2408 $descr = undef;
2409 $conf = $res->{$section} = {};
2410 next;
2411
2412 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2413 $section = $1;
2414 if (defined($descr)) {
2415 $descr =~ s/\s+$//;
2416 $conf->{description} = $descr;
2417 }
2418 $descr = undef;
2419 $conf = $res->{snapshots}->{$section} = {};
2420 next;
2421 }
2422
2423 if ($line =~ m/^\#(.*)$/) {
2424 $descr = '' if !defined($descr);
2425 $descr .= PVE::Tools::decode_text($1) . "\n";
2426 next;
2427 }
2428
2429 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2430 $descr = '' if !defined($descr);
2431 $descr .= PVE::Tools::decode_text($2);
2432 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2433 $conf->{snapstate} = $1;
2434 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2435 my $key = $1;
2436 my $value = $2;
2437 $conf->{$key} = $value;
2438 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2439 my $value = $1;
2440 if ($section eq 'pending') {
2441 $conf->{delete} = $value; # we parse this later
2442 } else {
2443 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2444 }
2445 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2446 my $key = $1;
2447 my $value = $2;
2448 eval { $value = check_type($key, $value); };
2449 if ($@) {
2450 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2451 } else {
2452 $key = 'ide2' if $key eq 'cdrom';
2453 my $fmt = $confdesc->{$key}->{format};
2454 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2455 my $v = parse_drive($key, $value);
2456 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2457 $v->{file} = $volid;
2458 $value = print_drive($v);
2459 } else {
2460 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2461 next;
2462 }
2463 }
2464
2465 $conf->{$key} = $value;
2466 }
2467 } else {
2468 $handle_error->("vm $vmid - unable to parse config: $line\n");
2469 }
2470 }
2471
2472 if (defined($descr)) {
2473 $descr =~ s/\s+$//;
2474 $conf->{description} = $descr;
2475 }
2476 delete $res->{snapstate}; # just to be sure
2477
2478 return $res;
2479 }
2480
2481 sub write_vm_config {
2482 my ($filename, $conf) = @_;
2483
2484 delete $conf->{snapstate}; # just to be sure
2485
2486 if ($conf->{cdrom}) {
2487 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2488 $conf->{ide2} = $conf->{cdrom};
2489 delete $conf->{cdrom};
2490 }
2491
2492 # we do not use 'smp' any longer
2493 if ($conf->{sockets}) {
2494 delete $conf->{smp};
2495 } elsif ($conf->{smp}) {
2496 $conf->{sockets} = $conf->{smp};
2497 delete $conf->{cores};
2498 delete $conf->{smp};
2499 }
2500
2501 my $used_volids = {};
2502
2503 my $cleanup_config = sub {
2504 my ($cref, $pending, $snapname) = @_;
2505
2506 foreach my $key (keys %$cref) {
2507 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2508 $key eq 'snapstate' || $key eq 'pending';
2509 my $value = $cref->{$key};
2510 if ($key eq 'delete') {
2511 die "propertry 'delete' is only allowed in [PENDING]\n"
2512 if !$pending;
2513 # fixme: check syntax?
2514 next;
2515 }
2516 eval { $value = check_type($key, $value); };
2517 die "unable to parse value of '$key' - $@" if $@;
2518
2519 $cref->{$key} = $value;
2520
2521 if (!$snapname && is_valid_drivename($key)) {
2522 my $drive = parse_drive($key, $value);
2523 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2524 }
2525 }
2526 };
2527
2528 &$cleanup_config($conf);
2529
2530 &$cleanup_config($conf->{pending}, 1);
2531
2532 foreach my $snapname (keys %{$conf->{snapshots}}) {
2533 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2534 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2535 }
2536
2537 # remove 'unusedX' settings if we re-add a volume
2538 foreach my $key (keys %$conf) {
2539 my $value = $conf->{$key};
2540 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2541 delete $conf->{$key};
2542 }
2543 }
2544
2545 my $generate_raw_config = sub {
2546 my ($conf, $pending) = @_;
2547
2548 my $raw = '';
2549
2550 # add description as comment to top of file
2551 if (defined(my $descr = $conf->{description})) {
2552 if ($descr) {
2553 foreach my $cl (split(/\n/, $descr)) {
2554 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2555 }
2556 } else {
2557 $raw .= "#\n" if $pending;
2558 }
2559 }
2560
2561 foreach my $key (sort keys %$conf) {
2562 next if $key =~ /^(digest|description|pending|snapshots)$/;
2563 $raw .= "$key: $conf->{$key}\n";
2564 }
2565 return $raw;
2566 };
2567
2568 my $raw = &$generate_raw_config($conf);
2569
2570 if (scalar(keys %{$conf->{pending}})){
2571 $raw .= "\n[PENDING]\n";
2572 $raw .= &$generate_raw_config($conf->{pending}, 1);
2573 }
2574
2575 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2576 $raw .= "\n[$snapname]\n";
2577 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2578 }
2579
2580 return $raw;
2581 }
2582
2583 sub load_defaults {
2584
2585 my $res = {};
2586
2587 # we use static defaults from our JSON schema configuration
2588 foreach my $key (keys %$confdesc) {
2589 if (defined(my $default = $confdesc->{$key}->{default})) {
2590 $res->{$key} = $default;
2591 }
2592 }
2593
2594 return $res;
2595 }
2596
2597 sub config_list {
2598 my $vmlist = PVE::Cluster::get_vmlist();
2599 my $res = {};
2600 return $res if !$vmlist || !$vmlist->{ids};
2601 my $ids = $vmlist->{ids};
2602 my $nodename = nodename();
2603
2604 foreach my $vmid (keys %$ids) {
2605 my $d = $ids->{$vmid};
2606 next if !$d->{node} || $d->{node} ne $nodename;
2607 next if !$d->{type} || $d->{type} ne 'qemu';
2608 $res->{$vmid}->{exists} = 1;
2609 }
2610 return $res;
2611 }
2612
2613 # test if VM uses local resources (to prevent migration)
2614 sub check_local_resources {
2615 my ($conf, $noerr) = @_;
2616
2617 my @loc_res = ();
2618
2619 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2620 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2621
2622 push @loc_res, "ivshmem" if $conf->{ivshmem};
2623
2624 foreach my $k (keys %$conf) {
2625 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2626 # sockets are safe: they will recreated be on the target side post-migrate
2627 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2628 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2629 }
2630
2631 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2632
2633 return \@loc_res;
2634 }
2635
2636 # check if used storages are available on all nodes (use by migrate)
2637 sub check_storage_availability {
2638 my ($storecfg, $conf, $node) = @_;
2639
2640 PVE::QemuConfig->foreach_volume($conf, sub {
2641 my ($ds, $drive) = @_;
2642
2643 my $volid = $drive->{file};
2644 return if !$volid;
2645
2646 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2647 return if !$sid;
2648
2649 # check if storage is available on both nodes
2650 my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2651 PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2652
2653 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2654
2655 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2656 if !$scfg->{content}->{$vtype};
2657 });
2658 }
2659
2660 # list nodes where all VM images are available (used by has_feature API)
2661 sub shared_nodes {
2662 my ($conf, $storecfg) = @_;
2663
2664 my $nodelist = PVE::Cluster::get_nodelist();
2665 my $nodehash = { map { $_ => 1 } @$nodelist };
2666 my $nodename = nodename();
2667
2668 PVE::QemuConfig->foreach_volume($conf, sub {
2669 my ($ds, $drive) = @_;
2670
2671 my $volid = $drive->{file};
2672 return if !$volid;
2673
2674 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2675 if ($storeid) {
2676 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2677 if ($scfg->{disable}) {
2678 $nodehash = {};
2679 } elsif (my $avail = $scfg->{nodes}) {
2680 foreach my $node (keys %$nodehash) {
2681 delete $nodehash->{$node} if !$avail->{$node};
2682 }
2683 } elsif (!$scfg->{shared}) {
2684 foreach my $node (keys %$nodehash) {
2685 delete $nodehash->{$node} if $node ne $nodename
2686 }
2687 }
2688 }
2689 });
2690
2691 return $nodehash
2692 }
2693
2694 sub check_local_storage_availability {
2695 my ($conf, $storecfg) = @_;
2696
2697 my $nodelist = PVE::Cluster::get_nodelist();
2698 my $nodehash = { map { $_ => {} } @$nodelist };
2699
2700 PVE::QemuConfig->foreach_volume($conf, sub {
2701 my ($ds, $drive) = @_;
2702
2703 my $volid = $drive->{file};
2704 return if !$volid;
2705
2706 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2707 if ($storeid) {
2708 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2709
2710 if ($scfg->{disable}) {
2711 foreach my $node (keys %$nodehash) {
2712 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2713 }
2714 } elsif (my $avail = $scfg->{nodes}) {
2715 foreach my $node (keys %$nodehash) {
2716 if (!$avail->{$node}) {
2717 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2718 }
2719 }
2720 }
2721 }
2722 });
2723
2724 foreach my $node (values %$nodehash) {
2725 if (my $unavail = $node->{unavailable_storages}) {
2726 $node->{unavailable_storages} = [ sort keys %$unavail ];
2727 }
2728 }
2729
2730 return $nodehash
2731 }
2732
2733 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2734 sub check_running {
2735 my ($vmid, $nocheck, $node) = @_;
2736
2737 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2738 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2739 }
2740
2741 sub vzlist {
2742
2743 my $vzlist = config_list();
2744
2745 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2746
2747 while (defined(my $de = $fd->read)) {
2748 next if $de !~ m/^(\d+)\.pid$/;
2749 my $vmid = $1;
2750 next if !defined($vzlist->{$vmid});
2751 if (my $pid = check_running($vmid)) {
2752 $vzlist->{$vmid}->{pid} = $pid;
2753 }
2754 }
2755
2756 return $vzlist;
2757 }
2758
2759 our $vmstatus_return_properties = {
2760 vmid => get_standard_option('pve-vmid'),
2761 status => {
2762 description => "Qemu process status.",
2763 type => 'string',
2764 enum => ['stopped', 'running'],
2765 },
2766 maxmem => {
2767 description => "Maximum memory in bytes.",
2768 type => 'integer',
2769 optional => 1,
2770 renderer => 'bytes',
2771 },
2772 maxdisk => {
2773 description => "Root disk size in bytes.",
2774 type => 'integer',
2775 optional => 1,
2776 renderer => 'bytes',
2777 },
2778 name => {
2779 description => "VM name.",
2780 type => 'string',
2781 optional => 1,
2782 },
2783 qmpstatus => {
2784 description => "Qemu QMP agent status.",
2785 type => 'string',
2786 optional => 1,
2787 },
2788 pid => {
2789 description => "PID of running qemu process.",
2790 type => 'integer',
2791 optional => 1,
2792 },
2793 uptime => {
2794 description => "Uptime.",
2795 type => 'integer',
2796 optional => 1,
2797 renderer => 'duration',
2798 },
2799 cpus => {
2800 description => "Maximum usable CPUs.",
2801 type => 'number',
2802 optional => 1,
2803 },
2804 lock => {
2805 description => "The current config lock, if any.",
2806 type => 'string',
2807 optional => 1,
2808 },
2809 tags => {
2810 description => "The current configured tags, if any",
2811 type => 'string',
2812 optional => 1,
2813 },
2814 'running-machine' => {
2815 description => "The currently running machine type (if running).",
2816 type => 'string',
2817 optional => 1,
2818 },
2819 'running-qemu' => {
2820 description => "The currently running QEMU version (if running).",
2821 type => 'string',
2822 optional => 1,
2823 },
2824 };
2825
2826 my $last_proc_pid_stat;
2827
2828 # get VM status information
2829 # This must be fast and should not block ($full == false)
2830 # We only query KVM using QMP if $full == true (this can be slow)
2831 sub vmstatus {
2832 my ($opt_vmid, $full) = @_;
2833
2834 my $res = {};
2835
2836 my $storecfg = PVE::Storage::config();
2837
2838 my $list = vzlist();
2839 my $defaults = load_defaults();
2840
2841 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2842
2843 my $cpucount = $cpuinfo->{cpus} || 1;
2844
2845 foreach my $vmid (keys %$list) {
2846 next if $opt_vmid && ($vmid ne $opt_vmid);
2847
2848 my $conf = PVE::QemuConfig->load_config($vmid);
2849
2850 my $d = { vmid => int($vmid) };
2851 $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2852
2853 # fixme: better status?
2854 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2855
2856 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2857 if (defined($size)) {
2858 $d->{disk} = 0; # no info available
2859 $d->{maxdisk} = $size;
2860 } else {
2861 $d->{disk} = 0;
2862 $d->{maxdisk} = 0;
2863 }
2864
2865 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2866 * ($conf->{cores} || $defaults->{cores});
2867 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2868 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2869
2870 $d->{name} = $conf->{name} || "VM $vmid";
2871 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2872 : $defaults->{memory}*(1024*1024);
2873
2874 if ($conf->{balloon}) {
2875 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2876 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2877 : $defaults->{shares};
2878 }
2879
2880 $d->{uptime} = 0;
2881 $d->{cpu} = 0;
2882 $d->{mem} = 0;
2883
2884 $d->{netout} = 0;
2885 $d->{netin} = 0;
2886
2887 $d->{diskread} = 0;
2888 $d->{diskwrite} = 0;
2889
2890 $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2891
2892 $d->{serial} = 1 if conf_has_serial($conf);
2893 $d->{lock} = $conf->{lock} if $conf->{lock};
2894 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2895
2896 $res->{$vmid} = $d;
2897 }
2898
2899 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2900 foreach my $dev (keys %$netdev) {
2901 next if $dev !~ m/^tap([1-9]\d*)i/;
2902 my $vmid = $1;
2903 my $d = $res->{$vmid};
2904 next if !$d;
2905
2906 $d->{netout} += $netdev->{$dev}->{receive};
2907 $d->{netin} += $netdev->{$dev}->{transmit};
2908
2909 if ($full) {
2910 $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2911 $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2912 }
2913
2914 }
2915
2916 my $ctime = gettimeofday;
2917
2918 foreach my $vmid (keys %$list) {
2919
2920 my $d = $res->{$vmid};
2921 my $pid = $d->{pid};
2922 next if !$pid;
2923
2924 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2925 next if !$pstat; # not running
2926
2927 my $used = $pstat->{utime} + $pstat->{stime};
2928
2929 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2930
2931 if ($pstat->{vsize}) {
2932 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2933 }
2934
2935 my $old = $last_proc_pid_stat->{$pid};
2936 if (!$old) {
2937 $last_proc_pid_stat->{$pid} = {
2938 time => $ctime,
2939 used => $used,
2940 cpu => 0,
2941 };
2942 next;
2943 }
2944
2945 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2946
2947 if ($dtime > 1000) {
2948 my $dutime = $used - $old->{used};
2949
2950 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2951 $last_proc_pid_stat->{$pid} = {
2952 time => $ctime,
2953 used => $used,
2954 cpu => $d->{cpu},
2955 };
2956 } else {
2957 $d->{cpu} = $old->{cpu};
2958 }
2959 }
2960
2961 return $res if !$full;
2962
2963 my $qmpclient = PVE::QMPClient->new();
2964
2965 my $ballooncb = sub {
2966 my ($vmid, $resp) = @_;
2967
2968 my $info = $resp->{'return'};
2969 return if !$info->{max_mem};
2970
2971 my $d = $res->{$vmid};
2972
2973 # use memory assigned to VM
2974 $d->{maxmem} = $info->{max_mem};
2975 $d->{balloon} = $info->{actual};
2976
2977 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2978 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2979 $d->{freemem} = $info->{free_mem};
2980 }
2981
2982 $d->{ballooninfo} = $info;
2983 };
2984
2985 my $blockstatscb = sub {
2986 my ($vmid, $resp) = @_;
2987 my $data = $resp->{'return'} || [];
2988 my $totalrdbytes = 0;
2989 my $totalwrbytes = 0;
2990
2991 for my $blockstat (@$data) {
2992 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2993 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2994
2995 $blockstat->{device} =~ s/drive-//;
2996 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2997 }
2998 $res->{$vmid}->{diskread} = $totalrdbytes;
2999 $res->{$vmid}->{diskwrite} = $totalwrbytes;
3000 };
3001
3002 my $machinecb = sub {
3003 my ($vmid, $resp) = @_;
3004 my $data = $resp->{'return'} || [];
3005
3006 $res->{$vmid}->{'running-machine'} =
3007 PVE::QemuServer::Machine::current_from_query_machines($data);
3008 };
3009
3010 my $versioncb = sub {
3011 my ($vmid, $resp) = @_;
3012 my $data = $resp->{'return'} // {};
3013 my $version = 'unknown';
3014
3015 if (my $v = $data->{qemu}) {
3016 $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3017 }
3018
3019 $res->{$vmid}->{'running-qemu'} = $version;
3020 };
3021
3022 my $statuscb = sub {
3023 my ($vmid, $resp) = @_;
3024
3025 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3026 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3027 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3028 # this fails if ballon driver is not loaded, so this must be
3029 # the last commnand (following command are aborted if this fails).
3030 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3031
3032 my $status = 'unknown';
3033 if (!defined($status = $resp->{'return'}->{status})) {
3034 warn "unable to get VM status\n";
3035 return;
3036 }
3037
3038 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3039 };
3040
3041 foreach my $vmid (keys %$list) {
3042 next if $opt_vmid && ($vmid ne $opt_vmid);
3043 next if !$res->{$vmid}->{pid}; # not running
3044 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3045 }
3046
3047 $qmpclient->queue_execute(undef, 2);
3048
3049 foreach my $vmid (keys %$list) {
3050 next if $opt_vmid && ($vmid ne $opt_vmid);
3051 next if !$res->{$vmid}->{pid}; #not running
3052
3053 # we can't use the $qmpclient since it might have already aborted on
3054 # 'query-balloon', but this might also fail for older versions...
3055 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3056 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3057 }
3058
3059 foreach my $vmid (keys %$list) {
3060 next if $opt_vmid && ($vmid ne $opt_vmid);
3061 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3062 }
3063
3064 return $res;
3065 }
3066
3067 sub conf_has_serial {
3068 my ($conf) = @_;
3069
3070 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3071 if ($conf->{"serial$i"}) {
3072 return 1;
3073 }
3074 }
3075
3076 return 0;
3077 }
3078
3079 sub conf_has_audio {
3080 my ($conf, $id) = @_;
3081
3082 $id //= 0;
3083 my $audio = $conf->{"audio$id"};
3084 return if !defined($audio);
3085
3086 my $audioproperties = parse_property_string($audio_fmt, $audio);
3087 my $audiodriver = $audioproperties->{driver} // 'spice';
3088
3089 return {
3090 dev => $audioproperties->{device},
3091 dev_id => "audiodev$id",
3092 backend => $audiodriver,
3093 backend_id => "$audiodriver-backend${id}",
3094 };
3095 }
3096
3097 sub audio_devs {
3098 my ($audio, $audiopciaddr, $machine_version) = @_;
3099
3100 my $devs = [];
3101
3102 my $id = $audio->{dev_id};
3103 my $audiodev = "";
3104 if (min_version($machine_version, 4, 2)) {
3105 $audiodev = ",audiodev=$audio->{backend_id}";
3106 }
3107
3108 if ($audio->{dev} eq 'AC97') {
3109 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3110 } elsif ($audio->{dev} =~ /intel\-hda$/) {
3111 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3112 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3113 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3114 } else {
3115 die "unkown audio device '$audio->{dev}', implement me!";
3116 }
3117
3118 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3119
3120 return $devs;
3121 }
3122
3123 sub get_tpm_paths {
3124 my ($vmid) = @_;
3125 return {
3126 socket => "/var/run/qemu-server/$vmid.swtpm",
3127 pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3128 };
3129 }
3130
3131 sub add_tpm_device {
3132 my ($vmid, $devices, $conf) = @_;
3133
3134 return if !$conf->{tpmstate0};
3135
3136 my $paths = get_tpm_paths($vmid);
3137
3138 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3139 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3140 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3141 }
3142
3143 sub start_swtpm {
3144 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3145
3146 return if !$tpmdrive;
3147
3148 my $state;
3149 my $tpm = parse_drive("tpmstate0", $tpmdrive);
3150 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3151 if ($storeid) {
3152 $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3153 } else {
3154 $state = $tpm->{file};
3155 }
3156
3157 my $paths = get_tpm_paths($vmid);
3158
3159 # during migration, we will get state from remote
3160 #
3161 if (!$migration) {
3162 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3163 my $setup_cmd = [
3164 "swtpm_setup",
3165 "--tpmstate",
3166 "file://$state",
3167 "--createek",
3168 "--create-ek-cert",
3169 "--create-platform-cert",
3170 "--lock-nvram",
3171 "--config",
3172 "/etc/swtpm_setup.conf", # do not use XDG configs
3173 "--runas",
3174 "0", # force creation as root, error if not possible
3175 "--not-overwrite", # ignore existing state, do not modify
3176 ];
3177
3178 push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3179 # TPM 2.0 supports ECC crypto, use if possible
3180 push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3181
3182 run_command($setup_cmd, outfunc => sub {
3183 print "swtpm_setup: $1\n";
3184 });
3185 }
3186
3187 my $emulator_cmd = [
3188 "swtpm",
3189 "socket",
3190 "--tpmstate",
3191 "backend-uri=file://$state,mode=0600",
3192 "--ctrl",
3193 "type=unixio,path=$paths->{socket},mode=0600",
3194 "--pid",
3195 "file=$paths->{pid}",
3196 "--terminate", # terminate on QEMU disconnect
3197 "--daemon",
3198 ];
3199 push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3200 run_command($emulator_cmd, outfunc => sub { print $1; });
3201
3202 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3203 while (! -e $paths->{pid}) {
3204 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3205 usleep(50_000);
3206 }
3207
3208 # return untainted PID of swtpm daemon so it can be killed on error
3209 file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3210 return $1;
3211 }
3212
3213 sub vga_conf_has_spice {
3214 my ($vga) = @_;
3215
3216 my $vgaconf = parse_vga($vga);
3217 my $vgatype = $vgaconf->{type};
3218 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3219
3220 return $1 || 1;
3221 }
3222
3223 sub is_native($) {
3224 my ($arch) = @_;
3225 return get_host_arch() eq $arch;
3226 }
3227
3228 sub get_vm_arch {
3229 my ($conf) = @_;
3230 return $conf->{arch} // get_host_arch();
3231 }
3232
3233 my $default_machines = {
3234 x86_64 => 'pc',
3235 aarch64 => 'virt',
3236 };
3237
3238 sub get_installed_machine_version {
3239 my ($kvmversion) = @_;
3240 $kvmversion = kvm_user_version() if !defined($kvmversion);
3241 $kvmversion =~ m/^(\d+\.\d+)/;
3242 return $1;
3243 }
3244
3245 sub windows_get_pinned_machine_version {
3246 my ($machine, $base_version, $kvmversion) = @_;
3247
3248 my $pin_version = $base_version;
3249 if (!defined($base_version) ||
3250 !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3251 ) {
3252 $pin_version = get_installed_machine_version($kvmversion);
3253 }
3254 if (!$machine || $machine eq 'pc') {
3255 $machine = "pc-i440fx-$pin_version";
3256 } elsif ($machine eq 'q35') {
3257 $machine = "pc-q35-$pin_version";
3258 } elsif ($machine eq 'virt') {
3259 $machine = "virt-$pin_version";
3260 } else {
3261 warn "unknown machine type '$machine', not touching that!\n";
3262 }
3263
3264 return $machine;
3265 }
3266
3267 sub get_vm_machine {
3268 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3269
3270 my $machine = $forcemachine || $conf->{machine};
3271
3272 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3273 $kvmversion //= kvm_user_version();
3274 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3275 # layout which confuses windows quite a bit and may result in various regressions..
3276 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3277 if (windows_version($conf->{ostype})) {
3278 $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3279 }
3280 $arch //= 'x86_64';
3281 $machine ||= $default_machines->{$arch};
3282 if ($add_pve_version) {
3283 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3284 $machine .= "+pve$pvever";
3285 }
3286 }
3287
3288 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3289 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3290 $machine = $1 if $is_pxe;
3291
3292 # for version-pinned machines that do not include a pve-version (e.g.
3293 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3294 $machine .= '+pve0';
3295
3296 $machine .= '.pxe' if $is_pxe;
3297 }
3298
3299 return $machine;
3300 }
3301
3302 sub get_ovmf_files($$$) {
3303 my ($arch, $efidisk, $smm) = @_;
3304
3305 my $types = $OVMF->{$arch}
3306 or die "no OVMF images known for architecture '$arch'\n";
3307
3308 my $type = 'default';
3309 if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3310 $type = $smm ? "4m" : "4m-no-smm";
3311 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3312 }
3313
3314 return $types->{$type}->@*;
3315 }
3316
3317 my $Arch2Qemu = {
3318 aarch64 => '/usr/bin/qemu-system-aarch64',
3319 x86_64 => '/usr/bin/qemu-system-x86_64',
3320 };
3321 sub get_command_for_arch($) {
3322 my ($arch) = @_;
3323 return '/usr/bin/kvm&