PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     vcpus => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "Number of hotplugged vcpus.",
 448         minimum => 1,
 449         default => 0,
 450     },
 451     acpi => {
 452         optional => 1,
 453         type => 'boolean',
 454         description => "Enable/disable ACPI.",
 455         default => 1,
 456     },
 457     agent => {
 458         optional => 1,
 459         description => "Enable/disable Qemu GuestAgent and its properties.",
 460         type => 'string',
 461         format => $agent_fmt,
 462     },
 463     kvm => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable KVM hardware virtualization.",
 467         default => 1,
 468     },
 469     tdf => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable time drift fix.",
 473         default => 0,
 474     },
 475     localtime => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 479     },
 480     freeze => {
 481         optional => 1,
 482         type => 'boolean',
 483         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 484     },
 485     vga => {
 486         optional => 1,
 487         type => 'string', format => $vga_fmt,
 488         description => "Configure the VGA hardware.",
 489         verbose_description => "Configure the VGA Hardware. If you want to use ".
 490             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 491             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 492             "is 'std' for all OS types besides some Windows versions (XP and " .
 493             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 494             "display server. For win* OS you can select how many independent " .
 495             "displays you want, Linux guests can add displays them self.\n".
 496             "You can also run without any graphic card, using a serial device as terminal.",
 497     },
 498     watchdog => {
 499         optional => 1,
 500         type => 'string', format => 'pve-qm-watchdog',
 501         description => "Create a virtual hardware watchdog device.",
 502         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 503             " (by a guest action), the watchdog must be periodically polled " .
 504             "by an agent inside the guest or else the watchdog will reset " .
 505             "the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 512         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 513         default => 'now',
 514     },
 515     startup =>  get_standard_option('pve-startup-order'),
 516     template => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Enable/disable Template.",
 520         default => 0,
 521     },
 522     args => {
 523         optional => 1,
 524         type => 'string',
 525         description => "Arbitrary arguments passed to kvm.",
 526         verbose_description => <<EODESCR,
 527 Arbitrary arguments passed to kvm, for example:
 528
 529 args: -no-reboot -no-hpet
 530
 531 NOTE: this option is for experts only.
 532 EODESCR
 533     },
 534     tablet => {
 535         optional => 1,
 536         type => 'boolean',
 537         default => 1,
 538         description => "Enable/disable the USB tablet device.",
 539         verbose_description => "Enable/disable the USB tablet device. This device is " .
 540             "usually needed to allow absolute mouse positioning with VNC. " .
 541             "Else the mouse runs out of sync with normal VNC clients. " .
 542             "If you're running lots of console-only guests on one host, " .
 543             "you may consider disabling this to save some context switches. " .
 544             "This is turned off by default if you use spice (-vga=qxl).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 586     },
 587     vmstatestorage => get_standard_option('pve-storage-id', {
 588         description => "Default storage for VM state volumes/files.",
 589         optional => 1,
 590     }),
 591     runningmachine => get_standard_option('pve-qemu-machine', {
 592         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 593     }),
 594     runningcpu => {
 595         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 596         optional => 1,
 597         type => 'string',
 598         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 599         format_description => 'QEMU -cpu parameter'
 600     },
 601     machine => get_standard_option('pve-qemu-machine'),
 602     arch => {
 603         description => "Virtual processor architecture. Defaults to the host.",
 604         optional => 1,
 605         type => 'string',
 606         enum => [qw(x86_64 aarch64)],
 607     },
 608     smbios1 => {
 609         description => "Specify SMBIOS type 1 fields.",
 610         type => 'string', format => 'pve-qm-smbios1',
 611         maxLength => 512,
 612         optional => 1,
 613     },
 614     protection => {
 615         optional => 1,
 616         type => 'boolean',
 617         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 618         default => 0,
 619     },
 620     bios => {
 621         optional => 1,
 622         type => 'string',
 623         enum => [ qw(seabios ovmf) ],
 624         description => "Select BIOS implementation.",
 625         default => 'seabios',
 626     },
 627     vmgenid => {
 628         type => 'string',
 629         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 630         format_description => 'UUID',
 631         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 632         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 633             " 128-bit integer value identifier to the guest OS. This allows to".
 634             " notify the guest operating system when the virtual machine is".
 635             " executed with a different configuration (e.g. snapshot execution".
 636             " or creation from a template). The guest operating system notices".
 637             " the change, and is then able to react as appropriate by marking".
 638             " its copies of distributed databases as dirty, re-initializing its".
 639             " random number generator, etc.\n".
 640             "Note that auto-creation only works when done throug API/CLI create".
 641             " or update methods, but not when manually editing the config file.",
 642         default => "1 (autogenerated)",
 643         optional => 1,
 644     },
 645     hookscript => {
 646         type => 'string',
 647         format => 'pve-volume-id',
 648         optional => 1,
 649         description => "Script that will be executed during various steps in the vms lifetime.",
 650     },
 651     ivshmem => {
 652         type => 'string',
 653         format => $ivshmem_fmt,
 654         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 655         optional => 1,
 656     },
 657     audio0 => {
 658         type => 'string',
 659         format => $audio_fmt,
 660         description => "Configure a audio device, useful in combination with QXL/Spice.",
 661         optional => 1
 662     },
 663     spice_enhancements => {
 664         type => 'string',
 665         format => $spice_enhancements_fmt,
 666         description => "Configure additional enhancements for SPICE.",
 667         optional => 1
 668     },
 669     tags => {
 670         type => 'string', format => 'pve-tag-list',
 671         description => 'Tags of the VM. This is only meta information.',
 672         optional => 1,
 673     },
 674     rng0 => {
 675         type => 'string',
 676         format => $rng_fmt,
 677         description => "Configure a VirtIO-based Random Number Generator.",
 678         optional => 1,
 679     },
 680 };
 681
 682 my $cicustom_fmt = {
 683     meta => {
 684         type => 'string',
 685         optional => 1,
 686         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 687         format => 'pve-volume-id',
 688         format_description => 'volume',
 689     },
 690     network => {
 691         type => 'string',
 692         optional => 1,
 693         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 694         format => 'pve-volume-id',
 695         format_description => 'volume',
 696     },
 697     user => {
 698         type => 'string',
 699         optional => 1,
 700         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 701         format => 'pve-volume-id',
 702         format_description => 'volume',
 703     },
 704 };
 705 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 706
 707 my $confdesc_cloudinit = {
 708     citype => {
 709         optional => 1,
 710         type => 'string',
 711         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 712         enum => ['configdrive2', 'nocloud'],
 713     },
 714     ciuser => {
 715         optional => 1,
 716         type => 'string',
 717         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 718     },
 719     cipassword => {
 720         optional => 1,
 721         type => 'string',
 722         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 723     },
 724     cicustom => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 728         format => 'pve-qm-cicustom',
 729     },
 730     searchdomain => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 734     },
 735     nameserver => {
 736         optional => 1,
 737         type => 'string', format => 'address-list',
 738         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 739     },
 740     sshkeys => {
 741         optional => 1,
 742         type => 'string',
 743         format => 'urlencoded',
 744         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 745     },
 746 };
 747
 748 # what about other qemu settings ?
 749 #cpu => 'string',
 750 #machine => 'string',
 751 #fda => 'file',
 752 #fdb => 'file',
 753 #mtdblock => 'file',
 754 #sd => 'file',
 755 #pflash => 'file',
 756 #snapshot => 'bool',
 757 #bootp => 'file',
 758 ##tftp => 'dir',
 759 ##smb => 'dir',
 760 #kernel => 'file',
 761 #append => 'string',
 762 #initrd => 'file',
 763 ##soundhw => 'string',
 764
 765 while (my ($k, $v) = each %$confdesc) {
 766     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 767 }
 768
 769 my $MAX_USB_DEVICES = 5;
 770 my $MAX_NETS = 32;
 771 my $MAX_HOSTPCI_DEVICES = 16;
 772 my $MAX_SERIAL_PORTS = 4;
 773 my $MAX_PARALLEL_PORTS = 3;
 774 my $MAX_NUMA = 8;
 775
 776 my $numa_fmt = {
 777     cpus => {
 778         type => "string",
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => "CPUs accessing this NUMA node.",
 781         format_description => "id[-id];...",
 782     },
 783     memory => {
 784         type => "number",
 785         description => "Amount of memory this NUMA node provides.",
 786         optional => 1,
 787     },
 788     hostnodes => {
 789         type => "string",
 790         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 791         description => "Host NUMA nodes to use.",
 792         format_description => "id[-id];...",
 793         optional => 1,
 794     },
 795     policy => {
 796         type => 'string',
 797         enum => [qw(preferred bind interleave)],
 798         description => "NUMA allocation policy.",
 799         optional => 1,
 800     },
 801 };
 802 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 803 my $numadesc = {
 804     optional => 1,
 805     type => 'string', format => $numa_fmt,
 806     description => "NUMA topology.",
 807 };
 808 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 809
 810 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 811     $confdesc->{"numa$i"} = $numadesc;
 812 }
 813
 814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 815                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 816                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 818
 819 my $net_fmt_bridge_descr = <<__EOD__;
 820 Bridge to attach the network device to. The Proxmox VE standard bridge
 821 is called 'vmbr0'.
 822
 823 If you do not specify a bridge, we create a kvm user (NATed) network
 824 device, which provides DHCP and DNS services. The following addresses
 825 are used:
 826
 827  10.0.2.2   Gateway
 828  10.0.2.3   DNS Server
 829  10.0.2.4   SMB Server
 830
 831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 832 __EOD__
 833
 834 my $net_fmt = {
 835     macaddr  => get_standard_option('mac-addr', {
 836         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 837     }),
 838     model => {
 839         type => 'string',
 840         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 841         enum => $nic_model_list,
 842         default_key => 1,
 843     },
 844     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 845     bridge => {
 846         type => 'string',
 847         description => $net_fmt_bridge_descr,
 848         format_description => 'bridge',
 849         pattern => '[-_.\w\d]+',
 850         optional => 1,
 851     },
 852     queues => {
 853         type => 'integer',
 854         minimum => 0, maximum => 16,
 855         description => 'Number of packet queues to be used on the device.',
 856         optional => 1,
 857     },
 858     rate => {
 859         type => 'number',
 860         minimum => 0,
 861         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 862         optional => 1,
 863     },
 864     tag => {
 865         type => 'integer',
 866         minimum => 1, maximum => 4094,
 867         description => 'VLAN tag to apply to packets on this interface.',
 868         optional => 1,
 869     },
 870     trunks => {
 871         type => 'string',
 872         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 873         description => 'VLAN trunks to pass through this interface.',
 874         format_description => 'vlanid[;vlanid...]',
 875         optional => 1,
 876     },
 877     firewall => {
 878         type => 'boolean',
 879         description => 'Whether this interface should be protected by the firewall.',
 880         optional => 1,
 881     },
 882     link_down => {
 883         type => 'boolean',
 884         description => 'Whether this interface should be disconnected (like pulling the plug).',
 885         optional => 1,
 886     },
 887     mtu => {
 888         type => 'integer',
 889         minimum => 1, maximum => 65520,
 890         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 891         optional => 1,
 892     },
 893 };
 894
 895 my $netdesc = {
 896     optional => 1,
 897     type => 'string', format => $net_fmt,
 898     description => "Specify network devices.",
 899 };
 900
 901 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 902
 903 my $ipconfig_fmt = {
 904     ip => {
 905         type => 'string',
 906         format => 'pve-ipv4-config',
 907         format_description => 'IPv4Format/CIDR',
 908         description => 'IPv4 address in CIDR format.',
 909         optional => 1,
 910         default => 'dhcp',
 911     },
 912     gw => {
 913         type => 'string',
 914         format => 'ipv4',
 915         format_description => 'GatewayIPv4',
 916         description => 'Default gateway for IPv4 traffic.',
 917         optional => 1,
 918         requires => 'ip',
 919     },
 920     ip6 => {
 921         type => 'string',
 922         format => 'pve-ipv6-config',
 923         format_description => 'IPv6Format/CIDR',
 924         description => 'IPv6 address in CIDR format.',
 925         optional => 1,
 926         default => 'dhcp',
 927     },
 928     gw6 => {
 929         type => 'string',
 930         format => 'ipv6',
 931         format_description => 'GatewayIPv6',
 932         description => 'Default gateway for IPv6 traffic.',
 933         optional => 1,
 934         requires => 'ip6',
 935     },
 936 };
 937 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 938 my $ipconfigdesc = {
 939     optional => 1,
 940     type => 'string', format => 'pve-qm-ipconfig',
 941     description => <<'EODESCR',
 942 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 943
 944 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 945
 946 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 947 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 948
 949 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 950 EODESCR
 951 };
 952 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 953
 954 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 955     $confdesc->{"net$i"} = $netdesc;
 956     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 957 }
 958
 959 foreach my $key (keys %$confdesc_cloudinit) {
 960     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 961 }
 962
 963 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 964 sub verify_volume_id_or_qm_path {
 965     my ($volid, $noerr) = @_;
 966
 967     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 968         return $volid;
 969     }
 970
 971     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 972     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 973     if ($@) {
 974         return undef if $noerr;
 975         die $@;
 976     }
 977     return $volid;
 978 }
 979
 980 my $usb_fmt = {
 981     host => {
 982         default_key => 1,
 983         type => 'string', format => 'pve-qm-usb-device',
 984         format_description => 'HOSTUSBDEVICE|spice',
 985         description => <<EODESCR,
 986 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 987
 988  'bus-port(.port)*' (decimal numbers) or
 989  'vendor_id:product_id' (hexadeciaml numbers) or
 990  'spice'
 991
 992 You can use the 'lsusb -t' command to list existing usb devices.
 993
 994 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 995
 996 The value 'spice' can be used to add a usb redirection devices for spice.
 997 EODESCR
 998     },
 999     usb3 => {
1000         optional => 1,
1001         type => 'boolean',
1002         description => "Specifies whether if given host option is a USB3 device or port.",
1003         default => 0,
1004     },
1005 };
1006
1007 my $usbdesc = {
1008     optional => 1,
1009     type => 'string', format => $usb_fmt,
1010     description => "Configure an USB device (n is 0 to 4).",
1011 };
1012 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1013
1014 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1015 my $hostpci_fmt = {
1016     host => {
1017         default_key => 1,
1018         type => 'string',
1019         pattern => qr/$PCIRE(;$PCIRE)*/,
1020         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1021         description => <<EODESCR,
1022 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1023 of PCI virtual functions of the host. HOSTPCIID syntax is:
1024
1025 'bus:dev.func' (hexadecimal numbers)
1026
1027 You can us the 'lspci' command to list existing PCI devices.
1028 EODESCR
1029     },
1030     rombar => {
1031         type => 'boolean',
1032         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1033         optional => 1,
1034         default => 1,
1035     },
1036     romfile => {
1037         type => 'string',
1038         pattern => '[^,;]+',
1039         format_description => 'string',
1040         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1041         optional => 1,
1042     },
1043     pcie => {
1044         type => 'boolean',
1045         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1046         optional => 1,
1047         default => 0,
1048     },
1049     'x-vga' => {
1050         type => 'boolean',
1051         description =>  "Enable vfio-vga device support.",
1052         optional => 1,
1053         default => 0,
1054     },
1055     'mdev' => {
1056         type => 'string',
1057         format_description => 'string',
1058         pattern => '[^/\.:]+',
1059         optional => 1,
1060         description => <<EODESCR
1061 The type of mediated device to use.
1062 An instance of this type will be created on startup of the VM and
1063 will be cleaned up when the VM stops.
1064 EODESCR
1065     }
1066 };
1067 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1068
1069 my $hostpcidesc = {
1070         optional => 1,
1071         type => 'string', format => 'pve-qm-hostpci',
1072         description => "Map host PCI devices into guest.",
1073         verbose_description =>  <<EODESCR,
1074 Map host PCI devices into guest.
1075
1076 NOTE: This option allows direct access to host hardware. So it is no longer
1077 possible to migrate such machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1083
1084 my $serialdesc = {
1085         optional => 1,
1086         type => 'string',
1087         pattern => '(/dev/.+|socket)',
1088         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1089         verbose_description =>  <<EODESCR,
1090 Create a serial device inside the VM (n is 0 to 3), and pass through a
1091 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1092 host side (use 'qm terminal' to open a terminal connection).
1093
1094 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1095
1096 CAUTION: Experimental! User reported problems with this option.
1097 EODESCR
1098 };
1099
1100 my $paralleldesc= {
1101         optional => 1,
1102         type => 'string',
1103         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1104         description =>  "Map host parallel devices (n is 0 to 2).",
1105         verbose_description =>  <<EODESCR,
1106 Map host parallel devices (n is 0 to 2).
1107
1108 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1109
1110 CAUTION: Experimental! User reported problems with this option.
1111 EODESCR
1112 };
1113
1114 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1115     $confdesc->{"parallel$i"} = $paralleldesc;
1116 }
1117
1118 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1119     $confdesc->{"serial$i"} = $serialdesc;
1120 }
1121
1122 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1123     $confdesc->{"hostpci$i"} = $hostpcidesc;
1124 }
1125
1126 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1127     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1128 }
1129
1130 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1131     $confdesc->{"usb$i"} = $usbdesc;
1132 }
1133
1134 my $kvm_api_version = 0;
1135
1136 sub kvm_version {
1137     return $kvm_api_version if $kvm_api_version;
1138
1139     open my $fh, '<', '/dev/kvm'
1140         or return undef;
1141
1142     # 0xae00 => KVM_GET_API_VERSION
1143     $kvm_api_version = ioctl($fh, 0xae00, 0);
1144
1145     return $kvm_api_version;
1146 }
1147
1148 my $kvm_user_version = {};
1149 my $kvm_mtime = {};
1150
1151 sub kvm_user_version {
1152     my ($binary) = @_;
1153
1154     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1155     my $st = stat($binary);
1156
1157     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1158     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1159         $cachedmtime == $st->mtime;
1160
1161     $kvm_user_version->{$binary} = 'unknown';
1162     $kvm_mtime->{$binary} = $st->mtime;
1163
1164     my $code = sub {
1165         my $line = shift;
1166         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1167             $kvm_user_version->{$binary} = $2;
1168         }
1169     };
1170
1171     eval { run_command([$binary, '--version'], outfunc => $code); };
1172     warn $@ if $@;
1173
1174     return $kvm_user_version->{$binary};
1175
1176 }
1177
1178 sub kernel_has_vhost_net {
1179     return -c '/dev/vhost-net';
1180 }
1181
1182 sub option_exists {
1183     my $key = shift;
1184     return defined($confdesc->{$key});
1185 }
1186
1187 my $cdrom_path;
1188 sub get_cdrom_path {
1189
1190     return  $cdrom_path if $cdrom_path;
1191
1192     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1193     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1194     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1195 }
1196
1197 sub get_iso_path {
1198     my ($storecfg, $vmid, $cdrom) = @_;
1199
1200     if ($cdrom eq 'cdrom') {
1201         return get_cdrom_path();
1202     } elsif ($cdrom eq 'none') {
1203         return '';
1204     } elsif ($cdrom =~ m|^/|) {
1205         return $cdrom;
1206     } else {
1207         return PVE::Storage::path($storecfg, $cdrom);
1208     }
1209 }
1210
1211 # try to convert old style file names to volume IDs
1212 sub filename_to_volume_id {
1213     my ($vmid, $file, $media) = @_;
1214
1215      if (!($file eq 'none' || $file eq 'cdrom' ||
1216           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1217
1218         return undef if $file =~ m|/|;
1219
1220         if ($media && $media eq 'cdrom') {
1221             $file = "local:iso/$file";
1222         } else {
1223             $file = "local:$vmid/$file";
1224         }
1225     }
1226
1227     return $file;
1228 }
1229
1230 sub verify_media_type {
1231     my ($opt, $vtype, $media) = @_;
1232
1233     return if !$media;
1234
1235     my $etype;
1236     if ($media eq 'disk') {
1237         $etype = 'images';
1238     } elsif ($media eq 'cdrom') {
1239         $etype = 'iso';
1240     } else {
1241         die "internal error";
1242     }
1243
1244     return if ($vtype eq $etype);
1245
1246     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1247 }
1248
1249 sub cleanup_drive_path {
1250     my ($opt, $storecfg, $drive) = @_;
1251
1252     # try to convert filesystem paths to volume IDs
1253
1254     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1255         ($drive->{file} !~ m|^/dev/.+|) &&
1256         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1257         ($drive->{file} !~ m/^\d+$/)) {
1258         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1259         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1260         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1261         verify_media_type($opt, $vtype, $drive->{media});
1262         $drive->{file} = $volid;
1263     }
1264
1265     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1266 }
1267
1268 sub parse_hotplug_features {
1269     my ($data) = @_;
1270
1271     my $res = {};
1272
1273     return $res if $data eq '0';
1274
1275     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1276
1277     foreach my $feature (PVE::Tools::split_list($data)) {
1278         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1279             $res->{$1} = 1;
1280         } else {
1281             die "invalid hotplug feature '$feature'\n";
1282         }
1283     }
1284     return $res;
1285 }
1286
1287 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1288 sub pve_verify_hotplug_features {
1289     my ($value, $noerr) = @_;
1290
1291     return $value if parse_hotplug_features($value);
1292
1293     return undef if $noerr;
1294
1295     die "unable to parse hotplug option\n";
1296 }
1297
1298 sub scsi_inquiry {
1299     my($fh, $noerr) = @_;
1300
1301     my $SG_IO = 0x2285;
1302     my $SG_GET_VERSION_NUM = 0x2282;
1303
1304     my $versionbuf = "\x00" x 8;
1305     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1306     if (!$ret) {
1307         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1308         return undef;
1309     }
1310     my $version = unpack("I", $versionbuf);
1311     if ($version < 30000) {
1312         die "scsi generic interface too old\n"  if !$noerr;
1313         return undef;
1314     }
1315
1316     my $buf = "\x00" x 36;
1317     my $sensebuf = "\x00" x 8;
1318     my $cmd = pack("C x3 C x1", 0x12, 36);
1319
1320     # see /usr/include/scsi/sg.h
1321     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1322
1323     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1324                       length($sensebuf), 0, length($buf), $buf,
1325                       $cmd, $sensebuf, 6000);
1326
1327     $ret = ioctl($fh, $SG_IO, $packet);
1328     if (!$ret) {
1329         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1330         return undef;
1331     }
1332
1333     my @res = unpack($sg_io_hdr_t, $packet);
1334     if ($res[17] || $res[18]) {
1335         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1336         return undef;
1337     }
1338
1339     my $res = {};
1340     (my $byte0, my $byte1, $res->{vendor},
1341      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1342
1343     $res->{removable} = $byte1 & 128 ? 1 : 0;
1344     $res->{type} = $byte0 & 31;
1345
1346     return $res;
1347 }
1348
1349 sub path_is_scsi {
1350     my ($path) = @_;
1351
1352     my $fh = IO::File->new("+<$path") || return undef;
1353     my $res = scsi_inquiry($fh, 1);
1354     close($fh);
1355
1356     return $res;
1357 }
1358
1359 sub print_tabletdevice_full {
1360     my ($conf, $arch) = @_;
1361
1362     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1363
1364     # we use uhci for old VMs because tablet driver was buggy in older qemu
1365     my $usbbus;
1366     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1367         $usbbus = 'ehci';
1368     } else {
1369         $usbbus = 'uhci';
1370     }
1371
1372     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1373 }
1374
1375 sub print_keyboarddevice_full {
1376     my ($conf, $arch, $machine) = @_;
1377
1378     return undef if $arch ne 'aarch64';
1379
1380     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1381 }
1382
1383 sub print_drivedevice_full {
1384     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1385
1386     my $device = '';
1387     my $maxdev = 0;
1388
1389     if ($drive->{interface} eq 'virtio') {
1390         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1391         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1392         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1393     } elsif ($drive->{interface} eq 'scsi') {
1394
1395         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1396         my $unit = $drive->{index} % $maxdev;
1397         my $devicetype = 'hd';
1398         my $path = '';
1399         if (drive_is_cdrom($drive)) {
1400             $devicetype = 'cd';
1401         } else {
1402             if ($drive->{file} =~ m|^/|) {
1403                 $path = $drive->{file};
1404                 if (my $info = path_is_scsi($path)) {
1405                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1406                         $devicetype = 'block';
1407                     } elsif ($info->{type} == 1) { # tape
1408                         $devicetype = 'generic';
1409                     }
1410                 }
1411             } else {
1412                  $path = PVE::Storage::path($storecfg, $drive->{file});
1413             }
1414
1415             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1416             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1417             if ($path =~ m/^iscsi\:\/\// &&
1418                !min_version($version, 4, 1)) {
1419                 $devicetype = 'generic';
1420             }
1421         }
1422
1423         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1424             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1425         } else {
1426             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1427         }
1428
1429         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1430             $device .= ",rotation_rate=1";
1431         }
1432         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1433
1434     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1435         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1436         my $controller = int($drive->{index} / $maxdev);
1437         my $unit = $drive->{index} % $maxdev;
1438         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1439
1440         $device = "ide-$devicetype";
1441         if ($drive->{interface} eq 'ide') {
1442             $device .= ",bus=ide.$controller,unit=$unit";
1443         } else {
1444             $device .= ",bus=ahci$controller.$unit";
1445         }
1446         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1447
1448         if ($devicetype eq 'hd') {
1449             if (my $model = $drive->{model}) {
1450                 $model = URI::Escape::uri_unescape($model);
1451                 $device .= ",model=$model";
1452             }
1453             if ($drive->{ssd}) {
1454                 $device .= ",rotation_rate=1";
1455             }
1456         }
1457         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1458     } elsif ($drive->{interface} eq 'usb') {
1459         die "implement me";
1460         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1461     } else {
1462         die "unsupported interface type";
1463     }
1464
1465     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1466
1467     if (my $serial = $drive->{serial}) {
1468         $serial = URI::Escape::uri_unescape($serial);
1469         $device .= ",serial=$serial";
1470     }
1471
1472
1473     return $device;
1474 }
1475
1476 sub get_initiator_name {
1477     my $initiator;
1478
1479     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1480     while (defined(my $line = <$fh>)) {
1481         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1482         $initiator = $1;
1483         last;
1484     }
1485     $fh->close();
1486
1487     return $initiator;
1488 }
1489
1490 sub print_drive_commandline_full {
1491     my ($storecfg, $vmid, $drive) = @_;
1492
1493     my $path;
1494     my $volid = $drive->{file};
1495     my $format;
1496
1497     if (drive_is_cdrom($drive)) {
1498         $path = get_iso_path($storecfg, $vmid, $volid);
1499     } else {
1500         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1501         if ($storeid) {
1502             $path = PVE::Storage::path($storecfg, $volid);
1503             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1504             $format = qemu_img_format($scfg, $volname);
1505         } else {
1506             $path = $volid;
1507             $format = "raw";
1508         }
1509    }
1510
1511     my $opts = '';
1512     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1513     foreach my $o (@qemu_drive_options) {
1514         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1515     }
1516
1517     # snapshot only accepts on|off
1518     if (defined($drive->{snapshot})) {
1519         my $v = $drive->{snapshot} ? 'on' : 'off';
1520         $opts .= ",snapshot=$v";
1521     }
1522
1523     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1524         my ($dir, $qmpname) = @$type;
1525         if (my $v = $drive->{"mbps$dir"}) {
1526             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1527         }
1528         if (my $v = $drive->{"mbps${dir}_max"}) {
1529             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1530         }
1531         if (my $v = $drive->{"bps${dir}_max_length"}) {
1532             $opts .= ",throttling.bps$qmpname-max-length=$v";
1533         }
1534         if (my $v = $drive->{"iops${dir}"}) {
1535             $opts .= ",throttling.iops$qmpname=$v";
1536         }
1537         if (my $v = $drive->{"iops${dir}_max"}) {
1538             $opts .= ",throttling.iops$qmpname-max=$v";
1539         }
1540         if (my $v = $drive->{"iops${dir}_max_length"}) {
1541             $opts .= ",throttling.iops$qmpname-max-length=$v";
1542         }
1543     }
1544
1545     $opts .= ",format=$format" if $format && !$drive->{format};
1546
1547     my $cache_direct = 0;
1548
1549     if (my $cache = $drive->{cache}) {
1550         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1551     } elsif (!drive_is_cdrom($drive)) {
1552         $opts .= ",cache=none";
1553         $cache_direct = 1;
1554     }
1555
1556     # aio native works only with O_DIRECT
1557     if (!$drive->{aio}) {
1558         if($cache_direct) {
1559             $opts .= ",aio=native";
1560         } else {
1561             $opts .= ",aio=threads";
1562         }
1563     }
1564
1565     if (!drive_is_cdrom($drive)) {
1566         my $detectzeroes;
1567         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1568             $detectzeroes = 'off';
1569         } elsif ($drive->{discard}) {
1570             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1571         } else {
1572             # This used to be our default with discard not being specified:
1573             $detectzeroes = 'on';
1574         }
1575         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1576     }
1577
1578     my $pathinfo = $path ? "file=$path," : '';
1579
1580     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1581 }
1582
1583 sub print_netdevice_full {
1584     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1585
1586     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1587
1588     my $device = $net->{model};
1589     if ($net->{model} eq 'virtio') {
1590          $device = 'virtio-net-pci';
1591      };
1592
1593     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1594     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1595     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1596         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1597         my $vectors = $net->{queues} * 2 + 2;
1598         $tmpstr .= ",vectors=$vectors,mq=on";
1599     }
1600     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1601
1602     if (my $mtu = $net->{mtu}) {
1603         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1604             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1605             if ($mtu == 1) {
1606                  $mtu = $bridge_mtu;
1607             } elsif ($mtu < 576) {
1608                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1609             } elsif ($mtu > $bridge_mtu) {
1610                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1611             }
1612             $tmpstr .= ",host_mtu=$mtu";
1613         } else {
1614             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1615         }
1616     }
1617
1618     if ($use_old_bios_files) {
1619         my $romfile;
1620         if ($device eq 'virtio-net-pci') {
1621             $romfile = 'pxe-virtio.rom';
1622         } elsif ($device eq 'e1000') {
1623             $romfile = 'pxe-e1000.rom';
1624         } elsif ($device eq 'ne2k') {
1625             $romfile = 'pxe-ne2k_pci.rom';
1626         } elsif ($device eq 'pcnet') {
1627             $romfile = 'pxe-pcnet.rom';
1628         } elsif ($device eq 'rtl8139') {
1629             $romfile = 'pxe-rtl8139.rom';
1630         }
1631         $tmpstr .= ",romfile=$romfile" if $romfile;
1632     }
1633
1634     return $tmpstr;
1635 }
1636
1637 sub print_netdev_full {
1638     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1639
1640     my $i = '';
1641     if ($netid =~ m/^net(\d+)$/) {
1642         $i = int($1);
1643     }
1644
1645     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1646
1647     my $ifname = "tap${vmid}i$i";
1648
1649     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1650     die "interface name '$ifname' is too long (max 15 character)\n"
1651         if length($ifname) >= 16;
1652
1653     my $vhostparam = '';
1654     if (is_native($arch)) {
1655         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1656     }
1657
1658     my $vmname = $conf->{name} || "vm$vmid";
1659
1660     my $netdev = "";
1661     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1662
1663     if ($net->{bridge}) {
1664         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1665     } else {
1666         $netdev = "type=user,id=$netid,hostname=$vmname";
1667     }
1668
1669     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1670
1671     return $netdev;
1672 }
1673
1674 my $vga_map = {
1675     'cirrus' => 'cirrus-vga',
1676     'std' => 'VGA',
1677     'vmware' => 'vmware-svga',
1678     'virtio' => 'virtio-vga',
1679 };
1680
1681 sub print_vga_device {
1682     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1683
1684     my $type = $vga_map->{$vga->{type}};
1685     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1686         $type = 'virtio-gpu';
1687     }
1688     my $vgamem_mb = $vga->{memory};
1689
1690     my $max_outputs = '';
1691     if ($qxlnum) {
1692         $type = $id ? 'qxl' : 'qxl-vga';
1693
1694         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1695             # set max outputs so linux can have up to 4 qxl displays with one device
1696             if (min_version($machine_version, 4, 1)) {
1697                 $max_outputs = ",max_outputs=4";
1698             }
1699         }
1700     }
1701
1702     die "no devicetype for $vga->{type}\n" if !$type;
1703
1704     my $memory = "";
1705     if ($vgamem_mb) {
1706         if ($vga->{type} eq 'virtio') {
1707             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1708             $memory = ",max_hostmem=$bytes";
1709         } elsif ($qxlnum) {
1710             # from https://www.spice-space.org/multiple-monitors.html
1711             $memory = ",vgamem_mb=$vga->{memory}";
1712             my $ram = $vgamem_mb * 4;
1713             my $vram = $vgamem_mb * 2;
1714             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1715         } else {
1716             $memory = ",vgamem_mb=$vga->{memory}";
1717         }
1718     } elsif ($qxlnum && $id) {
1719         $memory = ",ram_size=67108864,vram_size=33554432";
1720     }
1721
1722     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1723     my $vgaid = "vga" . ($id // '');
1724     my $pciaddr;
1725
1726     if ($q35 && $vgaid eq 'vga') {
1727         # the first display uses pcie.0 bus on q35 machines
1728         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1729     } else {
1730         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1731     }
1732
1733     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1734 }
1735
1736 sub parse_number_sets {
1737     my ($set) = @_;
1738     my $res = [];
1739     foreach my $part (split(/;/, $set)) {
1740         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1741             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1742             push @$res, [ $1, $2 ];
1743         } else {
1744             die "invalid range: $part\n";
1745         }
1746     }
1747     return $res;
1748 }
1749
1750 sub parse_numa {
1751     my ($data) = @_;
1752
1753     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1754     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1755     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1756     return $res;
1757 }
1758
1759 sub parse_hostpci {
1760     my ($value) = @_;
1761
1762     return undef if !$value;
1763
1764     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1765
1766     my @idlist = split(/;/, $res->{host});
1767     delete $res->{host};
1768     foreach my $id (@idlist) {
1769         my $devs = PVE::SysFSTools::lspci($id);
1770         die "no PCI device found for '$id'\n" if !scalar(@$devs);
1771         push @{$res->{pciid}}, @$devs;
1772     }
1773     return $res;
1774 }
1775
1776 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1777 sub parse_net {
1778     my ($data) = @_;
1779
1780     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1781     if ($@) {
1782         warn $@;
1783         return undef;
1784     }
1785     if (!defined($res->{macaddr})) {
1786         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1787         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1788     }
1789     return $res;
1790 }
1791
1792 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1793 sub parse_ipconfig {
1794     my ($data) = @_;
1795
1796     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1797     if ($@) {
1798         warn $@;
1799         return undef;
1800     }
1801
1802     if ($res->{gw} && !$res->{ip}) {
1803         warn 'gateway specified without specifying an IP address';
1804         return undef;
1805     }
1806     if ($res->{gw6} && !$res->{ip6}) {
1807         warn 'IPv6 gateway specified without specifying an IPv6 address';
1808         return undef;
1809     }
1810     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1811         warn 'gateway specified together with DHCP';
1812         return undef;
1813     }
1814     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1815         # gw6 + auto/dhcp
1816         warn "IPv6 gateway specified together with $res->{ip6} address";
1817         return undef;
1818     }
1819
1820     if (!$res->{ip} && !$res->{ip6}) {
1821         return { ip => 'dhcp', ip6 => 'dhcp' };
1822     }
1823
1824     return $res;
1825 }
1826
1827 sub print_net {
1828     my $net = shift;
1829
1830     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1831 }
1832
1833 sub add_random_macs {
1834     my ($settings) = @_;
1835
1836     foreach my $opt (keys %$settings) {
1837         next if $opt !~ m/^net(\d+)$/;
1838         my $net = parse_net($settings->{$opt});
1839         next if !$net;
1840         $settings->{$opt} = print_net($net);
1841     }
1842 }
1843
1844 sub vm_is_volid_owner {
1845     my ($storecfg, $vmid, $volid) = @_;
1846
1847     if ($volid !~  m|^/|) {
1848         my ($path, $owner);
1849         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1850         if ($owner && ($owner == $vmid)) {
1851             return 1;
1852         }
1853     }
1854
1855     return undef;
1856 }
1857
1858 sub vmconfig_register_unused_drive {
1859     my ($storecfg, $vmid, $conf, $drive) = @_;
1860
1861     if (drive_is_cloudinit($drive)) {
1862         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1863         warn $@ if $@;
1864     } elsif (!drive_is_cdrom($drive)) {
1865         my $volid = $drive->{file};
1866         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1867             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1868         }
1869     }
1870 }
1871
1872 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1873 my $smbios1_fmt = {
1874     uuid => {
1875         type => 'string',
1876         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1877         format_description => 'UUID',
1878         description => "Set SMBIOS1 UUID.",
1879         optional => 1,
1880     },
1881     version => {
1882         type => 'string',
1883         pattern => '[A-Za-z0-9+\/]+={0,2}',
1884         format_description => 'Base64 encoded string',
1885         description => "Set SMBIOS1 version.",
1886         optional => 1,
1887     },
1888     serial => {
1889         type => 'string',
1890         pattern => '[A-Za-z0-9+\/]+={0,2}',
1891         format_description => 'Base64 encoded string',
1892         description => "Set SMBIOS1 serial number.",
1893         optional => 1,
1894     },
1895     manufacturer => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 manufacturer.",
1900         optional => 1,
1901     },
1902     product => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 product ID.",
1907         optional => 1,
1908     },
1909     sku => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 SKU string.",
1914         optional => 1,
1915     },
1916     family => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 family string.",
1921         optional => 1,
1922     },
1923     base64 => {
1924         type => 'boolean',
1925         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1926         optional => 1,
1927     },
1928 };
1929
1930 sub parse_smbios1 {
1931     my ($data) = @_;
1932
1933     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1934     warn $@ if $@;
1935     return $res;
1936 }
1937
1938 sub print_smbios1 {
1939     my ($smbios1) = @_;
1940     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1941 }
1942
1943 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1944
1945 sub parse_watchdog {
1946     my ($value) = @_;
1947
1948     return undef if !$value;
1949
1950     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1951     warn $@ if $@;
1952     return $res;
1953 }
1954
1955 sub parse_guest_agent {
1956     my ($value) = @_;
1957
1958     return {} if !defined($value->{agent});
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1961     warn $@ if $@;
1962
1963     # if the agent is disabled ignore the other potentially set properties
1964     return {} if !$res->{enabled};
1965     return $res;
1966 }
1967
1968 sub parse_vga {
1969     my ($value) = @_;
1970
1971     return {} if !$value;
1972     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1973     warn $@ if $@;
1974     return $res;
1975 }
1976
1977 sub parse_rng {
1978     my ($value) = @_;
1979
1980     return undef if !$value;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1983     warn $@ if $@;
1984     return $res;
1985 }
1986
1987 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1988 sub verify_usb_device {
1989     my ($value, $noerr) = @_;
1990
1991     return $value if parse_usb_device($value);
1992
1993     return undef if $noerr;
1994
1995     die "unable to parse usb device\n";
1996 }
1997
1998 # add JSON properties for create and set function
1999 sub json_config_properties {
2000     my $prop = shift;
2001
2002     foreach my $opt (keys %$confdesc) {
2003         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2004             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2005         $prop->{$opt} = $confdesc->{$opt};
2006     }
2007
2008     return $prop;
2009 }
2010
2011 # return copy of $confdesc_cloudinit to generate documentation
2012 sub cloudinit_config_properties {
2013
2014     return dclone($confdesc_cloudinit);
2015 }
2016
2017 sub check_type {
2018     my ($key, $value) = @_;
2019
2020     die "unknown setting '$key'\n" if !$confdesc->{$key};
2021
2022     my $type = $confdesc->{$key}->{type};
2023
2024     if (!defined($value)) {
2025         die "got undefined value\n";
2026     }
2027
2028     if ($value =~ m/[\n\r]/) {
2029         die "property contains a line feed\n";
2030     }
2031
2032     if ($type eq 'boolean') {
2033         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2034         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2035         die "type check ('boolean') failed - got '$value'\n";
2036     } elsif ($type eq 'integer') {
2037         return int($1) if $value =~ m/^(\d+)$/;
2038         die "type check ('integer') failed - got '$value'\n";
2039     } elsif ($type eq 'number') {
2040         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2041         die "type check ('number') failed - got '$value'\n";
2042     } elsif ($type eq 'string') {
2043         if (my $fmt = $confdesc->{$key}->{format}) {
2044             PVE::JSONSchema::check_format($fmt, $value);
2045             return $value;
2046         }
2047         $value =~ s/^\"(.*)\"$/$1/;
2048         return $value;
2049     } else {
2050         die "internal error"
2051     }
2052 }
2053
2054 sub destroy_vm {
2055     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2056
2057     my $conf = PVE::QemuConfig->load_config($vmid);
2058
2059     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2060
2061     if ($conf->{template}) {
2062         # check if any base image is still used by a linked clone
2063         PVE::QemuConfig->foreach_volume($conf, sub {
2064                 my ($ds, $drive) = @_;
2065                 return if drive_is_cdrom($drive);
2066
2067                 my $volid = $drive->{file};
2068                 return if !$volid || $volid =~ m|^/|;
2069
2070                 die "base volume '$volid' is still in use by linked cloned\n"
2071                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2072
2073         });
2074     }
2075
2076     # only remove disks owned by this VM
2077     PVE::QemuConfig->foreach_volume($conf, sub {
2078         my ($ds, $drive) = @_;
2079         return if drive_is_cdrom($drive, 1);
2080
2081         my $volid = $drive->{file};
2082         return if !$volid || $volid =~ m|^/|;
2083
2084         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2085         return if !$path || !$owner || ($owner != $vmid);
2086
2087         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2088         warn "Could not remove disk '$volid', check manually: $@" if $@;
2089     });
2090
2091     # also remove unused disk
2092     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2093     PVE::Storage::foreach_volid($vmdisks, sub {
2094         my ($volid, $sid, $volname, $d) = @_;
2095         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2096         warn $@ if $@;
2097     });
2098
2099     if (defined $replacement_conf) {
2100         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2101     } else {
2102         PVE::QemuConfig->destroy_config($vmid);
2103     }
2104 }
2105
2106 sub parse_vm_config {
2107     my ($filename, $raw) = @_;
2108
2109     return undef if !defined($raw);
2110
2111     my $res = {
2112         digest => Digest::SHA::sha1_hex($raw),
2113         snapshots => {},
2114         pending => {},
2115     };
2116
2117     $filename =~ m|/qemu-server/(\d+)\.conf$|
2118         || die "got strange filename '$filename'";
2119
2120     my $vmid = $1;
2121
2122     my $conf = $res;
2123     my $descr;
2124     my $section = '';
2125
2126     my @lines = split(/\n/, $raw);
2127     foreach my $line (@lines) {
2128         next if $line =~ m/^\s*$/;
2129
2130         if ($line =~ m/^\[PENDING\]\s*$/i) {
2131             $section = 'pending';
2132             if (defined($descr)) {
2133                 $descr =~ s/\s+$//;
2134                 $conf->{description} = $descr;
2135             }
2136             $descr = undef;
2137             $conf = $res->{$section} = {};
2138             next;
2139
2140         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2141             $section = $1;
2142             if (defined($descr)) {
2143                 $descr =~ s/\s+$//;
2144                 $conf->{description} = $descr;
2145             }
2146             $descr = undef;
2147             $conf = $res->{snapshots}->{$section} = {};
2148             next;
2149         }
2150
2151         if ($line =~ m/^\#(.*)\s*$/) {
2152             $descr = '' if !defined($descr);
2153             $descr .= PVE::Tools::decode_text($1) . "\n";
2154             next;
2155         }
2156
2157         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2158             $descr = '' if !defined($descr);
2159             $descr .= PVE::Tools::decode_text($2);
2160         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2161             $conf->{snapstate} = $1;
2162         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2163             my $key = $1;
2164             my $value = $2;
2165             $conf->{$key} = $value;
2166         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2167             my $value = $1;
2168             if ($section eq 'pending') {
2169                 $conf->{delete} = $value; # we parse this later
2170             } else {
2171                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2172             }
2173         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2174             my $key = $1;
2175             my $value = $2;
2176             eval { $value = check_type($key, $value); };
2177             if ($@) {
2178                 warn "vm $vmid - unable to parse value of '$key' - $@";
2179             } else {
2180                 $key = 'ide2' if $key eq 'cdrom';
2181                 my $fmt = $confdesc->{$key}->{format};
2182                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2183                     my $v = parse_drive($key, $value);
2184                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2185                         $v->{file} = $volid;
2186                         $value = print_drive($v);
2187                     } else {
2188                         warn "vm $vmid - unable to parse value of '$key'\n";
2189                         next;
2190                     }
2191                 }
2192
2193                 $conf->{$key} = $value;
2194             }
2195         }
2196     }
2197
2198     if (defined($descr)) {
2199         $descr =~ s/\s+$//;
2200         $conf->{description} = $descr;
2201     }
2202     delete $res->{snapstate}; # just to be sure
2203
2204     return $res;
2205 }
2206
2207 sub write_vm_config {
2208     my ($filename, $conf) = @_;
2209
2210     delete $conf->{snapstate}; # just to be sure
2211
2212     if ($conf->{cdrom}) {
2213         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2214         $conf->{ide2} = $conf->{cdrom};
2215         delete $conf->{cdrom};
2216     }
2217
2218     # we do not use 'smp' any longer
2219     if ($conf->{sockets}) {
2220         delete $conf->{smp};
2221     } elsif ($conf->{smp}) {
2222         $conf->{sockets} = $conf->{smp};
2223         delete $conf->{cores};
2224         delete $conf->{smp};
2225     }
2226
2227     my $used_volids = {};
2228
2229     my $cleanup_config = sub {
2230         my ($cref, $pending, $snapname) = @_;
2231
2232         foreach my $key (keys %$cref) {
2233             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2234                 $key eq 'snapstate' || $key eq 'pending';
2235             my $value = $cref->{$key};
2236             if ($key eq 'delete') {
2237                 die "propertry 'delete' is only allowed in [PENDING]\n"
2238                     if !$pending;
2239                 # fixme: check syntax?
2240                 next;
2241             }
2242             eval { $value = check_type($key, $value); };
2243             die "unable to parse value of '$key' - $@" if $@;
2244
2245             $cref->{$key} = $value;
2246
2247             if (!$snapname && is_valid_drivename($key)) {
2248                 my $drive = parse_drive($key, $value);
2249                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2250             }
2251         }
2252     };
2253
2254     &$cleanup_config($conf);
2255
2256     &$cleanup_config($conf->{pending}, 1);
2257
2258     foreach my $snapname (keys %{$conf->{snapshots}}) {
2259         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2260         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2261     }
2262
2263     # remove 'unusedX' settings if we re-add a volume
2264     foreach my $key (keys %$conf) {
2265         my $value = $conf->{$key};
2266         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2267             delete $conf->{$key};
2268         }
2269     }
2270
2271     my $generate_raw_config = sub {
2272         my ($conf, $pending) = @_;
2273
2274         my $raw = '';
2275
2276         # add description as comment to top of file
2277         if (defined(my $descr = $conf->{description})) {
2278             if ($descr) {
2279                 foreach my $cl (split(/\n/, $descr)) {
2280                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2281                 }
2282             } else {
2283                 $raw .= "#\n" if $pending;
2284             }
2285         }
2286
2287         foreach my $key (sort keys %$conf) {
2288             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2289             $raw .= "$key: $conf->{$key}\n";
2290         }
2291         return $raw;
2292     };
2293
2294     my $raw = &$generate_raw_config($conf);
2295
2296     if (scalar(keys %{$conf->{pending}})){
2297         $raw .= "\n[PENDING]\n";
2298         $raw .= &$generate_raw_config($conf->{pending}, 1);
2299     }
2300
2301     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2302         $raw .= "\n[$snapname]\n";
2303         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2304     }
2305
2306     return $raw;
2307 }
2308
2309 sub load_defaults {
2310
2311     my $res = {};
2312
2313     # we use static defaults from our JSON schema configuration
2314     foreach my $key (keys %$confdesc) {
2315         if (defined(my $default = $confdesc->{$key}->{default})) {
2316             $res->{$key} = $default;
2317         }
2318     }
2319
2320     return $res;
2321 }
2322
2323 sub config_list {
2324     my $vmlist = PVE::Cluster::get_vmlist();
2325     my $res = {};
2326     return $res if !$vmlist || !$vmlist->{ids};
2327     my $ids = $vmlist->{ids};
2328     my $nodename = nodename();
2329
2330     foreach my $vmid (keys %$ids) {
2331         my $d = $ids->{$vmid};
2332         next if !$d->{node} || $d->{node} ne $nodename;
2333         next if !$d->{type} || $d->{type} ne 'qemu';
2334         $res->{$vmid}->{exists} = 1;
2335     }
2336     return $res;
2337 }
2338
2339 # test if VM uses local resources (to prevent migration)
2340 sub check_local_resources {
2341     my ($conf, $noerr) = @_;
2342
2343     my @loc_res = ();
2344
2345     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2346     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2347
2348     push @loc_res, "ivshmem" if $conf->{ivshmem};
2349
2350     foreach my $k (keys %$conf) {
2351         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2352         # sockets are safe: they will recreated be on the target side post-migrate
2353         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2354         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2355     }
2356
2357     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2358
2359     return \@loc_res;
2360 }
2361
2362 # check if used storages are available on all nodes (use by migrate)
2363 sub check_storage_availability {
2364     my ($storecfg, $conf, $node) = @_;
2365
2366     PVE::QemuConfig->foreach_volume($conf, sub {
2367         my ($ds, $drive) = @_;
2368
2369         my $volid = $drive->{file};
2370         return if !$volid;
2371
2372         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2373         return if !$sid;
2374
2375         # check if storage is available on both nodes
2376         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2377         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2378    });
2379 }
2380
2381 # list nodes where all VM images are available (used by has_feature API)
2382 sub shared_nodes {
2383     my ($conf, $storecfg) = @_;
2384
2385     my $nodelist = PVE::Cluster::get_nodelist();
2386     my $nodehash = { map { $_ => 1 } @$nodelist };
2387     my $nodename = nodename();
2388
2389     PVE::QemuConfig->foreach_volume($conf, sub {
2390         my ($ds, $drive) = @_;
2391
2392         my $volid = $drive->{file};
2393         return if !$volid;
2394
2395         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2396         if ($storeid) {
2397             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2398             if ($scfg->{disable}) {
2399                 $nodehash = {};
2400             } elsif (my $avail = $scfg->{nodes}) {
2401                 foreach my $node (keys %$nodehash) {
2402                     delete $nodehash->{$node} if !$avail->{$node};
2403                 }
2404             } elsif (!$scfg->{shared}) {
2405                 foreach my $node (keys %$nodehash) {
2406                     delete $nodehash->{$node} if $node ne $nodename
2407                 }
2408             }
2409         }
2410     });
2411
2412     return $nodehash
2413 }
2414
2415 sub check_local_storage_availability {
2416     my ($conf, $storecfg) = @_;
2417
2418     my $nodelist = PVE::Cluster::get_nodelist();
2419     my $nodehash = { map { $_ => {} } @$nodelist };
2420
2421     PVE::QemuConfig->foreach_volume($conf, sub {
2422         my ($ds, $drive) = @_;
2423
2424         my $volid = $drive->{file};
2425         return if !$volid;
2426
2427         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2428         if ($storeid) {
2429             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2430
2431             if ($scfg->{disable}) {
2432                 foreach my $node (keys %$nodehash) {
2433                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2434                 }
2435             } elsif (my $avail = $scfg->{nodes}) {
2436                 foreach my $node (keys %$nodehash) {
2437                     if (!$avail->{$node}) {
2438                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2439                     }
2440                 }
2441             }
2442         }
2443     });
2444
2445     foreach my $node (values %$nodehash) {
2446         if (my $unavail = $node->{unavailable_storages}) {
2447             $node->{unavailable_storages} = [ sort keys %$unavail ];
2448         }
2449     }
2450
2451     return $nodehash
2452 }
2453
2454 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2455 sub check_running {
2456     my ($vmid, $nocheck, $node) = @_;
2457
2458     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2459     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2460 }
2461
2462 sub vzlist {
2463
2464     my $vzlist = config_list();
2465
2466     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2467
2468     while (defined(my $de = $fd->read)) {
2469         next if $de !~ m/^(\d+)\.pid$/;
2470         my $vmid = $1;
2471         next if !defined($vzlist->{$vmid});
2472         if (my $pid = check_running($vmid)) {
2473             $vzlist->{$vmid}->{pid} = $pid;
2474         }
2475     }
2476
2477     return $vzlist;
2478 }
2479
2480 our $vmstatus_return_properties = {
2481     vmid => get_standard_option('pve-vmid'),
2482     status => {
2483         description => "Qemu process status.",
2484         type => 'string',
2485         enum => ['stopped', 'running'],
2486     },
2487     maxmem => {
2488         description => "Maximum memory in bytes.",
2489         type => 'integer',
2490         optional => 1,
2491         renderer => 'bytes',
2492     },
2493     maxdisk => {
2494         description => "Root disk size in bytes.",
2495         type => 'integer',
2496         optional => 1,
2497         renderer => 'bytes',
2498     },
2499     name => {
2500         description => "VM name.",
2501         type => 'string',
2502         optional => 1,
2503     },
2504     qmpstatus => {
2505         description => "Qemu QMP agent status.",
2506         type => 'string',
2507         optional => 1,
2508     },
2509     pid => {
2510         description => "PID of running qemu process.",
2511         type => 'integer',
2512         optional => 1,
2513     },
2514     uptime => {
2515         description => "Uptime.",
2516         type => 'integer',
2517         optional => 1,
2518         renderer => 'duration',
2519     },
2520     cpus => {
2521         description => "Maximum usable CPUs.",
2522         type => 'number',
2523         optional => 1,
2524     },
2525     lock => {
2526         description => "The current config lock, if any.",
2527         type => 'string',
2528         optional => 1,
2529     },
2530     tags => {
2531         description  => "The current configured tags, if any",
2532         type => 'string',
2533         optional => 1,
2534     },
2535 };
2536
2537 my $last_proc_pid_stat;
2538
2539 # get VM status information
2540 # This must be fast and should not block ($full == false)
2541 # We only query KVM using QMP if $full == true (this can be slow)
2542 sub vmstatus {
2543     my ($opt_vmid, $full) = @_;
2544
2545     my $res = {};
2546
2547     my $storecfg = PVE::Storage::config();
2548
2549     my $list = vzlist();
2550     my $defaults = load_defaults();
2551
2552     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2553
2554     my $cpucount = $cpuinfo->{cpus} || 1;
2555
2556     foreach my $vmid (keys %$list) {
2557         next if $opt_vmid && ($vmid ne $opt_vmid);
2558
2559         my $conf = PVE::QemuConfig->load_config($vmid);
2560
2561         my $d = { vmid => $vmid };
2562         $d->{pid} = $list->{$vmid}->{pid};
2563
2564         # fixme: better status?
2565         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2566
2567         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2568         if (defined($size)) {
2569             $d->{disk} = 0; # no info available
2570             $d->{maxdisk} = $size;
2571         } else {
2572             $d->{disk} = 0;
2573             $d->{maxdisk} = 0;
2574         }
2575
2576         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2577             * ($conf->{cores} || $defaults->{cores});
2578         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2579         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2580
2581         $d->{name} = $conf->{name} || "VM $vmid";
2582         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2583             : $defaults->{memory}*(1024*1024);
2584
2585         if ($conf->{balloon}) {
2586             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2587             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2588                 : $defaults->{shares};
2589         }
2590
2591         $d->{uptime} = 0;
2592         $d->{cpu} = 0;
2593         $d->{mem} = 0;
2594
2595         $d->{netout} = 0;
2596         $d->{netin} = 0;
2597
2598         $d->{diskread} = 0;
2599         $d->{diskwrite} = 0;
2600
2601         $d->{template} = PVE::QemuConfig->is_template($conf);
2602
2603         $d->{serial} = 1 if conf_has_serial($conf);
2604         $d->{lock} = $conf->{lock} if $conf->{lock};
2605         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2606
2607         $res->{$vmid} = $d;
2608     }
2609
2610     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2611     foreach my $dev (keys %$netdev) {
2612         next if $dev !~ m/^tap([1-9]\d*)i/;
2613         my $vmid = $1;
2614         my $d = $res->{$vmid};
2615         next if !$d;
2616
2617         $d->{netout} += $netdev->{$dev}->{receive};
2618         $d->{netin} += $netdev->{$dev}->{transmit};
2619
2620         if ($full) {
2621             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2622             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2623         }
2624
2625     }
2626
2627     my $ctime = gettimeofday;
2628
2629     foreach my $vmid (keys %$list) {
2630
2631         my $d = $res->{$vmid};
2632         my $pid = $d->{pid};
2633         next if !$pid;
2634
2635         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2636         next if !$pstat; # not running
2637
2638         my $used = $pstat->{utime} + $pstat->{stime};
2639
2640         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2641
2642         if ($pstat->{vsize}) {
2643             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2644         }
2645
2646         my $old = $last_proc_pid_stat->{$pid};
2647         if (!$old) {
2648             $last_proc_pid_stat->{$pid} = {
2649                 time => $ctime,
2650                 used => $used,
2651                 cpu => 0,
2652             };
2653             next;
2654         }
2655
2656         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2657
2658         if ($dtime > 1000) {
2659             my $dutime = $used -  $old->{used};
2660
2661             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2662             $last_proc_pid_stat->{$pid} = {
2663                 time => $ctime,
2664                 used => $used,
2665                 cpu => $d->{cpu},
2666             };
2667         } else {
2668             $d->{cpu} = $old->{cpu};
2669         }
2670     }
2671
2672     return $res if !$full;
2673
2674     my $qmpclient = PVE::QMPClient->new();
2675
2676     my $ballooncb = sub {
2677         my ($vmid, $resp) = @_;
2678
2679         my $info = $resp->{'return'};
2680         return if !$info->{max_mem};
2681
2682         my $d = $res->{$vmid};
2683
2684         # use memory assigned to VM
2685         $d->{maxmem} = $info->{max_mem};
2686         $d->{balloon} = $info->{actual};
2687
2688         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2689             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2690             $d->{freemem} = $info->{free_mem};
2691         }
2692
2693         $d->{ballooninfo} = $info;
2694     };
2695
2696     my $blockstatscb = sub {
2697         my ($vmid, $resp) = @_;
2698         my $data = $resp->{'return'} || [];
2699         my $totalrdbytes = 0;
2700         my $totalwrbytes = 0;
2701
2702         for my $blockstat (@$data) {
2703             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2704             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2705
2706             $blockstat->{device} =~ s/drive-//;
2707             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2708         }
2709         $res->{$vmid}->{diskread} = $totalrdbytes;
2710         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2711     };
2712
2713     my $statuscb = sub {
2714         my ($vmid, $resp) = @_;
2715
2716         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2717         # this fails if ballon driver is not loaded, so this must be
2718         # the last commnand (following command are aborted if this fails).
2719         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2720
2721         my $status = 'unknown';
2722         if (!defined($status = $resp->{'return'}->{status})) {
2723             warn "unable to get VM status\n";
2724             return;
2725         }
2726
2727         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2728     };
2729
2730     foreach my $vmid (keys %$list) {
2731         next if $opt_vmid && ($vmid ne $opt_vmid);
2732         next if !$res->{$vmid}->{pid}; # not running
2733         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2734     }
2735
2736     $qmpclient->queue_execute(undef, 2);
2737
2738     foreach my $vmid (keys %$list) {
2739         next if $opt_vmid && ($vmid ne $opt_vmid);
2740         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2741     }
2742
2743     return $res;
2744 }
2745
2746 sub conf_has_serial {
2747     my ($conf) = @_;
2748
2749     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2750         if ($conf->{"serial$i"}) {
2751             return 1;
2752         }
2753     }
2754
2755     return 0;
2756 }
2757
2758 sub conf_has_audio {
2759     my ($conf, $id) = @_;
2760
2761     $id //= 0;
2762     my $audio = $conf->{"audio$id"};
2763     return undef if !defined($audio);
2764
2765     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2766     my $audiodriver = $audioproperties->{driver} // 'spice';
2767
2768     return {
2769         dev => $audioproperties->{device},
2770         dev_id => "audiodev$id",
2771         backend => $audiodriver,
2772         backend_id => "$audiodriver-backend${id}",
2773     };
2774 }
2775
2776 sub audio_devs {
2777     my ($audio, $audiopciaddr, $machine_version) = @_;
2778
2779     my $devs = [];
2780
2781     my $id = $audio->{dev_id};
2782     my $audiodev = "";
2783     if (min_version($machine_version, 4, 2)) {
2784         $audiodev = ",audiodev=$audio->{backend_id}";
2785     }
2786
2787     if ($audio->{dev} eq 'AC97') {
2788         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2789     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2790         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2791         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2792         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2793     } else {
2794         die "unkown audio device '$audio->{dev}', implement me!";
2795     }
2796
2797     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2798
2799     return $devs;
2800 }
2801
2802 sub vga_conf_has_spice {
2803     my ($vga) = @_;
2804
2805     my $vgaconf = parse_vga($vga);
2806     my $vgatype = $vgaconf->{type};
2807     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2808
2809     return $1 || 1;
2810 }
2811
2812 sub is_native($) {
2813     my ($arch) = @_;
2814     return get_host_arch() eq $arch;
2815 }
2816
2817 sub get_vm_arch {
2818     my ($conf) = @_;
2819     return $conf->{arch} // get_host_arch();
2820 }
2821
2822 my $default_machines = {
2823     x86_64 => 'pc',
2824     aarch64 => 'virt',
2825 };
2826
2827 sub get_vm_machine {
2828     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2829
2830     my $machine = $forcemachine || $conf->{machine};
2831
2832     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2833         $arch //= 'x86_64';
2834         $machine ||= $default_machines->{$arch};
2835         if ($add_pve_version) {
2836             $kvmversion //= kvm_user_version();
2837             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2838             $machine .= "+pve$pvever";
2839         }
2840     }
2841
2842     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2843         # for version-pinned machines that do not include a pve-version (e.g.
2844         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2845         $machine .= '+pve0';
2846     }
2847
2848     return $machine;
2849 }
2850
2851 sub get_ovmf_files($) {
2852     my ($arch) = @_;
2853
2854     my $ovmf = $OVMF->{$arch}
2855         or die "no OVMF images known for architecture '$arch'\n";
2856
2857     return @$ovmf;
2858 }
2859
2860 my $Arch2Qemu = {
2861     aarch64 => '/usr/bin/qemu-system-aarch64',
2862     x86_64 => '/usr/bin/qemu-system-x86_64',
2863 };
2864 sub get_command_for_arch($) {
2865     my ($arch) = @_;
2866     return '/usr/bin/kvm' if is_native($arch);
2867
2868     my $cmd = $Arch2Qemu->{$arch}
2869         or die "don't know how to emulate architecture '$arch'\n";
2870     return $cmd;
2871 }
2872
2873 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2874 # to use in a QEMU command line (-cpu element), first array_intersect the result
2875 # of query_supported_ with query_understood_. This is necessary because:
2876 #
2877 # a) query_understood_ returns flags the host cannot use and
2878 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2879 #    flags, but CPU settings - with most of them being flags. Those settings
2880 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2881 #
2882 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2883 # expensive. If you need the value returned from this, you can get it much
2884 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2885 # $accel being 'kvm' or 'tcg'.
2886 #
2887 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2888 # changes, automatically populating pmxcfs.
2889 #
2890 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2891 # since kvm and tcg machines support different flags
2892 #
2893 sub query_supported_cpu_flags {
2894     my ($arch) = @_;
2895
2896     $arch //= get_host_arch();
2897     my $default_machine = $default_machines->{$arch};
2898
2899     my $flags = {};
2900
2901     # FIXME: Once this is merged, the code below should work for ARM as well:
2902     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2903     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2904         $arch eq "aarch64";
2905
2906     my $kvm_supported = defined(kvm_version());
2907     my $qemu_cmd = get_command_for_arch($arch);
2908     my $fakevmid = -1;
2909     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2910
2911     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2912     my $query_supported_run_qemu = sub {
2913         my ($kvm) = @_;
2914
2915         my $flags = {};
2916         my $cmd = [
2917             $qemu_cmd,
2918             '-machine', $default_machine,
2919             '-display', 'none',
2920             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2921             '-mon', 'chardev=qmp,mode=control',
2922             '-pidfile', $pidfile,
2923             '-S', '-daemonize'
2924         ];
2925
2926         if (!$kvm) {
2927             push @$cmd, '-accel', 'tcg';
2928         }
2929
2930         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2931         die "QEMU flag querying VM exited with code " . $rc if $rc;
2932
2933         eval {
2934             my $cmd_result = mon_cmd(
2935                 $fakevmid,
2936                 'query-cpu-model-expansion',
2937                 type => 'full',
2938                 model => { name => 'host' }
2939             );
2940
2941             my $props = $cmd_result->{model}->{props};
2942             foreach my $prop (keys %$props) {
2943                 next if $props->{$prop} ne '1';
2944                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2945                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2946                 # We only keep those with underscores, to match /proc/cpuinfo
2947                 $prop =~ s/\.|-/_/g;
2948                 $flags->{$prop} = 1;
2949             }
2950         };
2951         my $err = $@;
2952
2953         # force stop with 10 sec timeout and 'nocheck'
2954         # always stop, even if QMP failed
2955         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2956
2957         die $err if $err;
2958
2959         return [ sort keys %$flags ];
2960     };
2961
2962     # We need to query QEMU twice, since KVM and TCG have different supported flags
2963     PVE::QemuConfig->lock_config($fakevmid, sub {
2964         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2965         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2966
2967         if ($kvm_supported) {
2968             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2969             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2970         }
2971     });
2972
2973     return $flags;
2974 }
2975
2976 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2977 my $understood_cpu_flag_dir = "/usr/share/kvm";
2978 sub query_understood_cpu_flags {
2979     my $arch = get_host_arch();
2980     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2981
2982     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2983         if ! -e $filepath;
2984
2985     my $raw = file_get_contents($filepath);
2986     $raw =~ s/^\s+|\s+$//g;
2987     my @flags = split(/\s+/, $raw);
2988
2989     return \@flags;
2990 }
2991
2992 sub config_to_command {
2993     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2994
2995     my $cmd = [];
2996     my $globalFlags = [];
2997     my $machineFlags = [];
2998     my $rtcFlags = [];
2999     my $devices = [];
3000     my $pciaddr = '';
3001     my $bridges = {};
3002     my $ostype = $conf->{ostype};
3003     my $winversion = windows_version($ostype);
3004     my $kvm = $conf->{kvm};
3005     my $nodename = nodename();
3006
3007     my $arch = get_vm_arch($conf);
3008     my $kvm_binary = get_command_for_arch($arch);
3009     my $kvmver = kvm_user_version($kvm_binary);
3010
3011     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3012         $kvmver //= "undefined";
3013         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3014     }
3015
3016     my $add_pve_version = min_version($kvmver, 4, 1);
3017
3018     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3019     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3020     $kvm //= 1 if is_native($arch);
3021
3022     $machine_version =~ m/(\d+)\.(\d+)/;
3023     my ($machine_major, $machine_minor) = ($1, $2);
3024
3025     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3026         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3027     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3028         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3029     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3030         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3031         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3032             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3033     }
3034
3035     # if a specific +pve version is required for a feature, use $version_guard
3036     # instead of min_version to allow machines to be run with the minimum
3037     # required version
3038     my $required_pve_version = 0;
3039     my $version_guard = sub {
3040         my ($major, $minor, $pve) = @_;
3041         return 0 if !min_version($machine_version, $major, $minor, $pve);
3042         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3043         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3044         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3045         return 1;
3046     };
3047
3048     if ($kvm) {
3049         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3050             if !defined kvm_version();
3051     }
3052
3053     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3054     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3055     my $use_old_bios_files = undef;
3056     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3057
3058     my $cpuunits = defined($conf->{cpuunits}) ?
3059             $conf->{cpuunits} : $defaults->{cpuunits};
3060
3061     push @$cmd, $kvm_binary;
3062
3063     push @$cmd, '-id', $vmid;
3064
3065     my $vmname = $conf->{name} || "vm$vmid";
3066
3067     push @$cmd, '-name', $vmname;
3068
3069     my $use_virtio = 0;
3070
3071     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3072     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3073     push @$cmd, '-mon', "chardev=qmp,mode=control";
3074
3075     if (min_version($machine_version, 2, 12)) {
3076         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3077         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3078     }
3079
3080     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3081
3082     push @$cmd, '-daemonize';
3083
3084     if ($conf->{smbios1}) {
3085         my $smbios_conf = parse_smbios1($conf->{smbios1});
3086         if ($smbios_conf->{base64}) {
3087             # Do not pass base64 flag to qemu
3088             delete $smbios_conf->{base64};
3089             my $smbios_string = "";
3090             foreach my $key (keys %$smbios_conf) {
3091                 my $value;
3092                 if ($key eq "uuid") {
3093                     $value = $smbios_conf->{uuid}
3094                 } else {
3095                     $value = decode_base64($smbios_conf->{$key});
3096                 }
3097                 # qemu accepts any binary data, only commas need escaping by double comma
3098                 $value =~ s/,/,,/g;
3099                 $smbios_string .= "," . $key . "=" . $value if $value;
3100             }
3101             push @$cmd, '-smbios', "type=1" . $smbios_string;
3102         } else {
3103             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3104         }
3105     }
3106
3107     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3108         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3109         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3110
3111         my ($path, $format);
3112         if (my $efidisk = $conf->{efidisk0}) {
3113             my $d = parse_drive('efidisk0', $efidisk);
3114             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3115             $format = $d->{format};
3116             if ($storeid) {
3117                 $path = PVE::Storage::path($storecfg, $d->{file});
3118                 if (!defined($format)) {
3119                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3120                     $format = qemu_img_format($scfg, $volname);
3121                 }
3122             } else {
3123                 $path = $d->{file};
3124                 die "efidisk format must be specified\n"
3125                     if !defined($format);
3126             }
3127         } else {
3128             warn "no efidisk configured! Using temporary efivars disk.\n";
3129             $path = "/tmp/$vmid-ovmf.fd";
3130             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3131             $format = 'raw';
3132         }
3133
3134         my $size_str = "";
3135
3136         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3137             $size_str = ",size=" . (-s $ovmf_vars);
3138         }
3139
3140         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3141         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3142     }
3143
3144     # load q35 config
3145     if ($q35) {
3146         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3147         if (min_version($machine_version, 4, 0)) {
3148             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3149         } else {
3150             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3151         }
3152     }
3153
3154     if ($conf->{vmgenid}) {
3155         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3156     }
3157
3158     # add usb controllers
3159     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3160     push @$devices, @usbcontrollers if @usbcontrollers;
3161     my $vga = parse_vga($conf->{vga});
3162
3163     my $qxlnum = vga_conf_has_spice($conf->{vga});
3164     $vga->{type} = 'qxl' if $qxlnum;
3165
3166     if (!$vga->{type}) {
3167         if ($arch eq 'aarch64') {
3168             $vga->{type} = 'virtio';
3169         } elsif (min_version($machine_version, 2, 9)) {
3170             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3171         } else {
3172             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3173         }
3174     }
3175
3176     # enable absolute mouse coordinates (needed by vnc)
3177     my $tablet;
3178     if (defined($conf->{tablet})) {
3179         $tablet = $conf->{tablet};
3180     } else {
3181         $tablet = $defaults->{tablet};
3182         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3183         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3184     }
3185
3186     if ($tablet) {
3187         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3188         my $kbd = print_keyboarddevice_full($conf, $arch);
3189         push @$devices, '-device', $kbd if defined($kbd);
3190     }
3191
3192     my $kvm_off = 0;
3193     my $gpu_passthrough;
3194
3195     # host pci devices
3196     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3197         my $id = "hostpci$i";
3198         my $d = parse_hostpci($conf->{$id});
3199         next if !$d;
3200
3201         if (my $pcie = $d->{pcie}) {
3202             die "q35 machine model is not enabled" if !$q35;
3203             # win7 wants to have the pcie devices directly on the pcie bus
3204             # instead of in the root port
3205             if ($winversion == 7) {
3206                 $pciaddr = print_pcie_addr("${id}bus0");
3207             } else {
3208                 # add more root ports if needed, 4 are present by default
3209                 # by pve-q35 cfgs, rest added here on demand.
3210                 if ($i > 3) {
3211                     push @$devices, '-device', print_pcie_root_port($i);
3212                 }
3213                 $pciaddr = print_pcie_addr($id);
3214             }
3215         } else {
3216             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3217         }
3218
3219         my $xvga = '';
3220         if ($d->{'x-vga'}) {
3221             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3222             $kvm_off = 1;
3223             $vga->{type} = 'none' if !defined($conf->{vga});
3224             $gpu_passthrough = 1;
3225         }
3226
3227         my $pcidevices = $d->{pciid};
3228         my $multifunction = 1 if @$pcidevices > 1;
3229
3230         my $sysfspath;
3231         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3232             my $pci_id = $pcidevices->[0]->{id};
3233             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3234             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3235         } elsif ($d->{mdev}) {
3236             warn "ignoring mediated device '$id' with multifunction device\n";
3237         }
3238
3239         my $j=0;
3240         foreach my $pcidevice (@$pcidevices) {
3241             my $devicestr = "vfio-pci";
3242
3243             if ($sysfspath) {
3244                 $devicestr .= ",sysfsdev=$sysfspath";
3245             } else {
3246                 $devicestr .= ",host=$pcidevice->{id}";
3247             }
3248
3249             my $mf_addr = $multifunction ? ".$j" : '';
3250             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3251
3252             if ($j == 0) {
3253                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3254                 $devicestr .= "$xvga";
3255                 $devicestr .= ",multifunction=on" if $multifunction;
3256                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3257             }
3258
3259             push @$devices, '-device', $devicestr;
3260             $j++;
3261         }
3262     }
3263
3264     # usb devices
3265     my $usb_dev_features = {};
3266     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3267
3268     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3269     push @$devices, @usbdevices if @usbdevices;
3270     # serial devices
3271     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3272         if (my $path = $conf->{"serial$i"}) {
3273             if ($path eq 'socket') {
3274                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3275                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3276                 # On aarch64, serial0 is the UART device. Qemu only allows
3277                 # connecting UART devices via the '-serial' command line, as
3278                 # the device has a fixed slot on the hardware...
3279                 if ($arch eq 'aarch64' && $i == 0) {
3280                     push @$devices, '-serial', "chardev:serial$i";
3281                 } else {
3282                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3283                 }
3284             } else {
3285                 die "no such serial device\n" if ! -c $path;
3286                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3287                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3288             }
3289         }
3290     }
3291
3292     # parallel devices
3293     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3294         if (my $path = $conf->{"parallel$i"}) {
3295             die "no such parallel device\n" if ! -c $path;
3296             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3297             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3298             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3299         }
3300     }
3301
3302     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3303         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3304         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3305         push @$devices, @$audio_devs;
3306     }
3307
3308     my $sockets = 1;
3309     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3310     $sockets = $conf->{sockets} if  $conf->{sockets};
3311
3312     my $cores = $conf->{cores} || 1;
3313
3314     my $maxcpus = $sockets * $cores;
3315
3316     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3317
3318     my $allowed_vcpus = $cpuinfo->{cpus};
3319
3320     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3321         if ($allowed_vcpus < $maxcpus);
3322
3323     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3324
3325         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3326         for (my $i = 2; $i <= $vcpus; $i++)  {
3327             my $cpustr = print_cpu_device($conf,$i);
3328             push @$cmd, '-device', $cpustr;
3329         }
3330
3331     } else {
3332
3333         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3334     }
3335     push @$cmd, '-nodefaults';
3336
3337     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3338
3339     my $bootindex_hash = {};
3340     my $i = 1;
3341     foreach my $o (split(//, $bootorder)) {
3342         $bootindex_hash->{$o} = $i*100;
3343         $i++;
3344     }
3345
3346     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3347
3348     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3349
3350     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3351
3352     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3353         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3354         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3355         push @$cmd,  '-vnc', "unix:$socket,password";
3356     } else {
3357         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3358         push @$cmd, '-nographic';
3359     }
3360
3361     # time drift fix
3362     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3363     my $useLocaltime = $conf->{localtime};
3364
3365     if ($winversion >= 5) { # windows
3366         $useLocaltime = 1 if !defined($conf->{localtime});
3367
3368         # use time drift fix when acpi is enabled
3369         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3370             $tdf = 1 if !defined($conf->{tdf});
3371         }
3372     }
3373
3374     if ($winversion >= 6) {
3375         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3376         push @$cmd, '-no-hpet';
3377     }
3378
3379     push @$rtcFlags, 'driftfix=slew' if $tdf;
3380
3381     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3382         push @$rtcFlags, "base=$conf->{startdate}";
3383     } elsif ($useLocaltime) {
3384         push @$rtcFlags, 'base=localtime';
3385     }
3386
3387     if ($forcecpu) {
3388         push @$cmd, '-cpu', $forcecpu;
3389     } else {
3390         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3391     }
3392
3393     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3394
3395     push @$cmd, '-S' if $conf->{freeze};
3396
3397     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3398
3399     my $guest_agent = parse_guest_agent($conf);
3400
3401     if ($guest_agent->{enabled}) {
3402         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3403         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3404
3405         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3406             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3407             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3408             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3409         } elsif ($guest_agent->{type} eq 'isa') {
3410             push @$devices, '-device', "isa-serial,chardev=qga0";
3411         }
3412     }
3413
3414     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3415     if ($rng && &$version_guard(4, 1, 2)) {
3416         check_rng_source($rng->{source});
3417
3418         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3419         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3420         my $limiter_str = "";
3421         if ($max_bytes) {
3422             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3423         }
3424
3425         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3426         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3427         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3428     }
3429
3430     my $spice_port;
3431
3432     if ($qxlnum) {
3433         if ($qxlnum > 1) {
3434             if ($winversion){
3435                 for (my $i = 1; $i < $qxlnum; $i++){
3436                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3437                 }
3438             } else {
3439                 # assume other OS works like Linux
3440                 my ($ram, $vram) = ("134217728", "67108864");
3441                 if ($vga->{memory}) {
3442                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3443                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3444                 }
3445                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3446                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3447             }
3448         }
3449
3450         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3451
3452         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3453         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3454         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3455
3456         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3457         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3458         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3459
3460         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3461         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3462
3463         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3464         if ($spice_enhancement->{foldersharing}) {
3465             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3466             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3467         }
3468
3469         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3470         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3471         push @$devices, '-spice', "$spice_opts";
3472     }
3473
3474     # enable balloon by default, unless explicitly disabled
3475     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3476         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3477         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3478     }
3479
3480     if ($conf->{watchdog}) {
3481         my $wdopts = parse_watchdog($conf->{watchdog});
3482         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3483         my $watchdog = $wdopts->{model} || 'i6300esb';
3484         push @$devices, '-device', "$watchdog$pciaddr";
3485         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3486     }
3487
3488     my $vollist = [];
3489     my $scsicontroller = {};
3490     my $ahcicontroller = {};
3491     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3492
3493     # Add iscsi initiator name if available
3494     if (my $initiator = get_initiator_name()) {
3495         push @$devices, '-iscsi', "initiator-name=$initiator";
3496     }
3497
3498     PVE::QemuConfig->foreach_volume($conf, sub {
3499         my ($ds, $drive) = @_;
3500
3501         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3502             push @$vollist, $drive->{file};
3503         }
3504
3505         # ignore efidisk here, already added in bios/fw handling code above
3506         return if $drive->{interface} eq 'efidisk';
3507
3508         $use_virtio = 1 if $ds =~ m/^virtio/;
3509
3510         if (drive_is_cdrom ($drive)) {
3511             if ($bootindex_hash->{d}) {
3512                 $drive->{bootindex} = $bootindex_hash->{d};
3513                 $bootindex_hash->{d} += 1;
3514             }
3515         } else {
3516             if ($bootindex_hash->{c}) {
3517                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3518                 $bootindex_hash->{c} += 1;
3519             }
3520         }
3521
3522         if ($drive->{interface} eq 'virtio'){
3523            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3524         }
3525
3526         if ($drive->{interface} eq 'scsi') {
3527
3528             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3529
3530             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3531                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3532
3533             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3534             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3535
3536             my $iothread = '';
3537             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3538                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3539                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3540             } elsif ($drive->{iothread}) {
3541                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3542             }
3543
3544             my $queues = '';
3545             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3546                 $queues = ",num_queues=$drive->{queues}";
3547             }
3548
3549             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3550             $scsicontroller->{$controller}=1;
3551         }
3552
3553         if ($drive->{interface} eq 'sata') {
3554             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3555             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3556             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3557             $ahcicontroller->{$controller}=1;
3558         }
3559
3560         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3561         push @$devices, '-drive',$drive_cmd;
3562         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3563     });
3564
3565     for (my $i = 0; $i < $MAX_NETS; $i++) {
3566          next if !$conf->{"net$i"};
3567          my $d = parse_net($conf->{"net$i"});
3568          next if !$d;
3569
3570          $use_virtio = 1 if $d->{model} eq 'virtio';
3571
3572          if ($bootindex_hash->{n}) {
3573             $d->{bootindex} = $bootindex_hash->{n};
3574             $bootindex_hash->{n} += 1;
3575          }
3576
3577          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3578          push @$devices, '-netdev', $netdevfull;
3579
3580          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3581          push @$devices, '-device', $netdevicefull;
3582     }
3583
3584     if ($conf->{ivshmem}) {
3585         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3586
3587         my $bus;
3588         if ($q35) {
3589             $bus = print_pcie_addr("ivshmem");
3590         } else {
3591             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3592         }
3593
3594         my $ivshmem_name = $ivshmem->{name} // $vmid;
3595         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3596
3597         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3598         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3599     }
3600
3601     # pci.4 is nested in pci.1
3602     $bridges->{1} = 1 if $bridges->{4};
3603
3604     if (!$q35) {
3605         # add pci bridges
3606         if (min_version($machine_version, 2, 3)) {
3607            $bridges->{1} = 1;
3608            $bridges->{2} = 1;
3609         }
3610
3611         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3612
3613     }
3614
3615     for my $k (sort {$b cmp $a} keys %$bridges) {
3616         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3617         $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3618         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3619         if ($q35) {
3620             # add after -readconfig pve-q35.cfg
3621             splice @$devices, 2, 0, '-device', $devstr;
3622         } else {
3623             unshift @$devices, '-device', $devstr if $k > 0;
3624         }
3625     }
3626
3627     if (!$kvm) {
3628         push @$machineFlags, 'accel=tcg';
3629     }
3630
3631     my $machine_type_min = $machine_type;
3632     if ($add_pve_version) {
3633         $machine_type_min =~ s/\+pve\d+$//;
3634         $machine_type_min .= "+pve$required_pve_version";
3635     }
3636     push @$machineFlags, "type=${machine_type_min}";
3637
3638     push @$cmd, @$devices;
3639     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3640     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3641     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3642
3643     if (my $vmstate = $conf->{vmstate}) {
3644         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3645         push @$vollist, $vmstate;
3646         push @$cmd, '-loadstate', $statepath;
3647         print "activating and using '$vmstate' as vmstate\n";
3648     }
3649
3650     # add custom args
3651     if ($conf->{args}) {
3652         my $aa = PVE::Tools::split_args($conf->{args});
3653         push @$cmd, @$aa;
3654     }
3655
3656     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3657 }
3658
3659 sub check_rng_source {
3660     my ($source) = @_;
3661
3662     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3663     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3664         if ! -e $source;
3665
3666     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3667     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3668         # Needs to abort, otherwise QEMU crashes on first rng access.
3669         # Note that rng_current cannot be changed to 'none' manually, so
3670         # once the VM is past this point, it is no longer an issue.
3671         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3672             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3673             . " a compatible hardware-RNG is attached to the host.\n";
3674     }
3675 }
3676
3677 sub spice_port {
3678     my ($vmid) = @_;
3679
3680     my $res = mon_cmd($vmid, 'query-spice');
3681
3682     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3683 }
3684
3685 sub vm_devices_list {
3686     my ($vmid) = @_;
3687
3688     my $res = mon_cmd($vmid, 'query-pci');
3689     my $devices_to_check = [];
3690     my $devices = {};
3691     foreach my $pcibus (@$res) {
3692         push @$devices_to_check, @{$pcibus->{devices}},
3693     }
3694
3695     while (@$devices_to_check) {
3696         my $to_check = [];
3697         for my $d (@$devices_to_check) {
3698             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3699             next if !$d->{'pci_bridge'};
3700
3701             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3702             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3703         }
3704         $devices_to_check = $to_check;
3705     }
3706
3707     my $resblock = mon_cmd($vmid, 'query-block');
3708     foreach my $block (@$resblock) {
3709         if($block->{device} =~ m/^drive-(\S+)/){
3710                 $devices->{$1} = 1;
3711         }
3712     }
3713
3714     my $resmice = mon_cmd($vmid, 'query-mice');
3715     foreach my $mice (@$resmice) {
3716         if ($mice->{name} eq 'QEMU HID Tablet') {
3717             $devices->{tablet} = 1;
3718             last;
3719         }
3720     }
3721
3722     # for usb devices there is no query-usb
3723     # but we can iterate over the entries in
3724     # qom-list path=/machine/peripheral
3725     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3726     foreach my $per (@$resperipheral) {
3727         if ($per->{name} =~ m/^usb\d+$/) {
3728             $devices->{$per->{name}} = 1;
3729         }
3730     }
3731
3732     return $devices;
3733 }
3734
3735 sub vm_deviceplug {
3736     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3737
3738     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3739
3740     my $devices_list = vm_devices_list($vmid);
3741     return 1 if defined($devices_list->{$deviceid});
3742
3743     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3744
3745     if ($deviceid eq 'tablet') {
3746
3747         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3748
3749     } elsif ($deviceid eq 'keyboard') {
3750
3751         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3752
3753     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3754
3755         die "usb hotplug currently not reliable\n";
3756         # since we can't reliably hot unplug all added usb devices
3757         # and usb passthrough disables live migration
3758         # we disable usb hotplugging for now
3759         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3760
3761     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3762
3763         qemu_iothread_add($vmid, $deviceid, $device);
3764
3765         qemu_driveadd($storecfg, $vmid, $device);
3766         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3767
3768         qemu_deviceadd($vmid, $devicefull);
3769         eval { qemu_deviceaddverify($vmid, $deviceid); };
3770         if (my $err = $@) {
3771             eval { qemu_drivedel($vmid, $deviceid); };
3772             warn $@ if $@;
3773             die $err;
3774         }
3775
3776     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3777
3778
3779         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3780         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3781         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3782
3783         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3784
3785         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3786             qemu_iothread_add($vmid, $deviceid, $device);
3787             $devicefull .= ",iothread=iothread-$deviceid";
3788         }
3789
3790         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3791             $devicefull .= ",num_queues=$device->{queues}";
3792         }
3793
3794         qemu_deviceadd($vmid, $devicefull);
3795         qemu_deviceaddverify($vmid, $deviceid);
3796
3797     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3798
3799         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3800         qemu_driveadd($storecfg, $vmid, $device);
3801
3802         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3803         eval { qemu_deviceadd($vmid, $devicefull); };
3804         if (my $err = $@) {
3805             eval { qemu_drivedel($vmid, $deviceid); };
3806             warn $@ if $@;
3807             die $err;
3808         }
3809
3810     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3811
3812         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3813
3814         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3815         my $use_old_bios_files = undef;
3816         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3817
3818         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3819         qemu_deviceadd($vmid, $netdevicefull);
3820         eval {
3821             qemu_deviceaddverify($vmid, $deviceid);
3822             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3823         };
3824         if (my $err = $@) {
3825             eval { qemu_netdevdel($vmid, $deviceid); };
3826             warn $@ if $@;
3827             die $err;
3828         }
3829
3830     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3831
3832         my $bridgeid = $2;
3833         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3834         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3835
3836         qemu_deviceadd($vmid, $devicefull);
3837         qemu_deviceaddverify($vmid, $deviceid);
3838
3839     } else {
3840         die "can't hotplug device '$deviceid'\n";
3841     }
3842
3843     return 1;
3844 }
3845
3846 # fixme: this should raise exceptions on error!
3847 sub vm_deviceunplug {
3848     my ($vmid, $conf, $deviceid) = @_;
3849
3850     my $devices_list = vm_devices_list($vmid);
3851     return 1 if !defined($devices_list->{$deviceid});
3852
3853     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3854
3855     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3856
3857         qemu_devicedel($vmid, $deviceid);
3858
3859     } elsif ($deviceid =~ m/^usb\d+$/) {
3860
3861         die "usb hotplug currently not reliable\n";
3862         # when unplugging usb devices this way,
3863         # there may be remaining usb controllers/hubs
3864         # so we disable it for now
3865         qemu_devicedel($vmid, $deviceid);
3866         qemu_devicedelverify($vmid, $deviceid);
3867
3868     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3869
3870         qemu_devicedel($vmid, $deviceid);
3871         qemu_devicedelverify($vmid, $deviceid);
3872         qemu_drivedel($vmid, $deviceid);
3873         qemu_iothread_del($conf, $vmid, $deviceid);
3874
3875     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3876
3877         qemu_devicedel($vmid, $deviceid);
3878         qemu_devicedelverify($vmid, $deviceid);
3879         qemu_iothread_del($conf, $vmid, $deviceid);
3880
3881     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3882
3883         qemu_devicedel($vmid, $deviceid);
3884         qemu_drivedel($vmid, $deviceid);
3885         qemu_deletescsihw($conf, $vmid, $deviceid);
3886
3887     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3888
3889         qemu_devicedel($vmid, $deviceid);
3890         qemu_devicedelverify($vmid, $deviceid);
3891         qemu_netdevdel($vmid, $deviceid);
3892
3893     } else {
3894         die "can't unplug device '$deviceid'\n";
3895     }
3896
3897     return 1;
3898 }
3899
3900 sub qemu_deviceadd {
3901     my ($vmid, $devicefull) = @_;
3902
3903     $devicefull = "driver=".$devicefull;
3904     my %options =  split(/[=,]/, $devicefull);
3905
3906     mon_cmd($vmid, "device_add" , %options);
3907 }
3908
3909 sub qemu_devicedel {
3910     my ($vmid, $deviceid) = @_;
3911
3912     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3913 }
3914
3915 sub qemu_iothread_add {
3916     my($vmid, $deviceid, $device) = @_;
3917
3918     if ($device->{iothread}) {
3919         my $iothreads = vm_iothreads_list($vmid);
3920         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3921     }
3922 }
3923
3924 sub qemu_iothread_del {
3925     my($conf, $vmid, $deviceid) = @_;
3926
3927     my $confid = $deviceid;
3928     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3929         $confid = 'scsi' . $1;
3930     }
3931     my $device = parse_drive($confid, $conf->{$confid});
3932     if ($device->{iothread}) {
3933         my $iothreads = vm_iothreads_list($vmid);
3934         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3935     }
3936 }
3937
3938 sub qemu_objectadd {
3939     my($vmid, $objectid, $qomtype) = @_;
3940
3941     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3942
3943     return 1;
3944 }
3945
3946 sub qemu_objectdel {
3947     my($vmid, $objectid) = @_;
3948
3949     mon_cmd($vmid, "object-del", id => $objectid);
3950
3951     return 1;
3952 }
3953
3954 sub qemu_driveadd {
3955     my ($storecfg, $vmid, $device) = @_;
3956
3957     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3958     $drive =~ s/\\/\\\\/g;
3959     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3960
3961     # If the command succeeds qemu prints: "OK"
3962     return 1 if $ret =~ m/OK/s;
3963
3964     die "adding drive failed: $ret\n";
3965 }
3966
3967 sub qemu_drivedel {
3968     my($vmid, $deviceid) = @_;
3969
3970     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3971     $ret =~ s/^\s+//;
3972
3973     return 1 if $ret eq "";
3974
3975     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3976     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3977
3978     die "deleting drive $deviceid failed : $ret\n";
3979 }
3980
3981 sub qemu_deviceaddverify {
3982     my ($vmid, $deviceid) = @_;
3983
3984     for (my $i = 0; $i <= 5; $i++) {
3985          my $devices_list = vm_devices_list($vmid);
3986          return 1 if defined($devices_list->{$deviceid});
3987          sleep 1;
3988     }
3989
3990     die "error on hotplug device '$deviceid'\n";
3991 }
3992
3993
3994 sub qemu_devicedelverify {
3995     my ($vmid, $deviceid) = @_;
3996
3997     # need to verify that the device is correctly removed as device_del
3998     # is async and empty return is not reliable
3999
4000     for (my $i = 0; $i <= 5; $i++) {
4001          my $devices_list = vm_devices_list($vmid);
4002          return 1 if !defined($devices_list->{$deviceid});
4003          sleep 1;
4004     }
4005
4006     die "error on hot-unplugging device '$deviceid'\n";
4007 }
4008
4009 sub qemu_findorcreatescsihw {
4010     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4011
4012     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4013
4014     my $scsihwid="$controller_prefix$controller";
4015     my $devices_list = vm_devices_list($vmid);
4016
4017     if(!defined($devices_list->{$scsihwid})) {
4018         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4019     }
4020
4021     return 1;
4022 }
4023
4024 sub qemu_deletescsihw {
4025     my ($conf, $vmid, $opt) = @_;
4026
4027     my $device = parse_drive($opt, $conf->{$opt});
4028
4029     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4030         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4031         return 1;
4032     }
4033
4034     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4035
4036     my $devices_list = vm_devices_list($vmid);
4037     foreach my $opt (keys %{$devices_list}) {
4038         if (is_valid_drivename($opt)) {
4039             my $drive = parse_drive($opt, $conf->{$opt});
4040             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4041                 return 1;
4042             }
4043         }
4044     }
4045
4046     my $scsihwid="scsihw$controller";
4047
4048     vm_deviceunplug($vmid, $conf, $scsihwid);
4049
4050     return 1;
4051 }
4052
4053 sub qemu_add_pci_bridge {
4054     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4055
4056     my $bridges = {};
4057
4058     my $bridgeid;
4059
4060     print_pci_addr($device, $bridges, $arch, $machine_type);
4061
4062     while (my ($k, $v) = each %$bridges) {
4063         $bridgeid = $k;
4064     }
4065     return 1 if !defined($bridgeid) || $bridgeid < 1;
4066
4067     my $bridge = "pci.$bridgeid";
4068     my $devices_list = vm_devices_list($vmid);
4069
4070     if (!defined($devices_list->{$bridge})) {
4071         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4072     }
4073
4074     return 1;
4075 }
4076
4077 sub qemu_set_link_status {
4078     my ($vmid, $device, $up) = @_;
4079
4080     mon_cmd($vmid, "set_link", name => $device,
4081                up => $up ? JSON::true : JSON::false);
4082 }
4083
4084 sub qemu_netdevadd {
4085     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4086
4087     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4088     my %options =  split(/[=,]/, $netdev);
4089
4090     if (defined(my $vhost = $options{vhost})) {
4091         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4092     }
4093
4094     if (defined(my $queues = $options{queues})) {
4095         $options{queues} = $queues + 0;
4096     }
4097
4098     mon_cmd($vmid, "netdev_add",  %options);
4099     return 1;
4100 }
4101
4102 sub qemu_netdevdel {
4103     my ($vmid, $deviceid) = @_;
4104
4105     mon_cmd($vmid, "netdev_del", id => $deviceid);
4106 }
4107
4108 sub qemu_usb_hotplug {
4109     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4110
4111     return if !$device;
4112
4113     # remove the old one first
4114     vm_deviceunplug($vmid, $conf, $deviceid);
4115
4116     # check if xhci controller is necessary and available
4117     if ($device->{usb3}) {
4118
4119         my $devicelist = vm_devices_list($vmid);
4120
4121         if (!$devicelist->{xhci}) {
4122             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4123             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4124         }
4125     }
4126     my $d = parse_usb_device($device->{host});
4127     $d->{usb3} = $device->{usb3};
4128
4129     # add the new one
4130     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4131 }
4132
4133 sub qemu_cpu_hotplug {
4134     my ($vmid, $conf, $vcpus) = @_;
4135
4136     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4137
4138     my $sockets = 1;
4139     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4140     $sockets = $conf->{sockets} if  $conf->{sockets};
4141     my $cores = $conf->{cores} || 1;
4142     my $maxcpus = $sockets * $cores;
4143
4144     $vcpus = $maxcpus if !$vcpus;
4145
4146     die "you can't add more vcpus than maxcpus\n"
4147         if $vcpus > $maxcpus;
4148
4149     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4150
4151     if ($vcpus < $currentvcpus) {
4152
4153         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4154
4155             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4156                 qemu_devicedel($vmid, "cpu$i");
4157                 my $retry = 0;
4158                 my $currentrunningvcpus = undef;
4159                 while (1) {
4160                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4161                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4162                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4163                     $retry++;
4164                     sleep 1;
4165                 }
4166                 #update conf after each succesfull cpu unplug
4167                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4168                 PVE::QemuConfig->write_config($vmid, $conf);
4169             }
4170         } else {
4171             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4172         }
4173
4174         return;
4175     }
4176
4177     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4178     die "vcpus in running vm does not match its configuration\n"
4179         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4180
4181     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4182
4183         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4184             my $cpustr = print_cpu_device($conf, $i);
4185             qemu_deviceadd($vmid, $cpustr);
4186
4187             my $retry = 0;
4188             my $currentrunningvcpus = undef;
4189             while (1) {
4190                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4191                 last if scalar(@{$currentrunningvcpus}) == $i;
4192                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4193                 sleep 1;
4194                 $retry++;
4195             }
4196             #update conf after each succesfull cpu hotplug
4197             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4198             PVE::QemuConfig->write_config($vmid, $conf);
4199         }
4200     } else {
4201
4202         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4203             mon_cmd($vmid, "cpu-add", id => int($i));
4204         }
4205     }
4206 }
4207
4208 sub qemu_block_set_io_throttle {
4209     my ($vmid, $deviceid,
4210         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4211         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4212         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4213         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4214
4215     return if !check_running($vmid) ;
4216
4217     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4218         bps => int($bps),
4219         bps_rd => int($bps_rd),
4220         bps_wr => int($bps_wr),
4221         iops => int($iops),
4222         iops_rd => int($iops_rd),
4223         iops_wr => int($iops_wr),
4224         bps_max => int($bps_max),
4225         bps_rd_max => int($bps_rd_max),
4226         bps_wr_max => int($bps_wr_max),
4227         iops_max => int($iops_max),
4228         iops_rd_max => int($iops_rd_max),
4229         iops_wr_max => int($iops_wr_max),
4230         bps_max_length => int($bps_max_length),
4231         bps_rd_max_length => int($bps_rd_max_length),
4232         bps_wr_max_length => int($bps_wr_max_length),
4233         iops_max_length => int($iops_max_length),
4234         iops_rd_max_length => int($iops_rd_max_length),
4235         iops_wr_max_length => int($iops_wr_max_length),
4236     );
4237
4238 }
4239
4240 # old code, only used to shutdown old VM after update
4241 sub __read_avail {
4242     my ($fh, $timeout) = @_;
4243
4244     my $sel = new IO::Select;
4245     $sel->add($fh);
4246
4247     my $res = '';
4248     my $buf;
4249
4250     my @ready;
4251     while (scalar (@ready = $sel->can_read($timeout))) {
4252         my $count;
4253         if ($count = $fh->sysread($buf, 8192)) {
4254             if ($buf =~ /^(.*)\(qemu\) $/s) {
4255                 $res .= $1;
4256                 last;
4257             } else {
4258                 $res .= $buf;
4259             }
4260         } else {
4261             if (!defined($count)) {
4262                 die "$!\n";
4263             }
4264             last;
4265         }
4266     }
4267
4268     die "monitor read timeout\n" if !scalar(@ready);
4269
4270     return $res;
4271 }
4272
4273 sub qemu_block_resize {
4274     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4275
4276     my $running = check_running($vmid);
4277
4278     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4279
4280     return if !$running;
4281
4282     my $padding = (1024 - $size % 1024) % 1024;
4283     $size = $size + $padding;
4284
4285     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4286
4287 }
4288
4289 sub qemu_volume_snapshot {
4290     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4291
4292     my $running = check_running($vmid);
4293
4294     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4295         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4296     } else {
4297         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4298     }
4299 }
4300
4301 sub qemu_volume_snapshot_delete {
4302     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4303
4304     my $running = check_running($vmid);
4305
4306     if($running) {
4307
4308         $running = undef;
4309         my $conf = PVE::QemuConfig->load_config($vmid);
4310         PVE::QemuConfig->foreach_volume($conf, sub {
4311             my ($ds, $drive) = @_;
4312             $running = 1 if $drive->{file} eq $volid;
4313         });
4314     }
4315
4316     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4317         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4318     } else {
4319         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4320     }
4321 }
4322
4323 sub set_migration_caps {
4324     my ($vmid) = @_;
4325
4326     my $cap_ref = [];
4327
4328     my $enabled_cap = {
4329         "auto-converge" => 1,
4330         "xbzrle" => 1,
4331         "x-rdma-pin-all" => 0,
4332         "zero-blocks" => 0,
4333         "compress" => 0
4334     };
4335
4336     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4337
4338     for my $supported_capability (@$supported_capabilities) {
4339         push @$cap_ref, {
4340             capability => $supported_capability->{capability},
4341             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4342         };
4343     }
4344
4345     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4346 }
4347
4348 sub foreach_volid {
4349     my ($conf, $func, @param) = @_;
4350
4351     my $volhash = {};
4352
4353     my $test_volid = sub {
4354         my ($key, $drive, $snapname) = @_;
4355
4356         my $volid = $drive->{file};
4357         return if !$volid;
4358
4359         $volhash->{$volid}->{cdrom} //= 1;
4360         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4361
4362         my $replicate = $drive->{replicate} // 1;
4363         $volhash->{$volid}->{replicate} //= 0;
4364         $volhash->{$volid}->{replicate} = 1 if $replicate;
4365
4366         $volhash->{$volid}->{shared} //= 0;
4367         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4368
4369         $volhash->{$volid}->{referenced_in_config} //= 0;
4370         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4371
4372         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4373             if defined($snapname);
4374
4375         my $size = $drive->{size};
4376         $volhash->{$volid}->{size} //= $size if $size;
4377
4378         $volhash->{$volid}->{is_vmstate} //= 0;
4379         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4380
4381         $volhash->{$volid}->{is_unused} //= 0;
4382         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4383     };
4384
4385     my $include_opts = {
4386         extra_keys => ['vmstate'],
4387         include_unused => 1,
4388     };
4389
4390     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4391     foreach my $snapname (keys %{$conf->{snapshots}}) {
4392         my $snap = $conf->{snapshots}->{$snapname};
4393         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4394     }
4395
4396     foreach my $volid (keys %$volhash) {
4397         &$func($volid, $volhash->{$volid}, @param);
4398     }
4399 }
4400
4401 my $fast_plug_option = {
4402     'lock' => 1,
4403     'name' => 1,
4404     'onboot' => 1,
4405     'shares' => 1,
4406     'startup' => 1,
4407     'description' => 1,
4408     'protection' => 1,
4409     'vmstatestorage' => 1,
4410     'hookscript' => 1,
4411     'tags' => 1,
4412 };
4413
4414 # hotplug changes in [PENDING]
4415 # $selection hash can be used to only apply specified options, for
4416 # example: { cores => 1 } (only apply changed 'cores')
4417 # $errors ref is used to return error messages
4418 sub vmconfig_hotplug_pending {
4419     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4420
4421     my $defaults = load_defaults();
4422     my $arch = get_vm_arch($conf);
4423     my $machine_type = get_vm_machine($conf, undef, $arch);
4424
4425     # commit values which do not have any impact on running VM first
4426     # Note: those option cannot raise errors, we we do not care about
4427     # $selection and always apply them.
4428
4429     my $add_error = sub {
4430         my ($opt, $msg) = @_;
4431         $errors->{$opt} = "hotplug problem - $msg";
4432     };
4433
4434     my $changes = 0;
4435     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4436         if ($fast_plug_option->{$opt}) {
4437             $conf->{$opt} = $conf->{pending}->{$opt};
4438             delete $conf->{pending}->{$opt};
4439             $changes = 1;
4440         }
4441     }
4442
4443     if ($changes) {
4444         PVE::QemuConfig->write_config($vmid, $conf);
4445     }
4446
4447     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4448
4449     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4450     foreach my $opt (sort keys %$pending_delete_hash) {
4451         next if $selection && !$selection->{$opt};
4452         my $force = $pending_delete_hash->{$opt}->{force};
4453         eval {
4454             if ($opt eq 'hotplug') {
4455                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4456             } elsif ($opt eq 'tablet') {
4457                 die "skip\n" if !$hotplug_features->{usb};
4458                 if ($defaults->{tablet}) {
4459                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4460                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4461                         if $arch eq 'aarch64';
4462                 } else {
4463                     vm_deviceunplug($vmid, $conf, 'tablet');
4464                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4465                 }
4466             } elsif ($opt =~ m/^usb\d+/) {
4467                 die "skip\n";
4468                 # since we cannot reliably hot unplug usb devices
4469                 # we are disabling it
4470                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4471                 vm_deviceunplug($vmid, $conf, $opt);
4472             } elsif ($opt eq 'vcpus') {
4473                 die "skip\n" if !$hotplug_features->{cpu};
4474                 qemu_cpu_hotplug($vmid, $conf, undef);
4475             } elsif ($opt eq 'balloon') {
4476                 # enable balloon device is not hotpluggable
4477                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4478                 # here we reset the ballooning value to memory
4479                 my $balloon = $conf->{memory} || $defaults->{memory};
4480                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4481             } elsif ($fast_plug_option->{$opt}) {
4482                 # do nothing
4483             } elsif ($opt =~ m/^net(\d+)$/) {
4484                 die "skip\n" if !$hotplug_features->{network};
4485                 vm_deviceunplug($vmid, $conf, $opt);
4486             } elsif (is_valid_drivename($opt)) {
4487                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4488                 vm_deviceunplug($vmid, $conf, $opt);
4489                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4490             } elsif ($opt =~ m/^memory$/) {
4491                 die "skip\n" if !$hotplug_features->{memory};
4492                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4493             } elsif ($opt eq 'cpuunits') {
4494                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4495             } elsif ($opt eq 'cpulimit') {
4496                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4497             } else {
4498                 die "skip\n";
4499             }
4500         };
4501         if (my $err = $@) {
4502             &$add_error($opt, $err) if $err ne "skip\n";
4503         } else {
4504             delete $conf->{$opt};
4505             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4506         }
4507     }
4508
4509     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4510     $apply_pending_cloudinit = sub {
4511         return if $apply_pending_cloudinit_done; # once is enough
4512         $apply_pending_cloudinit_done = 1; # once is enough
4513
4514         my ($key, $value) = @_;
4515
4516         my @cloudinit_opts = keys %$confdesc_cloudinit;
4517         foreach my $opt (keys %{$conf->{pending}}) {
4518             next if !grep { $_ eq $opt } @cloudinit_opts;
4519             $conf->{$opt} = delete $conf->{pending}->{$opt};
4520         }
4521
4522         my $new_conf = { %$conf };
4523         $new_conf->{$key} = $value;
4524         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4525     };
4526
4527     foreach my $opt (keys %{$conf->{pending}}) {
4528         next if $selection && !$selection->{$opt};
4529         my $value = $conf->{pending}->{$opt};
4530         eval {
4531             if ($opt eq 'hotplug') {
4532                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4533             } elsif ($opt eq 'tablet') {
4534                 die "skip\n" if !$hotplug_features->{usb};
4535                 if ($value == 1) {
4536                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4537                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4538                         if $arch eq 'aarch64';
4539                 } elsif ($value == 0) {
4540                     vm_deviceunplug($vmid, $conf, 'tablet');
4541                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4542                 }
4543             } elsif ($opt =~ m/^usb\d+$/) {
4544                 die "skip\n";
4545                 # since we cannot reliably hot unplug usb devices
4546                 # we are disabling it
4547                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4548                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4549                 die "skip\n" if !$d;
4550                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4551             } elsif ($opt eq 'vcpus') {
4552                 die "skip\n" if !$hotplug_features->{cpu};
4553                 qemu_cpu_hotplug($vmid, $conf, $value);
4554             } elsif ($opt eq 'balloon') {
4555                 # enable/disable balloning device is not hotpluggable
4556                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4557                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4558                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4559
4560                 # allow manual ballooning if shares is set to zero
4561                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4562                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4563                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4564                 }
4565             } elsif ($opt =~ m/^net(\d+)$/) {
4566                 # some changes can be done without hotplug
4567                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4568                                     $vmid, $opt, $value, $arch, $machine_type);
4569             } elsif (is_valid_drivename($opt)) {
4570                 die "skip\n" if $opt eq 'efidisk0';
4571                 # some changes can be done without hotplug
4572                 my $drive = parse_drive($opt, $value);
4573                 if (drive_is_cloudinit($drive)) {
4574                     &$apply_pending_cloudinit($opt, $value);
4575                 }
4576                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4577                                      $vmid, $opt, $value, $arch, $machine_type);
4578             } elsif ($opt =~ m/^memory$/) { #dimms
4579                 die "skip\n" if !$hotplug_features->{memory};
4580                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4581             } elsif ($opt eq 'cpuunits') {
4582                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4583             } elsif ($opt eq 'cpulimit') {
4584                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4585                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4586             } else {
4587                 die "skip\n";  # skip non-hot-pluggable options
4588             }
4589         };
4590         if (my $err = $@) {
4591             &$add_error($opt, $err) if $err ne "skip\n";
4592         } else {
4593             $conf->{$opt} = $value;
4594             delete $conf->{pending}->{$opt};
4595         }
4596     }
4597
4598     PVE::QemuConfig->write_config($vmid, $conf);
4599 }
4600
4601 sub try_deallocate_drive {
4602     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4603
4604     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4605         my $volid = $drive->{file};
4606         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4607             my $sid = PVE::Storage::parse_volume_id($volid);
4608             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4609
4610             # check if the disk is really unused
4611             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4612                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4613             PVE::Storage::vdisk_free($storecfg, $volid);
4614             return 1;
4615         } else {
4616             # If vm is not owner of this disk remove from config
4617             return 1;
4618         }
4619     }
4620
4621     return undef;
4622 }
4623
4624 sub vmconfig_delete_or_detach_drive {
4625     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4626
4627     my $drive = parse_drive($opt, $conf->{$opt});
4628
4629     my $rpcenv = PVE::RPCEnvironment::get();
4630     my $authuser = $rpcenv->get_user();
4631
4632     if ($force) {
4633         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4634         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4635     } else {
4636         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4637     }
4638 }
4639
4640
4641
4642 sub vmconfig_apply_pending {
4643     my ($vmid, $conf, $storecfg, $errors) = @_;
4644
4645     my $add_apply_error = sub {
4646         my ($opt, $msg) = @_;
4647         my $err_msg = "unable to apply pending change $opt : $msg";
4648         $errors->{$opt} = $err_msg;
4649         warn $err_msg;
4650     };
4651
4652     # cold plug
4653
4654     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4655     foreach my $opt (sort keys %$pending_delete_hash) {
4656         my $force = $pending_delete_hash->{$opt}->{force};
4657         eval {
4658             if ($opt =~ m/^unused/) {
4659                 die "internal error";
4660             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4661                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4662             }
4663         };
4664         if (my $err = $@) {
4665             $add_apply_error->($opt, $err);
4666         } else {
4667             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4668             delete $conf->{$opt};
4669         }
4670     }
4671
4672     PVE::QemuConfig->cleanup_pending($conf);
4673
4674     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4675         next if $opt eq 'delete'; # just to be sure
4676         eval {
4677             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4678                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4679             }
4680         };
4681         if (my $err = $@) {
4682             $add_apply_error->($opt, $err);
4683         } else {
4684             $conf->{$opt} = delete $conf->{pending}->{$opt};
4685         }
4686     }
4687
4688     # write all changes at once to avoid unnecessary i/o
4689     PVE::QemuConfig->write_config($vmid, $conf);
4690 }
4691
4692 sub vmconfig_update_net {
4693     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4694
4695     my $newnet = parse_net($value);
4696
4697     if ($conf->{$opt}) {
4698         my $oldnet = parse_net($conf->{$opt});
4699
4700         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4701             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4702             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4703             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4704
4705             # for non online change, we try to hot-unplug
4706             die "skip\n" if !$hotplug;
4707             vm_deviceunplug($vmid, $conf, $opt);
4708         } else {
4709
4710             die "internal error" if $opt !~ m/net(\d+)/;
4711             my $iface = "tap${vmid}i$1";
4712
4713             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4714                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4715                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4716                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4717                 PVE::Network::tap_unplug($iface);
4718
4719                 if ($have_sdn) {
4720                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4721                 } else {
4722                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4723                 }
4724             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4725                 # Rate can be applied on its own but any change above needs to
4726                 # include the rate in tap_plug since OVS resets everything.
4727                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4728             }
4729
4730             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4731                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4732             }
4733
4734             return 1;
4735         }
4736     }
4737
4738     if ($hotplug) {
4739         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4740     } else {
4741         die "skip\n";
4742     }
4743 }
4744
4745 sub vmconfig_update_disk {
4746     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4747
4748     my $drive = parse_drive($opt, $value);
4749
4750     if ($conf->{$opt}) {
4751
4752         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4753
4754             my $media = $drive->{media} || 'disk';
4755             my $oldmedia = $old_drive->{media} || 'disk';
4756             die "unable to change media type\n" if $media ne $oldmedia;
4757
4758             if (!drive_is_cdrom($old_drive)) {
4759
4760                 if ($drive->{file} ne $old_drive->{file}) {
4761
4762                     die "skip\n" if !$hotplug;
4763
4764                     # unplug and register as unused
4765                     vm_deviceunplug($vmid, $conf, $opt);
4766                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4767
4768                 } else {
4769                     # update existing disk
4770
4771                     # skip non hotpluggable value
4772                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4773                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4774                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4775                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4776                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4777                         die "skip\n";
4778                     }
4779
4780                     # apply throttle
4781                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4782                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4783                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4784                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4785                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4786                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4787                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4788                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4789                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4790                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4791                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4792                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4793                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4794                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4795                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4796                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4797                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4798                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4799
4800                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4801                                                    ($drive->{mbps} || 0)*1024*1024,
4802                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4803                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4804                                                    $drive->{iops} || 0,
4805                                                    $drive->{iops_rd} || 0,
4806                                                    $drive->{iops_wr} || 0,
4807                                                    ($drive->{mbps_max} || 0)*1024*1024,
4808                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4809                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4810                                                    $drive->{iops_max} || 0,
4811                                                    $drive->{iops_rd_max} || 0,
4812                                                    $drive->{iops_wr_max} || 0,
4813                                                    $drive->{bps_max_length} || 1,
4814                                                    $drive->{bps_rd_max_length} || 1,
4815                                                    $drive->{bps_wr_max_length} || 1,
4816                                                    $drive->{iops_max_length} || 1,
4817                                                    $drive->{iops_rd_max_length} || 1,
4818                                                    $drive->{iops_wr_max_length} || 1);
4819
4820                     }
4821
4822                     return 1;
4823                 }
4824
4825             } else { # cdrom
4826
4827                 if ($drive->{file} eq 'none') {
4828                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4829                     if (drive_is_cloudinit($old_drive)) {
4830                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4831                     }
4832                 } else {
4833                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4834
4835                     # force eject if locked
4836                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4837
4838                     if ($path) {
4839                         mon_cmd($vmid, "blockdev-change-medium",
4840                             id => "$opt", filename => "$path");
4841                     }
4842                 }
4843
4844                 return 1;
4845             }
4846         }
4847     }
4848
4849     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4850     # hotplug new disks
4851     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4852     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4853 }
4854
4855 # called in locked context by incoming migration
4856 sub vm_migrate_get_nbd_disks {
4857     my ($storecfg, $conf, $replicated_volumes) = @_;
4858
4859     my $local_volumes = {};
4860     PVE::QemuConfig->foreach_volume($conf, sub {
4861         my ($ds, $drive) = @_;
4862
4863         return if drive_is_cdrom($drive);
4864
4865         my $volid = $drive->{file};
4866
4867         return if !$volid;
4868
4869         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4870
4871         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4872         return if $scfg->{shared};
4873
4874         # replicated disks re-use existing state via bitmap
4875         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4876         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4877     });
4878     return $local_volumes;
4879 }
4880
4881 # called in locked context by incoming migration
4882 sub vm_migrate_alloc_nbd_disks {
4883     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4884
4885     my $format = undef;
4886
4887     my $nbd = {};
4888     foreach my $opt (sort keys %$source_volumes) {
4889         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4890
4891         if ($use_existing) {
4892             $nbd->{$opt}->{drivestr} = print_drive($drive);
4893             $nbd->{$opt}->{volid} = $volid;
4894             $nbd->{$opt}->{replicated} = 1;
4895             next;
4896         }
4897
4898         # If a remote storage is specified and the format of the original
4899         # volume is not available there, fall back to the default format.
4900         # Otherwise use the same format as the original.
4901         if (!$storagemap->{identity}) {
4902             $storeid = map_storage($storagemap, $storeid);
4903             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4904             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4905             my $fileFormat = qemu_img_format($scfg, $volname);
4906             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4907         } else {
4908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4909             $format = qemu_img_format($scfg, $volname);
4910         }
4911
4912         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4913         my $newdrive = $drive;
4914         $newdrive->{format} = $format;
4915         $newdrive->{file} = $newvolid;
4916         my $drivestr = print_drive($newdrive);
4917         $nbd->{$opt}->{drivestr} = $drivestr;
4918         $nbd->{$opt}->{volid} = $newvolid;
4919     }
4920
4921     return $nbd;
4922 }
4923
4924 # see vm_start_nolock for parameters, additionally:
4925 # migrate_opts:
4926 #   storagemap = parsed storage map for allocating NBD disks
4927 sub vm_start {
4928     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4929
4930     return PVE::QemuConfig->lock_config($vmid, sub {
4931         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4932
4933         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4934
4935         $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended');
4936
4937         PVE::QemuConfig->check_lock($conf)
4938             if !($params->{skiplock} || $params->{resume});
4939
4940         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4941
4942         if (my $storagemap = $migrate_opts->{storagemap}) {
4943             my $replicated = $migrate_opts->{replicated_volumes};
4944             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4945             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4946
4947             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4948                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4949             }
4950         }
4951
4952         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4953     });
4954 }
4955
4956
4957 # params:
4958 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4959 #   skiplock => 0/1, skip checking for config lock
4960 #   forcemachine => to force Qemu machine (rollback/migration)
4961 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4962 #   timeout => in seconds
4963 #   paused => start VM in paused state (backup)
4964 #   resume => resume from hibernation
4965 # migrate_opts:
4966 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4967 #   migratedfrom => source node
4968 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4969 #   network => CIDR of migration network
4970 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4971 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4972 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4973 sub vm_start_nolock {
4974     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4975
4976     my $statefile = $params->{statefile};
4977     my $resume = $params->{resume};
4978
4979     my $migratedfrom = $migrate_opts->{migratedfrom};
4980     my $migration_type = $migrate_opts->{type};
4981
4982     my $res = {};
4983
4984     # clean up leftover reboot request files
4985     eval { clear_reboot_request($vmid); };
4986     warn $@ if $@;
4987
4988     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4989         vmconfig_apply_pending($vmid, $conf, $storecfg);
4990         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4991     }
4992
4993     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4994
4995     my $defaults = load_defaults();
4996
4997     # set environment variable useful inside network script
4998     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4999
5000     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5001
5002     my $forcemachine = $params->{forcemachine};
5003     my $forcecpu = $params->{forcecpu};
5004     if ($resume) {
5005         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5006         $forcemachine = $conf->{runningmachine};
5007         $forcecpu = $conf->{runningcpu};
5008         print "Resuming suspended VM\n";
5009     }
5010
5011     my ($cmd, $vollist, $spice_port) =
5012         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5013
5014     my $migration_ip;
5015     my $get_migration_ip = sub {
5016         my ($nodename) = @_;
5017
5018         return $migration_ip if defined($migration_ip);
5019
5020         my $cidr = $migrate_opts->{network};
5021
5022         if (!defined($cidr)) {
5023             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5024             $cidr = $dc_conf->{migration}->{network};
5025         }
5026
5027         if (defined($cidr)) {
5028             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5029
5030             die "could not get IP: no address configured on local " .
5031                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5032
5033             die "could not get IP: multiple addresses configured on local " .
5034                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5035
5036             $migration_ip = @$ips[0];
5037         }
5038
5039         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5040             if !defined($migration_ip);
5041
5042         return $migration_ip;
5043     };
5044
5045     my $migrate_uri;
5046     if ($statefile) {
5047         if ($statefile eq 'tcp') {
5048             my $localip = "localhost";
5049             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5050             my $nodename = nodename();
5051
5052             if (!defined($migration_type)) {
5053                 if (defined($datacenterconf->{migration}->{type})) {
5054                     $migration_type = $datacenterconf->{migration}->{type};
5055                 } else {
5056                     $migration_type = 'secure';
5057                 }
5058             }
5059
5060             if ($migration_type eq 'insecure') {
5061                 $localip = $get_migration_ip->($nodename);
5062                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5063             }
5064
5065             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5066             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5067             $migrate_uri = "tcp:${localip}:${migrate_port}";
5068             push @$cmd, '-incoming', $migrate_uri;
5069             push @$cmd, '-S';
5070
5071         } elsif ($statefile eq 'unix') {
5072             # should be default for secure migrations as a ssh TCP forward
5073             # tunnel is not deterministic reliable ready and fails regurarly
5074             # to set up in time, so use UNIX socket forwards
5075             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5076             unlink $socket_addr;
5077
5078             $migrate_uri = "unix:$socket_addr";
5079
5080             push @$cmd, '-incoming', $migrate_uri;
5081             push @$cmd, '-S';
5082
5083         } elsif (-e $statefile) {
5084             push @$cmd, '-loadstate', $statefile;
5085         } else {
5086             my $statepath = PVE::Storage::path($storecfg, $statefile);
5087             push @$vollist, $statefile;
5088             push @$cmd, '-loadstate', $statepath;
5089         }
5090     } elsif ($params->{paused}) {
5091         push @$cmd, '-S';
5092     }
5093
5094     # host pci devices
5095     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5096       my $d = parse_hostpci($conf->{"hostpci$i"});
5097       next if !$d;
5098       my $pcidevices = $d->{pciid};
5099       foreach my $pcidevice (@$pcidevices) {
5100             my $pciid = $pcidevice->{id};
5101
5102             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5103             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5104             die "no pci device info for device '$pciid'\n" if !$info;
5105
5106             if ($d->{mdev}) {
5107                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5108                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5109             } else {
5110                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5111                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5112                 die "can't reset pci device '$pciid'\n"
5113                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5114             }
5115       }
5116     }
5117
5118     PVE::Storage::activate_volumes($storecfg, $vollist);
5119
5120     eval {
5121         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5122             outfunc => sub {}, errfunc => sub {});
5123     };
5124     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5125     # timeout should be more than enough here...
5126     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5127
5128     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5129                                               : $defaults->{cpuunits};
5130
5131     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5132     my %run_params = (
5133         timeout => $statefile ? undef : $start_timeout,
5134         umask => 0077,
5135         noerr => 1,
5136     );
5137
5138     # when migrating, prefix QEMU output so other side can pick up any
5139     # errors that might occur and show the user
5140     if ($migratedfrom) {
5141         $run_params{quiet} = 1;
5142         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5143     }
5144
5145     my %properties = (
5146         Slice => 'qemu.slice',
5147         KillMode => 'none',
5148         CPUShares => $cpuunits
5149     );
5150
5151     if (my $cpulimit = $conf->{cpulimit}) {
5152         $properties{CPUQuota} = int($cpulimit * 100);
5153     }
5154     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5155
5156     my $run_qemu = sub {
5157         PVE::Tools::run_fork sub {
5158             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5159
5160             my $exitcode = run_command($cmd, %run_params);
5161             die "QEMU exited with code $exitcode\n" if $exitcode;
5162         };
5163     };
5164
5165     if ($conf->{hugepages}) {
5166
5167         my $code = sub {
5168             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5169             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5170
5171             PVE::QemuServer::Memory::hugepages_mount();
5172             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5173
5174             eval { $run_qemu->() };
5175             if (my $err = $@) {
5176                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5177                 die $err;
5178             }
5179
5180             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5181         };
5182         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5183
5184     } else {
5185         eval { $run_qemu->() };
5186     }
5187
5188     if (my $err = $@) {
5189         # deactivate volumes if start fails
5190         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5191         die "start failed: $err";
5192     }
5193
5194     print "migration listens on $migrate_uri\n" if $migrate_uri;
5195     $res->{migrate_uri} = $migrate_uri;
5196
5197     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5198         eval { mon_cmd($vmid, "cont"); };
5199         warn $@ if $@;
5200     }
5201
5202     #start nbd server for storage migration
5203     if (my $nbd = $migrate_opts->{nbd}) {
5204         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5205
5206         my $migrate_storage_uri;
5207         # nbd_protocol_version > 0 for unix socket support
5208         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5209             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5210             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5211             $migrate_storage_uri = "nbd:unix:$socket_path";
5212         } else {
5213             my $nodename = nodename();
5214             my $localip = $get_migration_ip->($nodename);
5215             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5216             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5217
5218             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5219             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5220             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5221         }
5222
5223         $res->{migrate_storage_uri} = $migrate_storage_uri;
5224
5225         foreach my $opt (sort keys %$nbd) {
5226             my $drivestr = $nbd->{$opt}->{drivestr};
5227             my $volid = $nbd->{$opt}->{volid};
5228             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5229             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5230             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5231             print "re-using replicated volume: $opt - $volid\n"
5232                 if $nbd->{$opt}->{replicated};
5233
5234             $res->{drives}->{$opt} = $nbd->{$opt};
5235             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5236         }
5237     }
5238
5239     if ($migratedfrom) {
5240         eval {
5241             set_migration_caps($vmid);
5242         };
5243         warn $@ if $@;
5244
5245         if ($spice_port) {
5246             print "spice listens on port $spice_port\n";
5247             $res->{spice_port} = $spice_port;
5248             if ($migrate_opts->{spice_ticket}) {
5249                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5250                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5251             }
5252         }
5253
5254     } else {
5255         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5256             if !$statefile && $conf->{balloon};
5257
5258         foreach my $opt (keys %$conf) {
5259             next if $opt !~  m/^net\d+$/;
5260             my $nicconf = parse_net($conf->{$opt});
5261             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5262         }
5263     }
5264
5265     mon_cmd($vmid, 'qom-set',
5266                 path => "machine/peripheral/balloon0",
5267                 property => "guest-stats-polling-interval",
5268                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5269
5270     if ($resume) {
5271         print "Resumed VM, removing state\n";
5272         if (my $vmstate = $conf->{vmstate}) {
5273             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5274             PVE::Storage::vdisk_free($storecfg, $vmstate);
5275         }
5276         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5277         PVE::QemuConfig->write_config($vmid, $conf);
5278     }
5279
5280     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5281
5282     return $res;
5283 }
5284
5285 sub vm_commandline {
5286     my ($storecfg, $vmid, $snapname) = @_;
5287
5288     my $conf = PVE::QemuConfig->load_config($vmid);
5289     my $forcemachine;
5290     my $forcecpu;
5291
5292     if ($snapname) {
5293         my $snapshot = $conf->{snapshots}->{$snapname};
5294         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5295
5296         # check for machine or CPU overrides in snapshot
5297         $forcemachine = $snapshot->{runningmachine};
5298         $forcecpu = $snapshot->{runningcpu};
5299
5300         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5301
5302         $conf = $snapshot;
5303     }
5304
5305     my $defaults = load_defaults();
5306
5307     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5308         $forcemachine, $forcecpu);
5309
5310     return PVE::Tools::cmd2string($cmd);
5311 }
5312
5313 sub vm_reset {
5314     my ($vmid, $skiplock) = @_;
5315
5316     PVE::QemuConfig->lock_config($vmid, sub {
5317
5318         my $conf = PVE::QemuConfig->load_config($vmid);
5319
5320         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5321
5322         mon_cmd($vmid, "system_reset");
5323     });
5324 }
5325
5326 sub get_vm_volumes {
5327     my ($conf) = @_;
5328
5329     my $vollist = [];
5330     foreach_volid($conf, sub {
5331         my ($volid, $attr) = @_;
5332
5333         return if $volid =~ m|^/|;
5334
5335         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5336         return if !$sid;
5337
5338         push @$vollist, $volid;
5339     });
5340
5341     return $vollist;
5342 }
5343
5344 sub vm_stop_cleanup {
5345     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5346
5347     eval {
5348
5349         if (!$keepActive) {
5350             my $vollist = get_vm_volumes($conf);
5351             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5352         }
5353
5354         foreach my $ext (qw(mon qmp pid vnc qga)) {
5355             unlink "/var/run/qemu-server/${vmid}.$ext";
5356         }
5357
5358         if ($conf->{ivshmem}) {
5359             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5360             # just delete it for now, VMs which have this already open do not
5361             # are affected, but new VMs will get a separated one. If this
5362             # becomes an issue we either add some sort of ref-counting or just
5363             # add a "don't delete on stop" flag to the ivshmem format.
5364             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5365         }
5366
5367         foreach my $key (keys %$conf) {
5368             next if $key !~ m/^hostpci(\d+)$/;
5369             my $hostpciindex = $1;
5370             my $d = parse_hostpci($conf->{$key});
5371             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5372
5373             foreach my $pci (@{$d->{pciid}}) {
5374                 my $pciid = $pci->{id};
5375                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5376             }
5377         }
5378
5379         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5380     };
5381     warn $@ if $@; # avoid errors - just warn
5382 }
5383
5384 # call only in locked context
5385 sub _do_vm_stop {
5386     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5387
5388     my $pid = check_running($vmid, $nocheck);
5389     return if !$pid;
5390
5391     my $conf;
5392     if (!$nocheck) {
5393         $conf = PVE::QemuConfig->load_config($vmid);
5394         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5395         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5396             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5397             $timeout = $opts->{down} if $opts->{down};
5398         }
5399         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5400     }
5401
5402     eval {
5403         if ($shutdown) {
5404             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5405                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5406             } else {
5407                 mon_cmd($vmid, "system_powerdown");
5408             }
5409         } else {
5410             mon_cmd($vmid, "quit");
5411         }
5412     };
5413     my $err = $@;
5414
5415     if (!$err) {
5416         $timeout = 60 if !defined($timeout);
5417
5418         my $count = 0;
5419         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5420             $count++;
5421             sleep 1;
5422         }
5423
5424         if ($count >= $timeout) {
5425             if ($force) {
5426                 warn "VM still running - terminating now with SIGTERM\n";
5427                 kill 15, $pid;
5428             } else {
5429                 die "VM quit/powerdown failed - got timeout\n";
5430             }
5431         } else {
5432             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5433             return;
5434         }
5435     } else {
5436        if (!check_running($vmid, $nocheck)) {
5437             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5438             return;
5439        }
5440        if ($force) {
5441             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5442             kill 15, $pid;
5443         } else {
5444             die "VM quit/powerdown failed\n";
5445         }
5446     }
5447
5448     # wait again
5449     $timeout = 10;
5450
5451     my $count = 0;
5452     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5453         $count++;
5454         sleep 1;
5455     }
5456
5457     if ($count >= $timeout) {
5458         warn "VM still running - terminating now with SIGKILL\n";
5459         kill 9, $pid;
5460         sleep 1;
5461     }
5462
5463     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5464 }
5465
5466 # Note: use $nocheck to skip tests if VM configuration file exists.
5467 # We need that when migration VMs to other nodes (files already moved)
5468 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5469 sub vm_stop {
5470     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5471
5472     $force = 1 if !defined($force) && !$shutdown;
5473
5474     if ($migratedfrom){
5475         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5476         kill 15, $pid if $pid;
5477         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5478         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5479         return;
5480     }
5481
5482     PVE::QemuConfig->lock_config($vmid, sub {
5483         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5484    });
5485 }
5486
5487 sub vm_reboot {
5488     my ($vmid, $timeout) = @_;
5489
5490     PVE::QemuConfig->lock_config($vmid, sub {
5491         eval {
5492
5493             # only reboot if running, as qmeventd starts it again on a stop event
5494             return if !check_running($vmid);
5495
5496             create_reboot_request($vmid);
5497
5498             my $storecfg = PVE::Storage::config();
5499             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5500
5501         };
5502         if (my $err = $@) {
5503             # avoid that the next normal shutdown will be confused for a reboot
5504             clear_reboot_request($vmid);
5505             die $err;
5506         }
5507    });
5508 }
5509
5510 # note: if using the statestorage parameter, the caller has to check privileges
5511 sub vm_suspend {
5512     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5513
5514     my $conf;
5515     my $path;
5516     my $storecfg;
5517     my $vmstate;
5518
5519     PVE::QemuConfig->lock_config($vmid, sub {
5520
5521         $conf = PVE::QemuConfig->load_config($vmid);
5522
5523         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5524         PVE::QemuConfig->check_lock($conf)
5525             if !($skiplock || $is_backing_up);
5526
5527         die "cannot suspend to disk during backup\n"
5528             if $is_backing_up && $includestate;
5529
5530         if ($includestate) {
5531             $conf->{lock} = 'suspending';
5532             my $date = strftime("%Y-%m-%d", localtime(time()));
5533             $storecfg = PVE::Storage::config();
5534             if (!$statestorage) {
5535                 $statestorage = find_vmstate_storage($conf, $storecfg);
5536                 # check permissions for the storage
5537                 my $rpcenv = PVE::RPCEnvironment::get();
5538                 if ($rpcenv->{type} ne 'cli') {
5539                     my $authuser = $rpcenv->get_user();
5540                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5541                 }
5542             }
5543
5544
5545             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5546             $path = PVE::Storage::path($storecfg, $vmstate);
5547             PVE::QemuConfig->write_config($vmid, $conf);
5548         } else {
5549             mon_cmd($vmid, "stop");
5550         }
5551     });
5552
5553     if ($includestate) {
5554         # save vm state
5555         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5556
5557         eval {
5558             mon_cmd($vmid, "savevm-start", statefile => $path);
5559             for(;;) {
5560                 my $state = mon_cmd($vmid, "query-savevm");
5561                 if (!$state->{status}) {
5562                     die "savevm not active\n";
5563                 } elsif ($state->{status} eq 'active') {
5564                     sleep(1);
5565                     next;
5566                 } elsif ($state->{status} eq 'completed') {
5567                     print "State saved, quitting\n";
5568                     last;
5569                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5570                     die "query-savevm failed with error '$state->{error}'\n"
5571                 } else {
5572                     die "query-savevm returned status '$state->{status}'\n";
5573                 }
5574             }
5575         };
5576         my $err = $@;
5577
5578         PVE::QemuConfig->lock_config($vmid, sub {
5579             $conf = PVE::QemuConfig->load_config($vmid);
5580             if ($err) {
5581                 # cleanup, but leave suspending lock, to indicate something went wrong
5582                 eval {
5583                     mon_cmd($vmid, "savevm-end");
5584                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5585                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5586                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5587                     PVE::QemuConfig->write_config($vmid, $conf);
5588                 };
5589                 warn $@ if $@;
5590                 die $err;
5591             }
5592
5593             die "lock changed unexpectedly\n"
5594                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5595
5596             mon_cmd($vmid, "quit");
5597             $conf->{lock} = 'suspended';
5598             PVE::QemuConfig->write_config($vmid, $conf);
5599         });
5600     }
5601 }
5602
5603 sub vm_resume {
5604     my ($vmid, $skiplock, $nocheck) = @_;
5605
5606     PVE::QemuConfig->lock_config($vmid, sub {
5607         my $res = mon_cmd($vmid, 'query-status');
5608         my $resume_cmd = 'cont';
5609
5610         if ($res->{status} && $res->{status} eq 'suspended') {
5611             $resume_cmd = 'system_wakeup';
5612         }
5613
5614         if (!$nocheck) {
5615
5616             my $conf = PVE::QemuConfig->load_config($vmid);
5617
5618             PVE::QemuConfig->check_lock($conf)
5619                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5620         }
5621
5622         mon_cmd($vmid, $resume_cmd);
5623     });
5624 }
5625
5626 sub vm_sendkey {
5627     my ($vmid, $skiplock, $key) = @_;
5628
5629     PVE::QemuConfig->lock_config($vmid, sub {
5630
5631         my $conf = PVE::QemuConfig->load_config($vmid);
5632
5633         # there is no qmp command, so we use the human monitor command
5634         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5635         die $res if $res ne '';
5636     });
5637 }
5638
5639 # vzdump restore implementaion
5640
5641 sub tar_archive_read_firstfile {
5642     my $archive = shift;
5643
5644     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5645
5646     # try to detect archive type first
5647     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5648         die "unable to open file '$archive'\n";
5649     my $firstfile = <$fh>;
5650     kill 15, $pid;
5651     close $fh;
5652
5653     die "ERROR: archive contaions no data\n" if !$firstfile;
5654     chomp $firstfile;
5655
5656     return $firstfile;
5657 }
5658
5659 sub tar_restore_cleanup {
5660     my ($storecfg, $statfile) = @_;
5661
5662     print STDERR "starting cleanup\n";
5663
5664     if (my $fd = IO::File->new($statfile, "r")) {
5665         while (defined(my $line = <$fd>)) {
5666             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5667                 my $volid = $2;
5668                 eval {
5669                     if ($volid =~ m|^/|) {
5670                         unlink $volid || die 'unlink failed\n';
5671                     } else {
5672                         PVE::Storage::vdisk_free($storecfg, $volid);
5673                     }
5674                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5675                 };
5676                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5677             } else {
5678                 print STDERR "unable to parse line in statfile - $line";
5679             }
5680         }
5681         $fd->close();
5682     }
5683 }
5684
5685 sub restore_file_archive {
5686     my ($archive, $vmid, $user, $opts) = @_;
5687
5688     return restore_vma_archive($archive, $vmid, $user, $opts)
5689         if $archive eq '-';
5690
5691     my $info = PVE::Storage::archive_info($archive);
5692     my $format = $opts->{format} // $info->{format};
5693     my $comp = $info->{compression};
5694
5695     # try to detect archive format
5696     if ($format eq 'tar') {
5697         return restore_tar_archive($archive, $vmid, $user, $opts);
5698     } else {
5699         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5700     }
5701 }
5702
5703 # hepler to remove disks that will not be used after restore
5704 my $restore_cleanup_oldconf = sub {
5705     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5706
5707     PVE::QemuConfig->foreach_volume($oldconf, sub {
5708         my ($ds, $drive) = @_;
5709
5710         return if drive_is_cdrom($drive, 1);
5711
5712         my $volid = $drive->{file};
5713         return if !$volid || $volid =~ m|^/|;
5714
5715         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5716         return if !$path || !$owner || ($owner != $vmid);
5717
5718         # Note: only delete disk we want to restore
5719         # other volumes will become unused
5720         if ($virtdev_hash->{$ds}) {
5721             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5722             if (my $err = $@) {
5723                 warn $err;
5724             }
5725         }
5726     });
5727
5728     # delete vmstate files, after the restore we have no snapshots anymore
5729     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5730         my $snap = $oldconf->{snapshots}->{$snapname};
5731         if ($snap->{vmstate}) {
5732             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5733             if (my $err = $@) {
5734                 warn $err;
5735             }
5736         }
5737     }
5738 };
5739
5740 # Helper to parse vzdump backup device hints
5741 #
5742 # $rpcenv: Environment, used to ckeck storage permissions
5743 # $user: User ID, to check storage permissions
5744 # $storecfg: Storage configuration
5745 # $fh: the file handle for reading the configuration
5746 # $devinfo: should contain device sizes for all backu-up'ed devices
5747 # $options: backup options (pool, default storage)
5748 #
5749 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5750 my $parse_backup_hints = sub {
5751     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5752
5753     my $virtdev_hash = {};
5754
5755     while (defined(my $line = <$fh>)) {
5756         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5757             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5758             die "archive does not contain data for drive '$virtdev'\n"
5759                 if !$devinfo->{$devname};
5760
5761             if (defined($options->{storage})) {
5762                 $storeid = $options->{storage} || 'local';
5763             } elsif (!$storeid) {
5764                 $storeid = 'local';
5765             }
5766             $format = 'raw' if !$format;
5767             $devinfo->{$devname}->{devname} = $devname;
5768             $devinfo->{$devname}->{virtdev} = $virtdev;
5769             $devinfo->{$devname}->{format} = $format;
5770             $devinfo->{$devname}->{storeid} = $storeid;
5771
5772             # check permission on storage
5773             my $pool = $options->{pool}; # todo: do we need that?
5774             if ($user ne 'root@pam') {
5775                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5776             }
5777
5778             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5779         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5780             my $virtdev = $1;
5781             my $drive = parse_drive($virtdev, $2);
5782             if (drive_is_cloudinit($drive)) {
5783                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5784                 $storeid = $options->{storage} if defined ($options->{storage});
5785                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5786                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5787
5788                 $virtdev_hash->{$virtdev} = {
5789                     format => $format,
5790                     storeid => $storeid,
5791                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5792                     is_cloudinit => 1,
5793                 };
5794             }
5795         }
5796     }
5797
5798     return $virtdev_hash;
5799 };
5800
5801 # Helper to allocate and activate all volumes required for a restore
5802 #
5803 # $storecfg: Storage configuration
5804 # $virtdev_hash: as returned by parse_backup_hints()
5805 #
5806 # Returns: { $virtdev => $volid }
5807 my $restore_allocate_devices = sub {
5808     my ($storecfg, $virtdev_hash, $vmid) = @_;
5809
5810     my $map = {};
5811     foreach my $virtdev (sort keys %$virtdev_hash) {
5812         my $d = $virtdev_hash->{$virtdev};
5813         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5814         my $storeid = $d->{storeid};
5815         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5816
5817         # test if requested format is supported
5818         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5819         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5820         $d->{format} = $defFormat if !$supported;
5821
5822         my $name;
5823         if ($d->{is_cloudinit}) {
5824             $name = "vm-$vmid-cloudinit";
5825             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5826         }
5827
5828         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5829
5830         print STDERR "new volume ID is '$volid'\n";
5831         $d->{volid} = $volid;
5832
5833         PVE::Storage::activate_volumes($storecfg, [$volid]);
5834
5835         $map->{$virtdev} = $volid;
5836     }
5837
5838     return $map;
5839 };
5840
5841 my $restore_update_config_line = sub {
5842     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5843
5844     return if $line =~ m/^\#qmdump\#/;
5845     return if $line =~ m/^\#vzdump\#/;
5846     return if $line =~ m/^lock:/;
5847     return if $line =~ m/^unused\d+:/;
5848     return if $line =~ m/^parent:/;
5849
5850     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5851     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5852         # try to convert old 1.X settings
5853         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5854         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5855             my ($model, $macaddr) = split(/\=/, $devconfig);
5856             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5857             my $net = {
5858                 model => $model,
5859                 bridge => "vmbr$ind",
5860                 macaddr => $macaddr,
5861             };
5862             my $netstr = print_net($net);
5863
5864             print $outfd "net$cookie->{netcount}: $netstr\n";
5865             $cookie->{netcount}++;
5866         }
5867     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5868         my ($id, $netstr) = ($1, $2);
5869         my $net = parse_net($netstr);
5870         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5871         $netstr = print_net($net);
5872         print $outfd "$id: $netstr\n";
5873     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5874         my $virtdev = $1;
5875         my $value = $3;
5876         my $di = parse_drive($virtdev, $value);
5877         if (defined($di->{backup}) && !$di->{backup}) {
5878             print $outfd "#$line";
5879         } elsif ($map->{$virtdev}) {
5880             delete $di->{format}; # format can change on restore
5881             $di->{file} = $map->{$virtdev};
5882             $value = print_drive($di);
5883             print $outfd "$virtdev: $value\n";
5884         } else {
5885             print $outfd $line;
5886         }
5887     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5888         my $vmgenid = $1;
5889         if ($vmgenid ne '0') {
5890             # always generate a new vmgenid if there was a valid one setup
5891             $vmgenid = generate_uuid();
5892         }
5893         print $outfd "vmgenid: $vmgenid\n";
5894     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5895         my ($uuid, $uuid_str);
5896         UUID::generate($uuid);
5897         UUID::unparse($uuid, $uuid_str);
5898         my $smbios1 = parse_smbios1($2);
5899         $smbios1->{uuid} = $uuid_str;
5900         print $outfd $1.print_smbios1($smbios1)."\n";
5901     } else {
5902         print $outfd $line;
5903     }
5904 };
5905
5906 my $restore_deactivate_volumes = sub {
5907     my ($storecfg, $devinfo) = @_;
5908
5909     my $vollist = [];
5910     foreach my $devname (keys %$devinfo) {
5911         my $volid = $devinfo->{$devname}->{volid};
5912         push @$vollist, $volid if $volid;
5913     }
5914
5915     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5916 };
5917
5918 my $restore_destroy_volumes = sub {
5919     my ($storecfg, $devinfo) = @_;
5920
5921     foreach my $devname (keys %$devinfo) {
5922         my $volid = $devinfo->{$devname}->{volid};
5923         next if !$volid;
5924         eval {
5925             if ($volid =~ m|^/|) {
5926                 unlink $volid || die 'unlink failed\n';
5927             } else {
5928                 PVE::Storage::vdisk_free($storecfg, $volid);
5929             }
5930             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5931         };
5932         print STDERR "unable to cleanup '$volid' - $@" if $@;
5933     }
5934 };
5935
5936 sub scan_volids {
5937     my ($cfg, $vmid) = @_;
5938
5939     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5940
5941     my $volid_hash = {};
5942     foreach my $storeid (keys %$info) {
5943         foreach my $item (@{$info->{$storeid}}) {
5944             next if !($item->{volid} && $item->{size});
5945             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5946             $volid_hash->{$item->{volid}} = $item;
5947         }
5948     }
5949
5950     return $volid_hash;
5951 }
5952
5953 sub update_disk_config {
5954     my ($vmid, $conf, $volid_hash) = @_;
5955
5956     my $changes;
5957     my $prefix = "VM $vmid:";
5958
5959     # used and unused disks
5960     my $referenced = {};
5961
5962     # Note: it is allowed to define multiple storages with same path (alias), so
5963     # we need to check both 'volid' and real 'path' (two different volid can point
5964     # to the same path).
5965
5966     my $referencedpath = {};
5967
5968     # update size info
5969     PVE::QemuConfig->foreach_volume($conf, sub {
5970         my ($opt, $drive) = @_;
5971
5972         my $volid = $drive->{file};
5973         return if !$volid;
5974
5975         # mark volid as "in-use" for next step
5976         $referenced->{$volid} = 1;
5977         if ($volid_hash->{$volid} &&
5978             (my $path = $volid_hash->{$volid}->{path})) {
5979             $referencedpath->{$path} = 1;
5980         }
5981
5982         return if drive_is_cdrom($drive);
5983         return if !$volid_hash->{$volid};
5984
5985         my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5986         if (defined($updated)) {
5987             $changes = 1;
5988             $conf->{$opt} = print_drive($updated);
5989             print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5990         }
5991     });
5992
5993     # remove 'unusedX' entry if volume is used
5994     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5995         my ($opt, $drive) = @_;
5996
5997         my $volid = $drive->{file};
5998         return if !$volid;
5999
6000         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6001         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6002             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6003             $changes = 1;
6004             delete $conf->{$opt};
6005         }
6006
6007         $referenced->{$volid} = 1;
6008         $referencedpath->{$path} = 1 if $path;
6009     });
6010
6011     foreach my $volid (sort keys %$volid_hash) {
6012         next if $volid =~ m/vm-$vmid-state-/;
6013         next if $referenced->{$volid};
6014         my $path = $volid_hash->{$volid}->{path};
6015         next if !$path; # just to be sure
6016         next if $referencedpath->{$path};
6017         $changes = 1;
6018         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6019         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6020         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6021     }
6022
6023     return $changes;
6024 }
6025
6026 sub rescan {
6027     my ($vmid, $nolock, $dryrun) = @_;
6028
6029     my $cfg = PVE::Storage::config();
6030
6031     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6032     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6033     foreach my $stor (keys %{$cfg->{ids}}) {
6034         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6035     }
6036
6037     print "rescan volumes...\n";
6038     my $volid_hash = scan_volids($cfg, $vmid);
6039
6040     my $updatefn =  sub {
6041         my ($vmid) = @_;
6042
6043         my $conf = PVE::QemuConfig->load_config($vmid);
6044
6045         PVE::QemuConfig->check_lock($conf);
6046
6047         my $vm_volids = {};
6048         foreach my $volid (keys %$volid_hash) {
6049             my $info = $volid_hash->{$volid};
6050             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6051         }
6052
6053         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6054
6055         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6056     };
6057
6058     if (defined($vmid)) {
6059         if ($nolock) {
6060             &$updatefn($vmid);
6061         } else {
6062             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6063         }
6064     } else {
6065         my $vmlist = config_list();
6066         foreach my $vmid (keys %$vmlist) {
6067             if ($nolock) {
6068                 &$updatefn($vmid);
6069             } else {
6070                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6071             }
6072         }
6073     }
6074 }
6075
6076 sub restore_proxmox_backup_archive {
6077     my ($archive, $vmid, $user, $options) = @_;
6078
6079     my $storecfg = PVE::Storage::config();
6080
6081     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6082     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6083
6084     my $server = $scfg->{server};
6085     my $datastore = $scfg->{datastore};
6086     my $username = $scfg->{username} // 'root@pam';
6087     my $fingerprint = $scfg->{fingerprint};
6088
6089     my $repo = "$username\@$server:$datastore";
6090     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6091     local $ENV{PBS_PASSWORD} = $password;
6092     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6093
6094     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6095         PVE::Storage::parse_volname($storecfg, $archive);
6096
6097     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6098
6099     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6100
6101     my $tmpdir = "/var/tmp/vzdumptmp$$";
6102     rmtree $tmpdir;
6103     mkpath $tmpdir;
6104
6105     my $conffile = PVE::QemuConfig->config_file($vmid);
6106     my $tmpfn = "$conffile.$$.tmp";
6107      # disable interrupts (always do cleanups)
6108     local $SIG{INT} =
6109         local $SIG{TERM} =
6110         local $SIG{QUIT} =
6111         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6112
6113     # Note: $oldconf is undef if VM does not exists
6114     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6115     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6116
6117     my $rpcenv = PVE::RPCEnvironment::get();
6118     my $devinfo = {};
6119
6120     eval {
6121         # enable interrupts
6122         local $SIG{INT} =
6123             local $SIG{TERM} =
6124             local $SIG{QUIT} =
6125             local $SIG{HUP} =
6126             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6127
6128         my $cfgfn = "$tmpdir/qemu-server.conf";
6129         my $firewall_config_fn = "$tmpdir/fw.conf";
6130         my $index_fn = "$tmpdir/index.json";
6131
6132         my $cmd = "restore";
6133
6134         my $param = [$pbs_backup_name, "index.json", $index_fn];
6135         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6136         my $index = PVE::Tools::file_get_contents($index_fn);
6137         $index = decode_json($index);
6138
6139         # print Dumper($index);
6140         foreach my $info (@{$index->{files}}) {
6141             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6142                 my $devname = $1;
6143                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6144                     $devinfo->{$devname}->{size} = $1;
6145                 } else {
6146                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6147                 }
6148             }
6149         }
6150
6151         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6152         if (!$is_qemu_server_backup) {
6153             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6154         }
6155         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6156
6157         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6158         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6159
6160         if ($has_firewall_config) {
6161             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6162             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6163
6164             my $pve_firewall_dir = '/etc/pve/firewall';
6165             mkdir $pve_firewall_dir; # make sure the dir exists
6166             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6167         }
6168
6169         my $fh = IO::File->new($cfgfn, "r") ||
6170             "unable to read qemu-server.conf - $!\n";
6171
6172         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6173
6174         # fixme: rate limit?
6175
6176         # create empty/temp config
6177         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6178
6179         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6180
6181         # allocate volumes
6182         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6183
6184         foreach my $virtdev (sort keys %$virtdev_hash) {
6185             my $d = $virtdev_hash->{$virtdev};
6186             next if $d->{is_cloudinit}; # no need to restore cloudinit
6187
6188             my $volid = $d->{volid};
6189
6190             my $path = PVE::Storage::path($storecfg, $volid);
6191
6192             my $pbs_restore_cmd = [
6193                 '/usr/bin/pbs-restore',
6194                 '--repository', $repo,
6195                 $pbs_backup_name,
6196                 "$d->{devname}.img.fidx",
6197                 $path,
6198                 '--verbose',
6199                 ];
6200
6201             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6202
6203             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6204                 push @$pbs_restore_cmd, '--skip-zero';
6205             }
6206
6207             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6208             print "restore proxmox backup image: $dbg_cmdstring\n";
6209             run_command($pbs_restore_cmd);
6210         }
6211
6212         $fh->seek(0, 0) || die "seek failed - $!\n";
6213
6214         my $outfd = new IO::File ($tmpfn, "w") ||
6215             die "unable to write config for VM $vmid\n";
6216
6217         my $cookie = { netcount => 0 };
6218         while (defined(my $line = <$fh>)) {
6219             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6220         }
6221
6222         $fh->close();
6223         $outfd->close();
6224     };
6225     my $err = $@;
6226
6227     $restore_deactivate_volumes->($storecfg, $devinfo);
6228
6229     rmtree $tmpdir;
6230
6231     if ($err) {
6232         unlink $tmpfn;
6233         $restore_destroy_volumes->($storecfg, $devinfo);
6234         die $err;
6235     }
6236
6237     rename($tmpfn, $conffile) ||
6238         die "unable to commit configuration file '$conffile'\n";
6239
6240     PVE::Cluster::cfs_update(); # make sure we read new file
6241
6242     eval { rescan($vmid, 1); };
6243     warn $@ if $@;
6244 }
6245
6246 sub restore_vma_archive {
6247     my ($archive, $vmid, $user, $opts, $comp) = @_;
6248
6249     my $readfrom = $archive;
6250
6251     my $cfg = PVE::Storage::config();
6252     my $commands = [];
6253     my $bwlimit = $opts->{bwlimit};
6254
6255     my $dbg_cmdstring = '';
6256     my $add_pipe = sub {
6257         my ($cmd) = @_;
6258         push @$commands, $cmd;
6259         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6260         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6261         $readfrom = '-';
6262     };
6263
6264     my $input = undef;
6265     if ($archive eq '-') {
6266         $input = '<&STDIN';
6267     } else {
6268         # If we use a backup from a PVE defined storage we also consider that
6269         # storage's rate limit:
6270         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6271         if (defined($volid)) {
6272             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6273             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6274             if ($readlimit) {
6275                 print STDERR "applying read rate limit: $readlimit\n";
6276                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6277                 $add_pipe->($cstream);
6278             }
6279         }
6280     }
6281
6282     if ($comp) {
6283         my $info = PVE::Storage::decompressor_info('vma', $comp);
6284         my $cmd = $info->{decompressor};
6285         push @$cmd, $readfrom;
6286         $add_pipe->($cmd);
6287     }
6288
6289     my $tmpdir = "/var/tmp/vzdumptmp$$";
6290     rmtree $tmpdir;
6291
6292     # disable interrupts (always do cleanups)
6293     local $SIG{INT} =
6294         local $SIG{TERM} =
6295         local $SIG{QUIT} =
6296         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6297
6298     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6299     POSIX::mkfifo($mapfifo, 0600);
6300     my $fifofh;
6301
6302     my $openfifo = sub {
6303         open($fifofh, '>', $mapfifo) || die $!;
6304     };
6305
6306     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6307
6308     my $oldtimeout;
6309     my $timeout = 5;
6310
6311     my $devinfo = {};
6312
6313     my $rpcenv = PVE::RPCEnvironment::get();
6314
6315     my $conffile = PVE::QemuConfig->config_file($vmid);
6316     my $tmpfn = "$conffile.$$.tmp";
6317
6318     # Note: $oldconf is undef if VM does not exist
6319     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6320     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6321
6322     my %storage_limits;
6323
6324     my $print_devmap = sub {
6325         my $cfgfn = "$tmpdir/qemu-server.conf";
6326
6327         # we can read the config - that is already extracted
6328         my $fh = IO::File->new($cfgfn, "r") ||
6329             "unable to read qemu-server.conf - $!\n";
6330
6331         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6332         if (-f $fwcfgfn) {
6333             my $pve_firewall_dir = '/etc/pve/firewall';
6334             mkdir $pve_firewall_dir; # make sure the dir exists
6335             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6336         }
6337
6338         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6339
6340         foreach my $key (keys %storage_limits) {
6341             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6342             next if !$limit;
6343             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6344             $storage_limits{$key} = $limit * 1024;
6345         }
6346
6347         foreach my $devname (keys %$devinfo) {
6348             die "found no device mapping information for device '$devname'\n"
6349                 if !$devinfo->{$devname}->{virtdev};
6350         }
6351
6352         # create empty/temp config
6353         if ($oldconf) {
6354             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6355             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6356         }
6357
6358         # allocate volumes
6359         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6360
6361         # print restore information to $fifofh
6362         foreach my $virtdev (sort keys %$virtdev_hash) {
6363             my $d = $virtdev_hash->{$virtdev};
6364             next if $d->{is_cloudinit}; # no need to restore cloudinit
6365
6366             my $storeid = $d->{storeid};
6367             my $volid = $d->{volid};
6368
6369             my $map_opts = '';
6370             if (my $limit = $storage_limits{$storeid}) {
6371                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6372             }
6373
6374             my $write_zeros = 1;
6375             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6376                 $write_zeros = 0;
6377             }
6378
6379             my $path = PVE::Storage::path($cfg, $volid);
6380
6381             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6382
6383             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6384         }
6385
6386         $fh->seek(0, 0) || die "seek failed - $!\n";
6387
6388         my $outfd = new IO::File ($tmpfn, "w") ||
6389             die "unable to write config for VM $vmid\n";
6390
6391         my $cookie = { netcount => 0 };
6392         while (defined(my $line = <$fh>)) {
6393             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6394         }
6395
6396         $fh->close();
6397         $outfd->close();
6398     };
6399
6400     eval {
6401         # enable interrupts
6402         local $SIG{INT} =
6403             local $SIG{TERM} =
6404             local $SIG{QUIT} =
6405             local $SIG{HUP} =
6406             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6407         local $SIG{ALRM} = sub { die "got timeout\n"; };
6408
6409         $oldtimeout = alarm($timeout);
6410
6411         my $parser = sub {
6412             my $line = shift;
6413
6414             print "$line\n";
6415
6416             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6417                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6418                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6419             } elsif ($line =~ m/^CTIME: /) {
6420                 # we correctly received the vma config, so we can disable
6421                 # the timeout now for disk allocation (set to 10 minutes, so
6422                 # that we always timeout if something goes wrong)
6423                 alarm(600);
6424                 &$print_devmap();
6425                 print $fifofh "done\n";
6426                 my $tmp = $oldtimeout || 0;
6427                 $oldtimeout = undef;
6428                 alarm($tmp);
6429                 close($fifofh);
6430             }
6431         };
6432
6433         print "restore vma archive: $dbg_cmdstring\n";
6434         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6435     };
6436     my $err = $@;
6437
6438     alarm($oldtimeout) if $oldtimeout;
6439
6440     $restore_deactivate_volumes->($cfg, $devinfo);
6441
6442     unlink $mapfifo;
6443     rmtree $tmpdir;
6444
6445     if ($err) {
6446         unlink $tmpfn;
6447         $restore_destroy_volumes->($cfg, $devinfo);
6448         die $err;
6449     }
6450
6451     rename($tmpfn, $conffile) ||
6452         die "unable to commit configuration file '$conffile'\n";
6453
6454     PVE::Cluster::cfs_update(); # make sure we read new file
6455
6456     eval { rescan($vmid, 1); };
6457     warn $@ if $@;
6458 }
6459
6460 sub restore_tar_archive {
6461     my ($archive, $vmid, $user, $opts) = @_;
6462
6463     if ($archive ne '-') {
6464         my $firstfile = tar_archive_read_firstfile($archive);
6465         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6466             if $firstfile ne 'qemu-server.conf';
6467     }
6468
6469     my $storecfg = PVE::Storage::config();
6470
6471     # avoid zombie disks when restoring over an existing VM -> cleanup first
6472     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6473     # skiplock=1 because qmrestore has set the 'create' lock itself already
6474     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6475     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6476
6477     my $tocmd = "/usr/lib/qemu-server/qmextract";
6478
6479     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6480     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6481     $tocmd .= ' --prealloc' if $opts->{prealloc};
6482     $tocmd .= ' --info' if $opts->{info};
6483
6484     # tar option "xf" does not autodetect compression when read from STDIN,
6485     # so we pipe to zcat
6486     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6487         PVE::Tools::shellquote("--to-command=$tocmd");
6488
6489     my $tmpdir = "/var/tmp/vzdumptmp$$";
6490     mkpath $tmpdir;
6491
6492     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6493     local $ENV{VZDUMP_VMID} = $vmid;
6494     local $ENV{VZDUMP_USER} = $user;
6495
6496     my $conffile = PVE::QemuConfig->config_file($vmid);
6497     my $tmpfn = "$conffile.$$.tmp";
6498
6499     # disable interrupts (always do cleanups)
6500     local $SIG{INT} =
6501         local $SIG{TERM} =
6502         local $SIG{QUIT} =
6503         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6504
6505     eval {
6506         # enable interrupts
6507         local $SIG{INT} =
6508             local $SIG{TERM} =
6509             local $SIG{QUIT} =
6510             local $SIG{HUP} =
6511             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6512
6513         if ($archive eq '-') {
6514             print "extracting archive from STDIN\n";
6515             run_command($cmd, input => "<&STDIN");
6516         } else {
6517             print "extracting archive '$archive'\n";
6518             run_command($cmd);
6519         }
6520
6521         return if $opts->{info};
6522
6523         # read new mapping
6524         my $map = {};
6525         my $statfile = "$tmpdir/qmrestore.stat";
6526         if (my $fd = IO::File->new($statfile, "r")) {
6527             while (defined (my $line = <$fd>)) {
6528                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6529                     $map->{$1} = $2 if $1;
6530                 } else {
6531                     print STDERR "unable to parse line in statfile - $line\n";
6532                 }
6533             }
6534             $fd->close();
6535         }
6536
6537         my $confsrc = "$tmpdir/qemu-server.conf";
6538
6539         my $srcfd = new IO::File($confsrc, "r") ||
6540             die "unable to open file '$confsrc'\n";
6541
6542         my $outfd = new IO::File ($tmpfn, "w") ||
6543             die "unable to write config for VM $vmid\n";
6544
6545         my $cookie = { netcount => 0 };
6546         while (defined (my $line = <$srcfd>)) {
6547             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6548         }
6549
6550         $srcfd->close();
6551         $outfd->close();
6552     };
6553     if (my $err = $@) {
6554         unlink $tmpfn;
6555         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6556         die $err;
6557     }
6558
6559     rmtree $tmpdir;
6560
6561     rename $tmpfn, $conffile ||
6562         die "unable to commit configuration file '$conffile'\n";
6563
6564     PVE::Cluster::cfs_update(); # make sure we read new file
6565
6566     eval { rescan($vmid, 1); };
6567     warn $@ if $@;
6568 };
6569
6570 sub foreach_storage_used_by_vm {
6571     my ($conf, $func) = @_;
6572
6573     my $sidhash = {};
6574
6575     PVE::QemuConfig->foreach_volume($conf, sub {
6576         my ($ds, $drive) = @_;
6577         return if drive_is_cdrom($drive);
6578
6579         my $volid = $drive->{file};
6580
6581         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6582         $sidhash->{$sid} = $sid if $sid;
6583     });
6584
6585     foreach my $sid (sort keys %$sidhash) {
6586         &$func($sid);
6587     }
6588 }
6589
6590 my $qemu_snap_storage = {
6591     rbd => 1,
6592 };
6593 sub do_snapshots_with_qemu {
6594     my ($storecfg, $volid) = @_;
6595
6596     my $storage_name = PVE::Storage::parse_volume_id($volid);
6597     my $scfg = $storecfg->{ids}->{$storage_name};
6598
6599     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6600         return 1;
6601     }
6602
6603     if ($volid =~ m/\.(qcow2|qed)$/){
6604         return 1;
6605     }
6606
6607     return undef;
6608 }
6609
6610 sub qga_check_running {
6611     my ($vmid, $nowarn) = @_;
6612
6613     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6614     if ($@) {
6615         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6616         return 0;
6617     }
6618     return 1;
6619 }
6620
6621 sub template_create {
6622     my ($vmid, $conf, $disk) = @_;
6623
6624     my $storecfg = PVE::Storage::config();
6625
6626     PVE::QemuConfig->foreach_volume($conf, sub {
6627         my ($ds, $drive) = @_;
6628
6629         return if drive_is_cdrom($drive);
6630         return if $disk && $ds ne $disk;
6631
6632         my $volid = $drive->{file};
6633         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6634
6635         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6636         $drive->{file} = $voliddst;
6637         $conf->{$ds} = print_drive($drive);
6638         PVE::QemuConfig->write_config($vmid, $conf);
6639     });
6640 }
6641
6642 sub convert_iscsi_path {
6643     my ($path) = @_;
6644
6645     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6646         my $portal = $1;
6647         my $target = $2;
6648         my $lun = $3;
6649
6650         my $initiator_name = get_initiator_name();
6651
6652         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6653                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6654     }
6655
6656     die "cannot convert iscsi path '$path', unkown format\n";
6657 }
6658
6659 sub qemu_img_convert {
6660     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6661
6662     my $storecfg = PVE::Storage::config();
6663     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6664     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6665
6666     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6667
6668     my $cachemode;
6669     my $src_path;
6670     my $src_is_iscsi = 0;
6671     my $src_format;
6672
6673     if ($src_storeid) {
6674         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6675         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6676         $src_format = qemu_img_format($src_scfg, $src_volname);
6677         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6678         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6679         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6680     } elsif (-f $src_volid) {
6681         $src_path = $src_volid;
6682         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6683             $src_format = $1;
6684         }
6685     }
6686
6687     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6688
6689     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6690     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6691     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6692     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6693
6694     my $cmd = [];
6695     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6696     push @$cmd, '-l', "snapshot.name=$snapname"
6697         if $snapname && $src_format && $src_format eq "qcow2";
6698     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6699     push @$cmd, '-T', $cachemode if defined($cachemode);
6700
6701     if ($src_is_iscsi) {
6702         push @$cmd, '--image-opts';
6703         $src_path = convert_iscsi_path($src_path);
6704     } elsif ($src_format) {
6705         push @$cmd, '-f', $src_format;
6706     }
6707
6708     if ($dst_is_iscsi) {
6709         push @$cmd, '--target-image-opts';
6710         $dst_path = convert_iscsi_path($dst_path);
6711     } else {
6712         push @$cmd, '-O', $dst_format;
6713     }
6714
6715     push @$cmd, $src_path;
6716
6717     if (!$dst_is_iscsi && $is_zero_initialized) {
6718         push @$cmd, "zeroinit:$dst_path";
6719     } else {
6720         push @$cmd, $dst_path;
6721     }
6722
6723     my $parser = sub {
6724         my $line = shift;
6725         if($line =~ m/\((\S+)\/100\%\)/){
6726             my $percent = $1;
6727             my $transferred = int($size * $percent / 100);
6728             my $remaining = $size - $transferred;
6729
6730             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6731         }
6732
6733     };
6734
6735     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6736     my $err = $@;
6737     die "copy failed: $err" if $err;
6738 }
6739
6740 sub qemu_img_format {
6741     my ($scfg, $volname) = @_;
6742
6743     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6744         return $1;
6745     } else {
6746         return "raw";
6747     }
6748 }
6749
6750 sub qemu_drive_mirror {
6751     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6752
6753     $jobs = {} if !$jobs;
6754
6755     my $qemu_target;
6756     my $format;
6757     $jobs->{"drive-$drive"} = {};
6758
6759     if ($dst_volid =~ /^nbd:/) {
6760         $qemu_target = $dst_volid;
6761         $format = "nbd";
6762     } else {
6763         my $storecfg = PVE::Storage::config();
6764         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6765
6766         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6767
6768         $format = qemu_img_format($dst_scfg, $dst_volname);
6769
6770         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6771
6772         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6773     }
6774
6775     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6776     $opts->{format} = $format if $format;
6777
6778     if (defined($src_bitmap)) {
6779         $opts->{sync} = 'incremental';
6780         $opts->{bitmap} = $src_bitmap;
6781         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6782     }
6783
6784     if (defined($bwlimit)) {
6785         $opts->{speed} = $bwlimit * 1024;
6786         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6787     } else {
6788         print "drive mirror is starting for drive-$drive\n";
6789     }
6790
6791     # if a job already runs for this device we get an error, catch it for cleanup
6792     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6793     if (my $err = $@) {
6794         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6795         warn "$@\n" if $@;
6796         die "mirroring error: $err\n";
6797     }
6798
6799     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6800 }
6801
6802 # $completion can be either
6803 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6804 # 'cancel': wait until all jobs are ready, block-job-cancel them
6805 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6806 sub qemu_drive_mirror_monitor {
6807     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6808
6809     $completion //= 'complete';
6810
6811     eval {
6812         my $err_complete = 0;
6813
6814         while (1) {
6815             die "storage migration timed out\n" if $err_complete > 300;
6816
6817             my $stats = mon_cmd($vmid, "query-block-jobs");
6818
6819             my $running_mirror_jobs = {};
6820             foreach my $stat (@$stats) {
6821                 next if $stat->{type} ne 'mirror';
6822                 $running_mirror_jobs->{$stat->{device}} = $stat;
6823             }
6824
6825             my $readycounter = 0;
6826
6827             foreach my $job (keys %$jobs) {
6828
6829                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6830                     print "$job : finished\n";
6831                     delete $jobs->{$job};
6832                     next;
6833                 }
6834
6835                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6836
6837                 my $busy = $running_mirror_jobs->{$job}->{busy};
6838                 my $ready = $running_mirror_jobs->{$job}->{ready};
6839                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6840                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6841                     my $remaining = $total - $transferred;
6842                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6843
6844                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6845                 }
6846
6847                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6848             }
6849
6850             last if scalar(keys %$jobs) == 0;
6851
6852             if ($readycounter == scalar(keys %$jobs)) {
6853                 print "all mirroring jobs are ready \n";
6854                 last if $completion eq 'skip'; #do the complete later
6855
6856                 if ($vmiddst && $vmiddst != $vmid) {
6857                     my $agent_running = $qga && qga_check_running($vmid);
6858                     if ($agent_running) {
6859                         print "freeze filesystem\n";
6860                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6861                     } else {
6862                         print "suspend vm\n";
6863                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6864                     }
6865
6866                     # if we clone a disk for a new target vm, we don't switch the disk
6867                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6868
6869                     if ($agent_running) {
6870                         print "unfreeze filesystem\n";
6871                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6872                     } else {
6873                         print "resume vm\n";
6874                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6875                     }
6876
6877                     last;
6878                 } else {
6879
6880                     foreach my $job (keys %$jobs) {
6881                         # try to switch the disk if source and destination are on the same guest
6882                         print "$job: Completing block job...\n";
6883
6884                         my $op;
6885                         if ($completion eq 'complete') {
6886                             $op = 'block-job-complete';
6887                         } elsif ($completion eq 'cancel') {
6888                             $op = 'block-job-cancel';
6889                         } else {
6890                             die "invalid completion value: $completion\n";
6891                         }
6892                         eval { mon_cmd($vmid, $op, device => $job) };
6893                         if ($@ =~ m/cannot be completed/) {
6894                             print "$job: Block job cannot be completed, try again.\n";
6895                             $err_complete++;
6896                         }else {
6897                             print "$job: Completed successfully.\n";
6898                             $jobs->{$job}->{complete} = 1;
6899                         }
6900                     }
6901                 }
6902             }
6903             sleep 1;
6904         }
6905     };
6906     my $err = $@;
6907
6908     if ($err) {
6909         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6910         die "mirroring error: $err";
6911     }
6912
6913 }
6914
6915 sub qemu_blockjobs_cancel {
6916     my ($vmid, $jobs) = @_;
6917
6918     foreach my $job (keys %$jobs) {
6919         print "$job: Cancelling block job\n";
6920         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6921         $jobs->{$job}->{cancel} = 1;
6922     }
6923
6924     while (1) {
6925         my $stats = mon_cmd($vmid, "query-block-jobs");
6926
6927         my $running_jobs = {};
6928         foreach my $stat (@$stats) {
6929             $running_jobs->{$stat->{device}} = $stat;
6930         }
6931
6932         foreach my $job (keys %$jobs) {
6933
6934             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6935                 print "$job: Done.\n";
6936                 delete $jobs->{$job};
6937             }
6938         }
6939
6940         last if scalar(keys %$jobs) == 0;
6941
6942         sleep 1;
6943     }
6944 }
6945
6946 sub clone_disk {
6947     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6948         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6949
6950     my $newvolid;
6951
6952     if (!$full) {
6953         print "create linked clone of drive $drivename ($drive->{file})\n";
6954         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6955         push @$newvollist, $newvolid;
6956     } else {
6957
6958         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6959         $storeid = $storage if $storage;
6960
6961         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6962         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6963
6964         print "create full clone of drive $drivename ($drive->{file})\n";
6965         my $name = undef;
6966         if (drive_is_cloudinit($drive)) {
6967             $name = "vm-$newvmid-cloudinit";
6968             $name .= ".$dst_format" if $dst_format ne 'raw';
6969             $snapname = undef;
6970             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6971         } elsif ($drivename eq 'efidisk0') {
6972             $size = get_efivars_size($conf);
6973         }
6974         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6975         push @$newvollist, $newvolid;
6976
6977         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6978
6979         if (drive_is_cloudinit($drive)) {
6980             goto no_data_clone;
6981         }
6982
6983         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6984         if (!$running || $snapname) {
6985             # TODO: handle bwlimits
6986             if ($drivename eq 'efidisk0') {
6987                 # the relevant data on the efidisk may be smaller than the source
6988                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6989                 # that is given by the OVMF_VARS.fd
6990                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6991                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6992                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6993             } else {
6994                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6995             }
6996         } else {
6997
6998             my $kvmver = get_running_qemu_version ($vmid);
6999             if (!min_version($kvmver, 2, 7)) {
7000                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7001                     if $drive->{iothread};
7002             }
7003
7004             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
7005         }
7006     }
7007
7008 no_data_clone:
7009     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7010
7011     my $disk = $drive;
7012     $disk->{format} = undef;
7013     $disk->{file} = $newvolid;
7014     $disk->{size} = $size;
7015
7016     return $disk;
7017 }
7018
7019 sub get_running_qemu_version {
7020     my ($vmid) = @_;
7021     my $res = mon_cmd($vmid, "query-version");
7022     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7023 }
7024
7025 sub qemu_use_old_bios_files {
7026     my ($machine_type) = @_;
7027
7028     return if !$machine_type;
7029
7030     my $use_old_bios_files = undef;
7031
7032     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7033         $machine_type = $1;
7034         $use_old_bios_files = 1;
7035     } else {
7036         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7037         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7038         # load new efi bios files on migration. So this hack is required to allow
7039         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7040         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7041         $use_old_bios_files = !min_version($version, 2, 4);
7042     }
7043
7044     return ($use_old_bios_files, $machine_type);
7045 }
7046
7047 sub get_efivars_size {
7048     my ($conf) = @_;
7049     my $arch = get_vm_arch($conf);
7050     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7051     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7052     return -s $ovmf_vars;
7053 }
7054
7055 sub update_efidisk_size {
7056     my ($conf) = @_;
7057
7058     return if !defined($conf->{efidisk0});
7059
7060     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7061     $disk->{size} = get_efivars_size($conf);
7062     $conf->{efidisk0} = print_drive($disk);
7063
7064     return;
7065 }
7066
7067 sub create_efidisk($$$$$) {
7068     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7069
7070     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7071     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7072
7073     my $vars_size_b = -s $ovmf_vars;
7074     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7075     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7076     PVE::Storage::activate_volumes($storecfg, [$volid]);
7077
7078     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7079     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7080
7081     return ($volid, $size/1024);
7082 }
7083
7084 sub vm_iothreads_list {
7085     my ($vmid) = @_;
7086
7087     my $res = mon_cmd($vmid, 'query-iothreads');
7088
7089     my $iothreads = {};
7090     foreach my $iothread (@$res) {
7091         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7092     }
7093
7094     return $iothreads;
7095 }
7096
7097 sub scsihw_infos {
7098     my ($conf, $drive) = @_;
7099
7100     my $maxdev = 0;
7101
7102     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7103         $maxdev = 7;
7104     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7105         $maxdev = 1;
7106     } else {
7107         $maxdev = 256;
7108     }
7109
7110     my $controller = int($drive->{index} / $maxdev);
7111     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7112
7113     return ($maxdev, $controller, $controller_prefix);
7114 }
7115
7116 sub windows_version {
7117     my ($ostype) = @_;
7118
7119     return 0 if !$ostype;
7120
7121     my $winversion = 0;
7122
7123     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7124         $winversion = 5;
7125     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7126         $winversion = 6;
7127     } elsif ($ostype =~ m/^win(\d+)$/) {
7128         $winversion = $1;
7129     }
7130
7131     return $winversion;
7132 }
7133
7134 sub resolve_dst_disk_format {
7135         my ($storecfg, $storeid, $src_volname, $format) = @_;
7136         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7137
7138         if (!$format) {
7139             # if no target format is specified, use the source disk format as hint
7140             if ($src_volname) {
7141                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7142                 $format = qemu_img_format($scfg, $src_volname);
7143             } else {
7144                 return $defFormat;
7145             }
7146         }
7147
7148         # test if requested format is supported - else use default
7149         my $supported = grep { $_ eq $format } @$validFormats;
7150         $format = $defFormat if !$supported;
7151         return $format;
7152 }
7153
7154 # NOTE: if this logic changes, please update docs & possibly gui logic
7155 sub find_vmstate_storage {
7156     my ($conf, $storecfg) = @_;
7157
7158     # first, return storage from conf if set
7159     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7160
7161     my ($target, $shared, $local);
7162
7163     foreach_storage_used_by_vm($conf, sub {
7164         my ($sid) = @_;
7165         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7166         my $dst = $scfg->{shared} ? \$shared : \$local;
7167         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7168     });
7169
7170     # second, use shared storage where VM has at least one disk
7171     # third, use local storage where VM has at least one disk
7172     # fall back to local storage
7173     $target = $shared // $local // 'local';
7174
7175     return $target;
7176 }
7177
7178 sub generate_uuid {
7179     my ($uuid, $uuid_str);
7180     UUID::generate($uuid);
7181     UUID::unparse($uuid, $uuid_str);
7182     return $uuid_str;
7183 }
7184
7185 sub generate_smbios1_uuid {
7186     return "uuid=".generate_uuid();
7187 }
7188
7189 sub nbd_stop {
7190     my ($vmid) = @_;
7191
7192     mon_cmd($vmid, 'nbd-server-stop');
7193 }
7194
7195 sub create_reboot_request {
7196     my ($vmid) = @_;
7197     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7198         or die "failed to create reboot trigger file: $!\n";
7199     close($fh);
7200 }
7201
7202 sub clear_reboot_request {
7203     my ($vmid) = @_;
7204     my $path = "/run/qemu-server/$vmid.reboot";
7205     my $res = 0;
7206
7207     $res = unlink($path);
7208     die "could not remove reboot request for $vmid: $!"
7209         if !$res && $! != POSIX::ENOENT;
7210
7211     return $res;
7212 }
7213
7214 # bash completion helper
7215
7216 sub complete_backup_archives {
7217     my ($cmdname, $pname, $cvalue) = @_;
7218
7219     my $cfg = PVE::Storage::config();
7220
7221     my $storeid;
7222
7223     if ($cvalue =~ m/^([^:]+):/) {
7224         $storeid = $1;
7225     }
7226
7227     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7228
7229     my $res = [];
7230     foreach my $id (keys %$data) {
7231         foreach my $item (@{$data->{$id}}) {
7232             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7233             push @$res, $item->{volid} if defined($item->{volid});
7234         }
7235     }
7236
7237     return $res;
7238 }
7239
7240 my $complete_vmid_full = sub {
7241     my ($running) = @_;
7242
7243     my $idlist = vmstatus();
7244
7245     my $res = [];
7246
7247     foreach my $id (keys %$idlist) {
7248         my $d = $idlist->{$id};
7249         if (defined($running)) {
7250             next if $d->{template};
7251             next if $running && $d->{status} ne 'running';
7252             next if !$running && $d->{status} eq 'running';
7253         }
7254         push @$res, $id;
7255
7256     }
7257     return $res;
7258 };
7259
7260 sub complete_vmid {
7261     return &$complete_vmid_full();
7262 }
7263
7264 sub complete_vmid_stopped {
7265     return &$complete_vmid_full(0);
7266 }
7267
7268 sub complete_vmid_running {
7269     return &$complete_vmid_full(1);
7270 }
7271
7272 sub complete_storage {
7273
7274     my $cfg = PVE::Storage::config();
7275     my $ids = $cfg->{ids};
7276
7277     my $res = [];
7278     foreach my $sid (keys %$ids) {
7279         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7280         next if !$ids->{$sid}->{content}->{images};
7281         push @$res, $sid;
7282     }
7283
7284     return $res;
7285 }
7286
7287 sub complete_migration_storage {
7288     my ($cmd, $param, $current_value, $all_args) = @_;
7289
7290     my $targetnode = @$all_args[1];
7291
7292     my $cfg = PVE::Storage::config();
7293     my $ids = $cfg->{ids};
7294
7295     my $res = [];
7296     foreach my $sid (keys %$ids) {
7297         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7298         next if !$ids->{$sid}->{content}->{images};
7299         push @$res, $sid;
7300     }
7301
7302     return $res;
7303 }
7304
7305 1;