PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  47 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  48 use PVE::QemuServer::Machine;
  49 use PVE::QemuServer::Memory;
  50 use PVE::QemuServer::Monitor qw(mon_cmd);
  51 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  52 use PVE::QemuServer::USB qw(parse_usb_device);
  53
  54 my $have_sdn;
  55 eval {
  56     require PVE::Network::SDN::Zones;
  57     $have_sdn = 1;
  58 };
  59
  60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  61 my $OVMF = {
  62     x86_64 => [
  63         "$EDK2_FW_BASE/OVMF_CODE.fd",
  64         "$EDK2_FW_BASE/OVMF_VARS.fd"
  65     ],
  66     aarch64 => [
  67         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  68         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  69     ],
  70 };
  71
  72 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  73
  74 # Note about locking: we use flock on the config file protect
  75 # against concurent actions.
  76 # Aditionaly, we have a 'lock' setting in the config file. This
  77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  78 # allowed when such lock is set. But you can ignore this kind of
  79 # lock with the --skiplock flag.
  80
  81 cfs_register_file('/qemu-server/',
  82                   \&parse_vm_config,
  83                   \&write_vm_config);
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  86     description => "Some command save/restore state from this location.",
  87     type => 'string',
  88     maxLength => 128,
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100
 101 sub map_storage {
 102     my ($map, $source) = @_;
 103
 104     return $source if !defined($map);
 105
 106     return $map->{entries}->{$source}
 107         if $map->{entries} && defined($map->{entries}->{$source});
 108
 109     return $map->{default} if $map->{default};
 110
 111     # identity (fallback)
 112     return $source;
 113 }
 114
 115 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 116     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 117     type => 'string',
 118     format => 'storagepair-list',
 119     optional => 1,
 120 });
 121
 122 #no warnings 'redefine';
 123
 124 sub cgroups_write {
 125    my ($controller, $vmid, $option, $value) = @_;
 126
 127    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 128    PVE::ProcFSTools::write_proc_entry($path, $value);
 129
 130 }
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable Qemu GuestAgent.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after cloning/moving a disk.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most"
 248                      . " cases /dev/urandom should be preferred over /dev/random"
 249                      . " to avoid entropy-starvation issues on the host. Using"
 250                      . " urandom does *not* decrease security in any meaningful"
 251                      . " way, as it's still seeded from real entropy, and the"
 252                      . " bytes provided will most likely be mixed with real"
 253                      . " entropy on the guest as well. /dev/hwrng can be used"
 254                      . " to pass through a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy injected into the guest every"
 259                      . " 'period' milliseconds. Prefer a lower value when using"
 260                      . " /dev/random as source. Use 0 to disable limiting"
 261                      . " (potentially dangerous!).",
 262         optional => 1,
 263
 264         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 265         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 266         # of overwhelming the host, provided we're reading from /dev/urandom
 267         default => 1024,
 268     },
 269     period => {
 270         type => 'integer',
 271         description => "Every 'period' milliseconds the entropy-injection quota"
 272                      . " is reset, allowing the guest to retrieve another"
 273                      . " 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $confdesc = {
 280     onboot => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Specifies whether a VM will be started during system bootup.",
 284         default => 0,
 285     },
 286     autostart => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Automatic restart after crash (currently ignored).",
 290         default => 0,
 291     },
 292     hotplug => {
 293         optional => 1,
 294         type => 'string', format => 'pve-hotplug-features',
 295         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 296         default => 'network,disk,usb',
 297     },
 298     reboot => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 302         default => 1,
 303     },
 304     lock => {
 305         optional => 1,
 306         type => 'string',
 307         description => "Lock/unlock the VM.",
 308         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 309     },
 310     cpulimit => {
 311         optional => 1,
 312         type => 'number',
 313         description => "Limit of CPU usage.",
 314         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 315         minimum => 0,
 316         maximum => 128,
 317         default => 0,
 318     },
 319     cpuunits => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "CPU weight for a VM.",
 323         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 324         minimum => 2,
 325         maximum => 262144,
 326         default => 1024,
 327     },
 328     memory => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 332         minimum => 16,
 333         default => 512,
 334     },
 335     balloon => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 339         minimum => 0,
 340     },
 341     shares => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 345         minimum => 0,
 346         maximum => 50000,
 347         default => 1000,
 348     },
 349     keyboard => {
 350         optional => 1,
 351         type => 'string',
 352         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 353                        "It should not be necessary to set it.",
 354         enum => PVE::Tools::kvmkeymaplist(),
 355         default => undef,
 356     },
 357     name => {
 358         optional => 1,
 359         type => 'string', format => 'dns-name',
 360         description => "Set a name for the VM. Only used on the configuration web interface.",
 361     },
 362     scsihw => {
 363         optional => 1,
 364         type => 'string',
 365         description => "SCSI controller model",
 366         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 367         default => 'lsi',
 368     },
 369     description => {
 370         optional => 1,
 371         type => 'string',
 372         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 373     },
 374     ostype => {
 375         optional => 1,
 376         type => 'string',
 377         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 378         description => "Specify guest operating system.",
 379         verbose_description => <<EODESC,
 380 Specify guest operating system. This is used to enable special
 381 optimization/features for specific operating systems:
 382
 383 [horizontal]
 384 other;; unspecified OS
 385 wxp;; Microsoft Windows XP
 386 w2k;; Microsoft Windows 2000
 387 w2k3;; Microsoft Windows 2003
 388 w2k8;; Microsoft Windows 2008
 389 wvista;; Microsoft Windows Vista
 390 win7;; Microsoft Windows 7
 391 win8;; Microsoft Windows 8/2012/2012r2
 392 win10;; Microsoft Windows 10/2016
 393 l24;; Linux 2.4 Kernel
 394 l26;; Linux 2.6 - 5.X Kernel
 395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 396 EODESC
 397     },
 398     boot => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 402         pattern => '[acdn]{1,4}',
 403         default => 'cdn',
 404     },
 405     bootdisk => {
 406         optional => 1,
 407         type => 'string', format => 'pve-qm-bootdisk',
 408         description => "Enable booting from specified disk.",
 409         pattern => '(ide|sata|scsi|virtio)\d+',
 410     },
 411     smp => {
 412         optional => 1,
 413         type => 'integer',
 414         description => "The number of CPUs. Please use option -sockets instead.",
 415         minimum => 1,
 416         default => 1,
 417     },
 418     sockets => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "The number of CPU sockets.",
 422         minimum => 1,
 423         default => 1,
 424     },
 425     cores => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of cores per socket.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     numa => {
 433         optional => 1,
 434         type => 'boolean',
 435         description => "Enable/disable NUMA.",
 436         default => 0,
 437     },
 438     hugepages => {
 439         optional => 1,
 440         type => 'string',
 441         description => "Enable/disable hugepages memory.",
 442         enum => [qw(any 2 1024)],
 443     },
 444     keephugepages => {
 445         optional => 1,
 446         type => 'boolean',
 447         default => 0,
 448         description => "Use together with hugepages. If enabled, hugepages will"
 449                      . " not be deleted after VM shutdown and can be used for"
 450                      . " subsequent starts.",
 451     },
 452     vcpus => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "Number of hotplugged vcpus.",
 456         minimum => 1,
 457         default => 0,
 458     },
 459     acpi => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable ACPI.",
 463         default => 1,
 464     },
 465     agent => {
 466         optional => 1,
 467         description => "Enable/disable Qemu GuestAgent and its properties.",
 468         type => 'string',
 469         format => $agent_fmt,
 470     },
 471     kvm => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Enable/disable KVM hardware virtualization.",
 475         default => 1,
 476     },
 477     tdf => {
 478         optional => 1,
 479         type => 'boolean',
 480         description => "Enable/disable time drift fix.",
 481         default => 0,
 482     },
 483     localtime => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 487     },
 488     freeze => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 492     },
 493     vga => {
 494         optional => 1,
 495         type => 'string', format => $vga_fmt,
 496         description => "Configure the VGA hardware.",
 497         verbose_description => "Configure the VGA Hardware. If you want to use ".
 498             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 499             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 500             "is 'std' for all OS types besides some Windows versions (XP and " .
 501             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 502             "display server. For win* OS you can select how many independent " .
 503             "displays you want, Linux guests can add displays them self.\n".
 504             "You can also run without any graphic card, using a serial device as terminal.",
 505     },
 506     watchdog => {
 507         optional => 1,
 508         type => 'string', format => 'pve-qm-watchdog',
 509         description => "Create a virtual hardware watchdog device.",
 510         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 511             " (by a guest action), the watchdog must be periodically polled " .
 512             "by an agent inside the guest or else the watchdog will reset " .
 513             "the guest (or execute the respective action specified)",
 514     },
 515     startdate => {
 516         optional => 1,
 517         type => 'string',
 518         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 519         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 520         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 521         default => 'now',
 522     },
 523     startup =>  get_standard_option('pve-startup-order'),
 524     template => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Enable/disable Template.",
 528         default => 0,
 529     },
 530     args => {
 531         optional => 1,
 532         type => 'string',
 533         description => "Arbitrary arguments passed to kvm.",
 534         verbose_description => <<EODESCR,
 535 Arbitrary arguments passed to kvm, for example:
 536
 537 args: -no-reboot -no-hpet
 538
 539 NOTE: this option is for experts only.
 540 EODESCR
 541     },
 542     tablet => {
 543         optional => 1,
 544         type => 'boolean',
 545         default => 1,
 546         description => "Enable/disable the USB tablet device.",
 547         verbose_description => "Enable/disable the USB tablet device. This device is " .
 548             "usually needed to allow absolute mouse positioning with VNC. " .
 549             "Else the mouse runs out of sync with normal VNC clients. " .
 550             "If you're running lots of console-only guests on one host, " .
 551             "you may consider disabling this to save some context switches. " .
 552             "This is turned off by default if you use spice (-vga=qxl).",
 553     },
 554     migrate_speed => {
 555         optional => 1,
 556         type => 'integer',
 557         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 558         minimum => 0,
 559         default => 0,
 560     },
 561     migrate_downtime => {
 562         optional => 1,
 563         type => 'number',
 564         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 565         minimum => 0,
 566         default => 0.1,
 567     },
 568     cdrom => {
 569         optional => 1,
 570         type => 'string', format => 'pve-qm-ide',
 571         typetext => '<volume>',
 572         description => "This is an alias for option -ide2",
 573     },
 574     cpu => {
 575         optional => 1,
 576         description => "Emulated CPU type.",
 577         type => 'string',
 578         format => 'pve-vm-cpu-conf',
 579     },
 580     parent => get_standard_option('pve-snapshot-name', {
 581         optional => 1,
 582         description => "Parent snapshot name. This is used internally, and should not be modified.",
 583     }),
 584     snaptime => {
 585         optional => 1,
 586         description => "Timestamp for snapshots.",
 587         type => 'integer',
 588         minimum => 0,
 589     },
 590     vmstate => {
 591         optional => 1,
 592         type => 'string', format => 'pve-volume-id',
 593         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 594     },
 595     vmstatestorage => get_standard_option('pve-storage-id', {
 596         description => "Default storage for VM state volumes/files.",
 597         optional => 1,
 598     }),
 599     runningmachine => get_standard_option('pve-qemu-machine', {
 600         description => "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
 601     }),
 602     runningcpu => {
 603         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
 604         optional => 1,
 605         type => 'string',
 606         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 607         format_description => 'QEMU -cpu parameter'
 608     },
 609     machine => get_standard_option('pve-qemu-machine'),
 610     arch => {
 611         description => "Virtual processor architecture. Defaults to the host.",
 612         optional => 1,
 613         type => 'string',
 614         enum => [qw(x86_64 aarch64)],
 615     },
 616     smbios1 => {
 617         description => "Specify SMBIOS type 1 fields.",
 618         type => 'string', format => 'pve-qm-smbios1',
 619         maxLength => 512,
 620         optional => 1,
 621     },
 622     protection => {
 623         optional => 1,
 624         type => 'boolean',
 625         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 626         default => 0,
 627     },
 628     bios => {
 629         optional => 1,
 630         type => 'string',
 631         enum => [ qw(seabios ovmf) ],
 632         description => "Select BIOS implementation.",
 633         default => 'seabios',
 634     },
 635     vmgenid => {
 636         type => 'string',
 637         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 638         format_description => 'UUID',
 639         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 640         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 641             " 128-bit integer value identifier to the guest OS. This allows to".
 642             " notify the guest operating system when the virtual machine is".
 643             " executed with a different configuration (e.g. snapshot execution".
 644             " or creation from a template). The guest operating system notices".
 645             " the change, and is then able to react as appropriate by marking".
 646             " its copies of distributed databases as dirty, re-initializing its".
 647             " random number generator, etc.\n".
 648             "Note that auto-creation only works when done throug API/CLI create".
 649             " or update methods, but not when manually editing the config file.",
 650         default => "1 (autogenerated)",
 651         optional => 1,
 652     },
 653     hookscript => {
 654         type => 'string',
 655         format => 'pve-volume-id',
 656         optional => 1,
 657         description => "Script that will be executed during various steps in the vms lifetime.",
 658     },
 659     ivshmem => {
 660         type => 'string',
 661         format => $ivshmem_fmt,
 662         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 663         optional => 1,
 664     },
 665     audio0 => {
 666         type => 'string',
 667         format => $audio_fmt,
 668         description => "Configure a audio device, useful in combination with QXL/Spice.",
 669         optional => 1
 670     },
 671     spice_enhancements => {
 672         type => 'string',
 673         format => $spice_enhancements_fmt,
 674         description => "Configure additional enhancements for SPICE.",
 675         optional => 1
 676     },
 677     tags => {
 678         type => 'string', format => 'pve-tag-list',
 679         description => 'Tags of the VM. This is only meta information.',
 680         optional => 1,
 681     },
 682     rng0 => {
 683         type => 'string',
 684         format => $rng_fmt,
 685         description => "Configure a VirtIO-based Random Number Generator.",
 686         optional => 1,
 687     },
 688 };
 689
 690 my $cicustom_fmt = {
 691     meta => {
 692         type => 'string',
 693         optional => 1,
 694         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 695         format => 'pve-volume-id',
 696         format_description => 'volume',
 697     },
 698     network => {
 699         type => 'string',
 700         optional => 1,
 701         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 702         format => 'pve-volume-id',
 703         format_description => 'volume',
 704     },
 705     user => {
 706         type => 'string',
 707         optional => 1,
 708         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712 };
 713 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 714
 715 my $confdesc_cloudinit = {
 716     citype => {
 717         optional => 1,
 718         type => 'string',
 719         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 720         enum => ['configdrive2', 'nocloud'],
 721     },
 722     ciuser => {
 723         optional => 1,
 724         type => 'string',
 725         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 726     },
 727     cipassword => {
 728         optional => 1,
 729         type => 'string',
 730         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 731     },
 732     cicustom => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 736         format => 'pve-qm-cicustom',
 737     },
 738     searchdomain => {
 739         optional => 1,
 740         type => 'string',
 741         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 742     },
 743     nameserver => {
 744         optional => 1,
 745         type => 'string', format => 'address-list',
 746         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 747     },
 748     sshkeys => {
 749         optional => 1,
 750         type => 'string',
 751         format => 'urlencoded',
 752         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 753     },
 754 };
 755
 756 # what about other qemu settings ?
 757 #cpu => 'string',
 758 #machine => 'string',
 759 #fda => 'file',
 760 #fdb => 'file',
 761 #mtdblock => 'file',
 762 #sd => 'file',
 763 #pflash => 'file',
 764 #snapshot => 'bool',
 765 #bootp => 'file',
 766 ##tftp => 'dir',
 767 ##smb => 'dir',
 768 #kernel => 'file',
 769 #append => 'string',
 770 #initrd => 'file',
 771 ##soundhw => 'string',
 772
 773 while (my ($k, $v) = each %$confdesc) {
 774     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 775 }
 776
 777 my $MAX_USB_DEVICES = 5;
 778 my $MAX_NETS = 32;
 779 my $MAX_SERIAL_PORTS = 4;
 780 my $MAX_PARALLEL_PORTS = 3;
 781 my $MAX_NUMA = 8;
 782
 783 my $numa_fmt = {
 784     cpus => {
 785         type => "string",
 786         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 787         description => "CPUs accessing this NUMA node.",
 788         format_description => "id[-id];...",
 789     },
 790     memory => {
 791         type => "number",
 792         description => "Amount of memory this NUMA node provides.",
 793         optional => 1,
 794     },
 795     hostnodes => {
 796         type => "string",
 797         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 798         description => "Host NUMA nodes to use.",
 799         format_description => "id[-id];...",
 800         optional => 1,
 801     },
 802     policy => {
 803         type => 'string',
 804         enum => [qw(preferred bind interleave)],
 805         description => "NUMA allocation policy.",
 806         optional => 1,
 807     },
 808 };
 809 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 810 my $numadesc = {
 811     optional => 1,
 812     type => 'string', format => $numa_fmt,
 813     description => "NUMA topology.",
 814 };
 815 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 816
 817 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 818     $confdesc->{"numa$i"} = $numadesc;
 819 }
 820
 821 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 822                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 823                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 824 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 825
 826 my $net_fmt_bridge_descr = <<__EOD__;
 827 Bridge to attach the network device to. The Proxmox VE standard bridge
 828 is called 'vmbr0'.
 829
 830 If you do not specify a bridge, we create a kvm user (NATed) network
 831 device, which provides DHCP and DNS services. The following addresses
 832 are used:
 833
 834  10.0.2.2   Gateway
 835  10.0.2.3   DNS Server
 836  10.0.2.4   SMB Server
 837
 838 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 839 __EOD__
 840
 841 my $net_fmt = {
 842     macaddr  => get_standard_option('mac-addr', {
 843         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 844     }),
 845     model => {
 846         type => 'string',
 847         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 848         enum => $nic_model_list,
 849         default_key => 1,
 850     },
 851     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 852     bridge => {
 853         type => 'string',
 854         description => $net_fmt_bridge_descr,
 855         format_description => 'bridge',
 856         pattern => '[-_.\w\d]+',
 857         optional => 1,
 858     },
 859     queues => {
 860         type => 'integer',
 861         minimum => 0, maximum => 16,
 862         description => 'Number of packet queues to be used on the device.',
 863         optional => 1,
 864     },
 865     rate => {
 866         type => 'number',
 867         minimum => 0,
 868         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 869         optional => 1,
 870     },
 871     tag => {
 872         type => 'integer',
 873         minimum => 1, maximum => 4094,
 874         description => 'VLAN tag to apply to packets on this interface.',
 875         optional => 1,
 876     },
 877     trunks => {
 878         type => 'string',
 879         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 880         description => 'VLAN trunks to pass through this interface.',
 881         format_description => 'vlanid[;vlanid...]',
 882         optional => 1,
 883     },
 884     firewall => {
 885         type => 'boolean',
 886         description => 'Whether this interface should be protected by the firewall.',
 887         optional => 1,
 888     },
 889     link_down => {
 890         type => 'boolean',
 891         description => 'Whether this interface should be disconnected (like pulling the plug).',
 892         optional => 1,
 893     },
 894     mtu => {
 895         type => 'integer',
 896         minimum => 1, maximum => 65520,
 897         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 898         optional => 1,
 899     },
 900 };
 901
 902 my $netdesc = {
 903     optional => 1,
 904     type => 'string', format => $net_fmt,
 905     description => "Specify network devices.",
 906 };
 907
 908 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 909
 910 my $ipconfig_fmt = {
 911     ip => {
 912         type => 'string',
 913         format => 'pve-ipv4-config',
 914         format_description => 'IPv4Format/CIDR',
 915         description => 'IPv4 address in CIDR format.',
 916         optional => 1,
 917         default => 'dhcp',
 918     },
 919     gw => {
 920         type => 'string',
 921         format => 'ipv4',
 922         format_description => 'GatewayIPv4',
 923         description => 'Default gateway for IPv4 traffic.',
 924         optional => 1,
 925         requires => 'ip',
 926     },
 927     ip6 => {
 928         type => 'string',
 929         format => 'pve-ipv6-config',
 930         format_description => 'IPv6Format/CIDR',
 931         description => 'IPv6 address in CIDR format.',
 932         optional => 1,
 933         default => 'dhcp',
 934     },
 935     gw6 => {
 936         type => 'string',
 937         format => 'ipv6',
 938         format_description => 'GatewayIPv6',
 939         description => 'Default gateway for IPv6 traffic.',
 940         optional => 1,
 941         requires => 'ip6',
 942     },
 943 };
 944 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 945 my $ipconfigdesc = {
 946     optional => 1,
 947     type => 'string', format => 'pve-qm-ipconfig',
 948     description => <<'EODESCR',
 949 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 950
 951 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 952
 953 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 954 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 955
 956 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 957 EODESCR
 958 };
 959 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 960
 961 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 962     $confdesc->{"net$i"} = $netdesc;
 963     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 964 }
 965
 966 foreach my $key (keys %$confdesc_cloudinit) {
 967     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 968 }
 969
 970 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 971 sub verify_volume_id_or_qm_path {
 972     my ($volid, $noerr) = @_;
 973
 974     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 975         return $volid;
 976     }
 977
 978     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 979     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 980     if ($@) {
 981         return undef if $noerr;
 982         die $@;
 983     }
 984     return $volid;
 985 }
 986
 987 my $usb_fmt = {
 988     host => {
 989         default_key => 1,
 990         type => 'string', format => 'pve-qm-usb-device',
 991         format_description => 'HOSTUSBDEVICE|spice',
 992         description => <<EODESCR,
 993 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 994
 995  'bus-port(.port)*' (decimal numbers) or
 996  'vendor_id:product_id' (hexadeciaml numbers) or
 997  'spice'
 998
 999 You can use the 'lsusb -t' command to list existing usb devices.
1000
1001 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1002
1003 The value 'spice' can be used to add a usb redirection devices for spice.
1004 EODESCR
1005     },
1006     usb3 => {
1007         optional => 1,
1008         type => 'boolean',
1009         description => "Specifies whether if given host option is a USB3 device or port.",
1010         default => 0,
1011     },
1012 };
1013
1014 my $usbdesc = {
1015     optional => 1,
1016     type => 'string', format => $usb_fmt,
1017     description => "Configure an USB device (n is 0 to 4).",
1018 };
1019 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1020
1021 my $serialdesc = {
1022         optional => 1,
1023         type => 'string',
1024         pattern => '(/dev/.+|socket)',
1025         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1026         verbose_description =>  <<EODESCR,
1027 Create a serial device inside the VM (n is 0 to 3), and pass through a
1028 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1029 host side (use 'qm terminal' to open a terminal connection).
1030
1031 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1032
1033 CAUTION: Experimental! User reported problems with this option.
1034 EODESCR
1035 };
1036
1037 my $paralleldesc= {
1038         optional => 1,
1039         type => 'string',
1040         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1041         description =>  "Map host parallel devices (n is 0 to 2).",
1042         verbose_description =>  <<EODESCR,
1043 Map host parallel devices (n is 0 to 2).
1044
1045 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1046
1047 CAUTION: Experimental! User reported problems with this option.
1048 EODESCR
1049 };
1050
1051 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1052     $confdesc->{"parallel$i"} = $paralleldesc;
1053 }
1054
1055 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1056     $confdesc->{"serial$i"} = $serialdesc;
1057 }
1058
1059 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1060     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1061 }
1062
1063 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1064     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1065 }
1066
1067 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1068     $confdesc->{"usb$i"} = $usbdesc;
1069 }
1070
1071 my $kvm_api_version = 0;
1072
1073 sub kvm_version {
1074     return $kvm_api_version if $kvm_api_version;
1075
1076     open my $fh, '<', '/dev/kvm'
1077         or return undef;
1078
1079     # 0xae00 => KVM_GET_API_VERSION
1080     $kvm_api_version = ioctl($fh, 0xae00, 0);
1081
1082     return $kvm_api_version;
1083 }
1084
1085 my $kvm_user_version = {};
1086 my $kvm_mtime = {};
1087
1088 sub kvm_user_version {
1089     my ($binary) = @_;
1090
1091     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1092     my $st = stat($binary);
1093
1094     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1095     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1096         $cachedmtime == $st->mtime;
1097
1098     $kvm_user_version->{$binary} = 'unknown';
1099     $kvm_mtime->{$binary} = $st->mtime;
1100
1101     my $code = sub {
1102         my $line = shift;
1103         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1104             $kvm_user_version->{$binary} = $2;
1105         }
1106     };
1107
1108     eval { run_command([$binary, '--version'], outfunc => $code); };
1109     warn $@ if $@;
1110
1111     return $kvm_user_version->{$binary};
1112
1113 }
1114
1115 sub kernel_has_vhost_net {
1116     return -c '/dev/vhost-net';
1117 }
1118
1119 sub option_exists {
1120     my $key = shift;
1121     return defined($confdesc->{$key});
1122 }
1123
1124 my $cdrom_path;
1125 sub get_cdrom_path {
1126
1127     return  $cdrom_path if $cdrom_path;
1128
1129     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1130     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1131     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1132 }
1133
1134 sub get_iso_path {
1135     my ($storecfg, $vmid, $cdrom) = @_;
1136
1137     if ($cdrom eq 'cdrom') {
1138         return get_cdrom_path();
1139     } elsif ($cdrom eq 'none') {
1140         return '';
1141     } elsif ($cdrom =~ m|^/|) {
1142         return $cdrom;
1143     } else {
1144         return PVE::Storage::path($storecfg, $cdrom);
1145     }
1146 }
1147
1148 # try to convert old style file names to volume IDs
1149 sub filename_to_volume_id {
1150     my ($vmid, $file, $media) = @_;
1151
1152      if (!($file eq 'none' || $file eq 'cdrom' ||
1153           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1154
1155         return undef if $file =~ m|/|;
1156
1157         if ($media && $media eq 'cdrom') {
1158             $file = "local:iso/$file";
1159         } else {
1160             $file = "local:$vmid/$file";
1161         }
1162     }
1163
1164     return $file;
1165 }
1166
1167 sub verify_media_type {
1168     my ($opt, $vtype, $media) = @_;
1169
1170     return if !$media;
1171
1172     my $etype;
1173     if ($media eq 'disk') {
1174         $etype = 'images';
1175     } elsif ($media eq 'cdrom') {
1176         $etype = 'iso';
1177     } else {
1178         die "internal error";
1179     }
1180
1181     return if ($vtype eq $etype);
1182
1183     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1184 }
1185
1186 sub cleanup_drive_path {
1187     my ($opt, $storecfg, $drive) = @_;
1188
1189     # try to convert filesystem paths to volume IDs
1190
1191     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1192         ($drive->{file} !~ m|^/dev/.+|) &&
1193         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1194         ($drive->{file} !~ m/^\d+$/)) {
1195         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1196         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1197         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1198         verify_media_type($opt, $vtype, $drive->{media});
1199         $drive->{file} = $volid;
1200     }
1201
1202     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1203 }
1204
1205 sub parse_hotplug_features {
1206     my ($data) = @_;
1207
1208     my $res = {};
1209
1210     return $res if $data eq '0';
1211
1212     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1213
1214     foreach my $feature (PVE::Tools::split_list($data)) {
1215         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1216             $res->{$1} = 1;
1217         } else {
1218             die "invalid hotplug feature '$feature'\n";
1219         }
1220     }
1221     return $res;
1222 }
1223
1224 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1225 sub pve_verify_hotplug_features {
1226     my ($value, $noerr) = @_;
1227
1228     return $value if parse_hotplug_features($value);
1229
1230     return undef if $noerr;
1231
1232     die "unable to parse hotplug option\n";
1233 }
1234
1235 sub scsi_inquiry {
1236     my($fh, $noerr) = @_;
1237
1238     my $SG_IO = 0x2285;
1239     my $SG_GET_VERSION_NUM = 0x2282;
1240
1241     my $versionbuf = "\x00" x 8;
1242     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1243     if (!$ret) {
1244         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1245         return undef;
1246     }
1247     my $version = unpack("I", $versionbuf);
1248     if ($version < 30000) {
1249         die "scsi generic interface too old\n"  if !$noerr;
1250         return undef;
1251     }
1252
1253     my $buf = "\x00" x 36;
1254     my $sensebuf = "\x00" x 8;
1255     my $cmd = pack("C x3 C x1", 0x12, 36);
1256
1257     # see /usr/include/scsi/sg.h
1258     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1259
1260     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1261                       length($sensebuf), 0, length($buf), $buf,
1262                       $cmd, $sensebuf, 6000);
1263
1264     $ret = ioctl($fh, $SG_IO, $packet);
1265     if (!$ret) {
1266         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1267         return undef;
1268     }
1269
1270     my @res = unpack($sg_io_hdr_t, $packet);
1271     if ($res[17] || $res[18]) {
1272         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1273         return undef;
1274     }
1275
1276     my $res = {};
1277     (my $byte0, my $byte1, $res->{vendor},
1278      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1279
1280     $res->{removable} = $byte1 & 128 ? 1 : 0;
1281     $res->{type} = $byte0 & 31;
1282
1283     return $res;
1284 }
1285
1286 sub path_is_scsi {
1287     my ($path) = @_;
1288
1289     my $fh = IO::File->new("+<$path") || return undef;
1290     my $res = scsi_inquiry($fh, 1);
1291     close($fh);
1292
1293     return $res;
1294 }
1295
1296 sub print_tabletdevice_full {
1297     my ($conf, $arch) = @_;
1298
1299     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1300
1301     # we use uhci for old VMs because tablet driver was buggy in older qemu
1302     my $usbbus;
1303     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1304         $usbbus = 'ehci';
1305     } else {
1306         $usbbus = 'uhci';
1307     }
1308
1309     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1310 }
1311
1312 sub print_keyboarddevice_full {
1313     my ($conf, $arch, $machine) = @_;
1314
1315     return undef if $arch ne 'aarch64';
1316
1317     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1318 }
1319
1320 sub print_drivedevice_full {
1321     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1322
1323     my $device = '';
1324     my $maxdev = 0;
1325
1326     if ($drive->{interface} eq 'virtio') {
1327         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1328         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1329         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1330     } elsif ($drive->{interface} eq 'scsi') {
1331
1332         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1333         my $unit = $drive->{index} % $maxdev;
1334         my $devicetype = 'hd';
1335         my $path = '';
1336         if (drive_is_cdrom($drive)) {
1337             $devicetype = 'cd';
1338         } else {
1339             if ($drive->{file} =~ m|^/|) {
1340                 $path = $drive->{file};
1341                 if (my $info = path_is_scsi($path)) {
1342                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1343                         $devicetype = 'block';
1344                     } elsif ($info->{type} == 1) { # tape
1345                         $devicetype = 'generic';
1346                     }
1347                 }
1348             } else {
1349                  $path = PVE::Storage::path($storecfg, $drive->{file});
1350             }
1351
1352             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1353             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1354             if ($path =~ m/^iscsi\:\/\// &&
1355                !min_version($version, 4, 1)) {
1356                 $devicetype = 'generic';
1357             }
1358         }
1359
1360         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1361             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1362         } else {
1363             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1364         }
1365
1366         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1367             $device .= ",rotation_rate=1";
1368         }
1369         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1370
1371     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1372         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1373         my $controller = int($drive->{index} / $maxdev);
1374         my $unit = $drive->{index} % $maxdev;
1375         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1376
1377         $device = "ide-$devicetype";
1378         if ($drive->{interface} eq 'ide') {
1379             $device .= ",bus=ide.$controller,unit=$unit";
1380         } else {
1381             $device .= ",bus=ahci$controller.$unit";
1382         }
1383         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1384
1385         if ($devicetype eq 'hd') {
1386             if (my $model = $drive->{model}) {
1387                 $model = URI::Escape::uri_unescape($model);
1388                 $device .= ",model=$model";
1389             }
1390             if ($drive->{ssd}) {
1391                 $device .= ",rotation_rate=1";
1392             }
1393         }
1394         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1395     } elsif ($drive->{interface} eq 'usb') {
1396         die "implement me";
1397         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1398     } else {
1399         die "unsupported interface type";
1400     }
1401
1402     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1403
1404     if (my $serial = $drive->{serial}) {
1405         $serial = URI::Escape::uri_unescape($serial);
1406         $device .= ",serial=$serial";
1407     }
1408
1409
1410     return $device;
1411 }
1412
1413 sub get_initiator_name {
1414     my $initiator;
1415
1416     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1417     while (defined(my $line = <$fh>)) {
1418         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1419         $initiator = $1;
1420         last;
1421     }
1422     $fh->close();
1423
1424     return $initiator;
1425 }
1426
1427 sub print_drive_commandline_full {
1428     my ($storecfg, $vmid, $drive) = @_;
1429
1430     my $path;
1431     my $volid = $drive->{file};
1432     my $format;
1433
1434     if (drive_is_cdrom($drive)) {
1435         $path = get_iso_path($storecfg, $vmid, $volid);
1436     } else {
1437         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1438         if ($storeid) {
1439             $path = PVE::Storage::path($storecfg, $volid);
1440             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1441             $format = qemu_img_format($scfg, $volname);
1442         } else {
1443             $path = $volid;
1444             $format = "raw";
1445         }
1446    }
1447
1448     my $opts = '';
1449     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1450     foreach my $o (@qemu_drive_options) {
1451         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1452     }
1453
1454     # snapshot only accepts on|off
1455     if (defined($drive->{snapshot})) {
1456         my $v = $drive->{snapshot} ? 'on' : 'off';
1457         $opts .= ",snapshot=$v";
1458     }
1459
1460     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1461         my ($dir, $qmpname) = @$type;
1462         if (my $v = $drive->{"mbps$dir"}) {
1463             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1464         }
1465         if (my $v = $drive->{"mbps${dir}_max"}) {
1466             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1467         }
1468         if (my $v = $drive->{"bps${dir}_max_length"}) {
1469             $opts .= ",throttling.bps$qmpname-max-length=$v";
1470         }
1471         if (my $v = $drive->{"iops${dir}"}) {
1472             $opts .= ",throttling.iops$qmpname=$v";
1473         }
1474         if (my $v = $drive->{"iops${dir}_max"}) {
1475             $opts .= ",throttling.iops$qmpname-max=$v";
1476         }
1477         if (my $v = $drive->{"iops${dir}_max_length"}) {
1478             $opts .= ",throttling.iops$qmpname-max-length=$v";
1479         }
1480     }
1481
1482     $opts .= ",format=$format" if $format && !$drive->{format};
1483
1484     my $cache_direct = 0;
1485
1486     if (my $cache = $drive->{cache}) {
1487         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1488     } elsif (!drive_is_cdrom($drive)) {
1489         $opts .= ",cache=none";
1490         $cache_direct = 1;
1491     }
1492
1493     # aio native works only with O_DIRECT
1494     if (!$drive->{aio}) {
1495         if($cache_direct) {
1496             $opts .= ",aio=native";
1497         } else {
1498             $opts .= ",aio=threads";
1499         }
1500     }
1501
1502     if (!drive_is_cdrom($drive)) {
1503         my $detectzeroes;
1504         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1505             $detectzeroes = 'off';
1506         } elsif ($drive->{discard}) {
1507             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1508         } else {
1509             # This used to be our default with discard not being specified:
1510             $detectzeroes = 'on';
1511         }
1512         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1513     }
1514
1515     my $pathinfo = $path ? "file=$path," : '';
1516
1517     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1518 }
1519
1520 sub print_netdevice_full {
1521     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1522
1523     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1524
1525     my $device = $net->{model};
1526     if ($net->{model} eq 'virtio') {
1527          $device = 'virtio-net-pci';
1528      };
1529
1530     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1531     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1532     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1533         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1534         my $vectors = $net->{queues} * 2 + 2;
1535         $tmpstr .= ",vectors=$vectors,mq=on";
1536     }
1537     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1538
1539     if (my $mtu = $net->{mtu}) {
1540         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1541             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1542             if ($mtu == 1) {
1543                  $mtu = $bridge_mtu;
1544             } elsif ($mtu < 576) {
1545                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1546             } elsif ($mtu > $bridge_mtu) {
1547                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1548             }
1549             $tmpstr .= ",host_mtu=$mtu";
1550         } else {
1551             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1552         }
1553     }
1554
1555     if ($use_old_bios_files) {
1556         my $romfile;
1557         if ($device eq 'virtio-net-pci') {
1558             $romfile = 'pxe-virtio.rom';
1559         } elsif ($device eq 'e1000') {
1560             $romfile = 'pxe-e1000.rom';
1561         } elsif ($device eq 'ne2k') {
1562             $romfile = 'pxe-ne2k_pci.rom';
1563         } elsif ($device eq 'pcnet') {
1564             $romfile = 'pxe-pcnet.rom';
1565         } elsif ($device eq 'rtl8139') {
1566             $romfile = 'pxe-rtl8139.rom';
1567         }
1568         $tmpstr .= ",romfile=$romfile" if $romfile;
1569     }
1570
1571     return $tmpstr;
1572 }
1573
1574 sub print_netdev_full {
1575     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1576
1577     my $i = '';
1578     if ($netid =~ m/^net(\d+)$/) {
1579         $i = int($1);
1580     }
1581
1582     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1583
1584     my $ifname = "tap${vmid}i$i";
1585
1586     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1587     die "interface name '$ifname' is too long (max 15 character)\n"
1588         if length($ifname) >= 16;
1589
1590     my $vhostparam = '';
1591     if (is_native($arch)) {
1592         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1593     }
1594
1595     my $vmname = $conf->{name} || "vm$vmid";
1596
1597     my $netdev = "";
1598     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1599
1600     if ($net->{bridge}) {
1601         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1602     } else {
1603         $netdev = "type=user,id=$netid,hostname=$vmname";
1604     }
1605
1606     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1607
1608     return $netdev;
1609 }
1610
1611 my $vga_map = {
1612     'cirrus' => 'cirrus-vga',
1613     'std' => 'VGA',
1614     'vmware' => 'vmware-svga',
1615     'virtio' => 'virtio-vga',
1616 };
1617
1618 sub print_vga_device {
1619     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1620
1621     my $type = $vga_map->{$vga->{type}};
1622     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1623         $type = 'virtio-gpu';
1624     }
1625     my $vgamem_mb = $vga->{memory};
1626
1627     my $max_outputs = '';
1628     if ($qxlnum) {
1629         $type = $id ? 'qxl' : 'qxl-vga';
1630
1631         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1632             # set max outputs so linux can have up to 4 qxl displays with one device
1633             if (min_version($machine_version, 4, 1)) {
1634                 $max_outputs = ",max_outputs=4";
1635             }
1636         }
1637     }
1638
1639     die "no devicetype for $vga->{type}\n" if !$type;
1640
1641     my $memory = "";
1642     if ($vgamem_mb) {
1643         if ($vga->{type} eq 'virtio') {
1644             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1645             $memory = ",max_hostmem=$bytes";
1646         } elsif ($qxlnum) {
1647             # from https://www.spice-space.org/multiple-monitors.html
1648             $memory = ",vgamem_mb=$vga->{memory}";
1649             my $ram = $vgamem_mb * 4;
1650             my $vram = $vgamem_mb * 2;
1651             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1652         } else {
1653             $memory = ",vgamem_mb=$vga->{memory}";
1654         }
1655     } elsif ($qxlnum && $id) {
1656         $memory = ",ram_size=67108864,vram_size=33554432";
1657     }
1658
1659     my $edidoff = "";
1660     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1661         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1662     }
1663
1664     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1665     my $vgaid = "vga" . ($id // '');
1666     my $pciaddr;
1667
1668     if ($q35 && $vgaid eq 'vga') {
1669         # the first display uses pcie.0 bus on q35 machines
1670         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1671     } else {
1672         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1673     }
1674
1675     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1676 }
1677
1678 sub parse_number_sets {
1679     my ($set) = @_;
1680     my $res = [];
1681     foreach my $part (split(/;/, $set)) {
1682         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1683             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1684             push @$res, [ $1, $2 ];
1685         } else {
1686             die "invalid range: $part\n";
1687         }
1688     }
1689     return $res;
1690 }
1691
1692 sub parse_numa {
1693     my ($data) = @_;
1694
1695     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1696     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1697     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1698     return $res;
1699 }
1700
1701 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1702 sub parse_net {
1703     my ($data) = @_;
1704
1705     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1706     if ($@) {
1707         warn $@;
1708         return undef;
1709     }
1710     if (!defined($res->{macaddr})) {
1711         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1712         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1713     }
1714     return $res;
1715 }
1716
1717 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1718 sub parse_ipconfig {
1719     my ($data) = @_;
1720
1721     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1722     if ($@) {
1723         warn $@;
1724         return undef;
1725     }
1726
1727     if ($res->{gw} && !$res->{ip}) {
1728         warn 'gateway specified without specifying an IP address';
1729         return undef;
1730     }
1731     if ($res->{gw6} && !$res->{ip6}) {
1732         warn 'IPv6 gateway specified without specifying an IPv6 address';
1733         return undef;
1734     }
1735     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1736         warn 'gateway specified together with DHCP';
1737         return undef;
1738     }
1739     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1740         # gw6 + auto/dhcp
1741         warn "IPv6 gateway specified together with $res->{ip6} address";
1742         return undef;
1743     }
1744
1745     if (!$res->{ip} && !$res->{ip6}) {
1746         return { ip => 'dhcp', ip6 => 'dhcp' };
1747     }
1748
1749     return $res;
1750 }
1751
1752 sub print_net {
1753     my $net = shift;
1754
1755     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1756 }
1757
1758 sub add_random_macs {
1759     my ($settings) = @_;
1760
1761     foreach my $opt (keys %$settings) {
1762         next if $opt !~ m/^net(\d+)$/;
1763         my $net = parse_net($settings->{$opt});
1764         next if !$net;
1765         $settings->{$opt} = print_net($net);
1766     }
1767 }
1768
1769 sub vm_is_volid_owner {
1770     my ($storecfg, $vmid, $volid) = @_;
1771
1772     if ($volid !~  m|^/|) {
1773         my ($path, $owner);
1774         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1775         if ($owner && ($owner == $vmid)) {
1776             return 1;
1777         }
1778     }
1779
1780     return undef;
1781 }
1782
1783 sub vmconfig_register_unused_drive {
1784     my ($storecfg, $vmid, $conf, $drive) = @_;
1785
1786     if (drive_is_cloudinit($drive)) {
1787         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1788         warn $@ if $@;
1789     } elsif (!drive_is_cdrom($drive)) {
1790         my $volid = $drive->{file};
1791         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1792             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1793         }
1794     }
1795 }
1796
1797 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1798 my $smbios1_fmt = {
1799     uuid => {
1800         type => 'string',
1801         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1802         format_description => 'UUID',
1803         description => "Set SMBIOS1 UUID.",
1804         optional => 1,
1805     },
1806     version => {
1807         type => 'string',
1808         pattern => '[A-Za-z0-9+\/]+={0,2}',
1809         format_description => 'Base64 encoded string',
1810         description => "Set SMBIOS1 version.",
1811         optional => 1,
1812     },
1813     serial => {
1814         type => 'string',
1815         pattern => '[A-Za-z0-9+\/]+={0,2}',
1816         format_description => 'Base64 encoded string',
1817         description => "Set SMBIOS1 serial number.",
1818         optional => 1,
1819     },
1820     manufacturer => {
1821         type => 'string',
1822         pattern => '[A-Za-z0-9+\/]+={0,2}',
1823         format_description => 'Base64 encoded string',
1824         description => "Set SMBIOS1 manufacturer.",
1825         optional => 1,
1826     },
1827     product => {
1828         type => 'string',
1829         pattern => '[A-Za-z0-9+\/]+={0,2}',
1830         format_description => 'Base64 encoded string',
1831         description => "Set SMBIOS1 product ID.",
1832         optional => 1,
1833     },
1834     sku => {
1835         type => 'string',
1836         pattern => '[A-Za-z0-9+\/]+={0,2}',
1837         format_description => 'Base64 encoded string',
1838         description => "Set SMBIOS1 SKU string.",
1839         optional => 1,
1840     },
1841     family => {
1842         type => 'string',
1843         pattern => '[A-Za-z0-9+\/]+={0,2}',
1844         format_description => 'Base64 encoded string',
1845         description => "Set SMBIOS1 family string.",
1846         optional => 1,
1847     },
1848     base64 => {
1849         type => 'boolean',
1850         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1851         optional => 1,
1852     },
1853 };
1854
1855 sub parse_smbios1 {
1856     my ($data) = @_;
1857
1858     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1859     warn $@ if $@;
1860     return $res;
1861 }
1862
1863 sub print_smbios1 {
1864     my ($smbios1) = @_;
1865     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1866 }
1867
1868 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1869
1870 sub parse_watchdog {
1871     my ($value) = @_;
1872
1873     return undef if !$value;
1874
1875     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1876     warn $@ if $@;
1877     return $res;
1878 }
1879
1880 sub parse_guest_agent {
1881     my ($value) = @_;
1882
1883     return {} if !defined($value->{agent});
1884
1885     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1886     warn $@ if $@;
1887
1888     # if the agent is disabled ignore the other potentially set properties
1889     return {} if !$res->{enabled};
1890     return $res;
1891 }
1892
1893 sub parse_vga {
1894     my ($value) = @_;
1895
1896     return {} if !$value;
1897     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1898     warn $@ if $@;
1899     return $res;
1900 }
1901
1902 sub parse_rng {
1903     my ($value) = @_;
1904
1905     return undef if !$value;
1906
1907     my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1908     warn $@ if $@;
1909     return $res;
1910 }
1911
1912 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1913 sub verify_usb_device {
1914     my ($value, $noerr) = @_;
1915
1916     return $value if parse_usb_device($value);
1917
1918     return undef if $noerr;
1919
1920     die "unable to parse usb device\n";
1921 }
1922
1923 # add JSON properties for create and set function
1924 sub json_config_properties {
1925     my $prop = shift;
1926
1927     foreach my $opt (keys %$confdesc) {
1928         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1929             $opt eq 'runningmachine' || $opt eq 'runningcpu';
1930         $prop->{$opt} = $confdesc->{$opt};
1931     }
1932
1933     return $prop;
1934 }
1935
1936 # return copy of $confdesc_cloudinit to generate documentation
1937 sub cloudinit_config_properties {
1938
1939     return dclone($confdesc_cloudinit);
1940 }
1941
1942 sub check_type {
1943     my ($key, $value) = @_;
1944
1945     die "unknown setting '$key'\n" if !$confdesc->{$key};
1946
1947     my $type = $confdesc->{$key}->{type};
1948
1949     if (!defined($value)) {
1950         die "got undefined value\n";
1951     }
1952
1953     if ($value =~ m/[\n\r]/) {
1954         die "property contains a line feed\n";
1955     }
1956
1957     if ($type eq 'boolean') {
1958         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1959         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1960         die "type check ('boolean') failed - got '$value'\n";
1961     } elsif ($type eq 'integer') {
1962         return int($1) if $value =~ m/^(\d+)$/;
1963         die "type check ('integer') failed - got '$value'\n";
1964     } elsif ($type eq 'number') {
1965         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1966         die "type check ('number') failed - got '$value'\n";
1967     } elsif ($type eq 'string') {
1968         if (my $fmt = $confdesc->{$key}->{format}) {
1969             PVE::JSONSchema::check_format($fmt, $value);
1970             return $value;
1971         }
1972         $value =~ s/^\"(.*)\"$/$1/;
1973         return $value;
1974     } else {
1975         die "internal error"
1976     }
1977 }
1978
1979 sub destroy_vm {
1980     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1981
1982     my $conf = PVE::QemuConfig->load_config($vmid);
1983
1984     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1985
1986     if ($conf->{template}) {
1987         # check if any base image is still used by a linked clone
1988         PVE::QemuConfig->foreach_volume($conf, sub {
1989                 my ($ds, $drive) = @_;
1990                 return if drive_is_cdrom($drive);
1991
1992                 my $volid = $drive->{file};
1993                 return if !$volid || $volid =~ m|^/|;
1994
1995                 die "base volume '$volid' is still in use by linked cloned\n"
1996                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
1997
1998         });
1999     }
2000
2001     # only remove disks owned by this VM
2002     PVE::QemuConfig->foreach_volume($conf, sub {
2003         my ($ds, $drive) = @_;
2004         return if drive_is_cdrom($drive, 1);
2005
2006         my $volid = $drive->{file};
2007         return if !$volid || $volid =~ m|^/|;
2008
2009         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2010         return if !$path || !$owner || ($owner != $vmid);
2011
2012         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2013         warn "Could not remove disk '$volid', check manually: $@" if $@;
2014     });
2015
2016     # also remove unused disk
2017     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2018     PVE::Storage::foreach_volid($vmdisks, sub {
2019         my ($volid, $sid, $volname, $d) = @_;
2020         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2021         warn $@ if $@;
2022     });
2023
2024     if (defined $replacement_conf) {
2025         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2026     } else {
2027         PVE::QemuConfig->destroy_config($vmid);
2028     }
2029 }
2030
2031 sub parse_vm_config {
2032     my ($filename, $raw) = @_;
2033
2034     return undef if !defined($raw);
2035
2036     my $res = {
2037         digest => Digest::SHA::sha1_hex($raw),
2038         snapshots => {},
2039         pending => {},
2040     };
2041
2042     $filename =~ m|/qemu-server/(\d+)\.conf$|
2043         || die "got strange filename '$filename'";
2044
2045     my $vmid = $1;
2046
2047     my $conf = $res;
2048     my $descr;
2049     my $section = '';
2050
2051     my @lines = split(/\n/, $raw);
2052     foreach my $line (@lines) {
2053         next if $line =~ m/^\s*$/;
2054
2055         if ($line =~ m/^\[PENDING\]\s*$/i) {
2056             $section = 'pending';
2057             if (defined($descr)) {
2058                 $descr =~ s/\s+$//;
2059                 $conf->{description} = $descr;
2060             }
2061             $descr = undef;
2062             $conf = $res->{$section} = {};
2063             next;
2064
2065         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2066             $section = $1;
2067             if (defined($descr)) {
2068                 $descr =~ s/\s+$//;
2069                 $conf->{description} = $descr;
2070             }
2071             $descr = undef;
2072             $conf = $res->{snapshots}->{$section} = {};
2073             next;
2074         }
2075
2076         if ($line =~ m/^\#(.*)\s*$/) {
2077             $descr = '' if !defined($descr);
2078             $descr .= PVE::Tools::decode_text($1) . "\n";
2079             next;
2080         }
2081
2082         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2083             $descr = '' if !defined($descr);
2084             $descr .= PVE::Tools::decode_text($2);
2085         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2086             $conf->{snapstate} = $1;
2087         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2088             my $key = $1;
2089             my $value = $2;
2090             $conf->{$key} = $value;
2091         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2092             my $value = $1;
2093             if ($section eq 'pending') {
2094                 $conf->{delete} = $value; # we parse this later
2095             } else {
2096                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2097             }
2098         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2099             my $key = $1;
2100             my $value = $2;
2101             eval { $value = check_type($key, $value); };
2102             if ($@) {
2103                 warn "vm $vmid - unable to parse value of '$key' - $@";
2104             } else {
2105                 $key = 'ide2' if $key eq 'cdrom';
2106                 my $fmt = $confdesc->{$key}->{format};
2107                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2108                     my $v = parse_drive($key, $value);
2109                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2110                         $v->{file} = $volid;
2111                         $value = print_drive($v);
2112                     } else {
2113                         warn "vm $vmid - unable to parse value of '$key'\n";
2114                         next;
2115                     }
2116                 }
2117
2118                 $conf->{$key} = $value;
2119             }
2120         }
2121     }
2122
2123     if (defined($descr)) {
2124         $descr =~ s/\s+$//;
2125         $conf->{description} = $descr;
2126     }
2127     delete $res->{snapstate}; # just to be sure
2128
2129     return $res;
2130 }
2131
2132 sub write_vm_config {
2133     my ($filename, $conf) = @_;
2134
2135     delete $conf->{snapstate}; # just to be sure
2136
2137     if ($conf->{cdrom}) {
2138         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2139         $conf->{ide2} = $conf->{cdrom};
2140         delete $conf->{cdrom};
2141     }
2142
2143     # we do not use 'smp' any longer
2144     if ($conf->{sockets}) {
2145         delete $conf->{smp};
2146     } elsif ($conf->{smp}) {
2147         $conf->{sockets} = $conf->{smp};
2148         delete $conf->{cores};
2149         delete $conf->{smp};
2150     }
2151
2152     my $used_volids = {};
2153
2154     my $cleanup_config = sub {
2155         my ($cref, $pending, $snapname) = @_;
2156
2157         foreach my $key (keys %$cref) {
2158             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2159                 $key eq 'snapstate' || $key eq 'pending';
2160             my $value = $cref->{$key};
2161             if ($key eq 'delete') {
2162                 die "propertry 'delete' is only allowed in [PENDING]\n"
2163                     if !$pending;
2164                 # fixme: check syntax?
2165                 next;
2166             }
2167             eval { $value = check_type($key, $value); };
2168             die "unable to parse value of '$key' - $@" if $@;
2169
2170             $cref->{$key} = $value;
2171
2172             if (!$snapname && is_valid_drivename($key)) {
2173                 my $drive = parse_drive($key, $value);
2174                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2175             }
2176         }
2177     };
2178
2179     &$cleanup_config($conf);
2180
2181     &$cleanup_config($conf->{pending}, 1);
2182
2183     foreach my $snapname (keys %{$conf->{snapshots}}) {
2184         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2185         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2186     }
2187
2188     # remove 'unusedX' settings if we re-add a volume
2189     foreach my $key (keys %$conf) {
2190         my $value = $conf->{$key};
2191         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2192             delete $conf->{$key};
2193         }
2194     }
2195
2196     my $generate_raw_config = sub {
2197         my ($conf, $pending) = @_;
2198
2199         my $raw = '';
2200
2201         # add description as comment to top of file
2202         if (defined(my $descr = $conf->{description})) {
2203             if ($descr) {
2204                 foreach my $cl (split(/\n/, $descr)) {
2205                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2206                 }
2207             } else {
2208                 $raw .= "#\n" if $pending;
2209             }
2210         }
2211
2212         foreach my $key (sort keys %$conf) {
2213             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2214             $raw .= "$key: $conf->{$key}\n";
2215         }
2216         return $raw;
2217     };
2218
2219     my $raw = &$generate_raw_config($conf);
2220
2221     if (scalar(keys %{$conf->{pending}})){
2222         $raw .= "\n[PENDING]\n";
2223         $raw .= &$generate_raw_config($conf->{pending}, 1);
2224     }
2225
2226     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2227         $raw .= "\n[$snapname]\n";
2228         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2229     }
2230
2231     return $raw;
2232 }
2233
2234 sub load_defaults {
2235
2236     my $res = {};
2237
2238     # we use static defaults from our JSON schema configuration
2239     foreach my $key (keys %$confdesc) {
2240         if (defined(my $default = $confdesc->{$key}->{default})) {
2241             $res->{$key} = $default;
2242         }
2243     }
2244
2245     return $res;
2246 }
2247
2248 sub config_list {
2249     my $vmlist = PVE::Cluster::get_vmlist();
2250     my $res = {};
2251     return $res if !$vmlist || !$vmlist->{ids};
2252     my $ids = $vmlist->{ids};
2253     my $nodename = nodename();
2254
2255     foreach my $vmid (keys %$ids) {
2256         my $d = $ids->{$vmid};
2257         next if !$d->{node} || $d->{node} ne $nodename;
2258         next if !$d->{type} || $d->{type} ne 'qemu';
2259         $res->{$vmid}->{exists} = 1;
2260     }
2261     return $res;
2262 }
2263
2264 # test if VM uses local resources (to prevent migration)
2265 sub check_local_resources {
2266     my ($conf, $noerr) = @_;
2267
2268     my @loc_res = ();
2269
2270     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2271     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2272
2273     push @loc_res, "ivshmem" if $conf->{ivshmem};
2274
2275     foreach my $k (keys %$conf) {
2276         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2277         # sockets are safe: they will recreated be on the target side post-migrate
2278         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2279         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2280     }
2281
2282     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2283
2284     return \@loc_res;
2285 }
2286
2287 # check if used storages are available on all nodes (use by migrate)
2288 sub check_storage_availability {
2289     my ($storecfg, $conf, $node) = @_;
2290
2291     PVE::QemuConfig->foreach_volume($conf, sub {
2292         my ($ds, $drive) = @_;
2293
2294         my $volid = $drive->{file};
2295         return if !$volid;
2296
2297         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2298         return if !$sid;
2299
2300         # check if storage is available on both nodes
2301         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2302         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2303    });
2304 }
2305
2306 # list nodes where all VM images are available (used by has_feature API)
2307 sub shared_nodes {
2308     my ($conf, $storecfg) = @_;
2309
2310     my $nodelist = PVE::Cluster::get_nodelist();
2311     my $nodehash = { map { $_ => 1 } @$nodelist };
2312     my $nodename = nodename();
2313
2314     PVE::QemuConfig->foreach_volume($conf, sub {
2315         my ($ds, $drive) = @_;
2316
2317         my $volid = $drive->{file};
2318         return if !$volid;
2319
2320         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2321         if ($storeid) {
2322             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2323             if ($scfg->{disable}) {
2324                 $nodehash = {};
2325             } elsif (my $avail = $scfg->{nodes}) {
2326                 foreach my $node (keys %$nodehash) {
2327                     delete $nodehash->{$node} if !$avail->{$node};
2328                 }
2329             } elsif (!$scfg->{shared}) {
2330                 foreach my $node (keys %$nodehash) {
2331                     delete $nodehash->{$node} if $node ne $nodename
2332                 }
2333             }
2334         }
2335     });
2336
2337     return $nodehash
2338 }
2339
2340 sub check_local_storage_availability {
2341     my ($conf, $storecfg) = @_;
2342
2343     my $nodelist = PVE::Cluster::get_nodelist();
2344     my $nodehash = { map { $_ => {} } @$nodelist };
2345
2346     PVE::QemuConfig->foreach_volume($conf, sub {
2347         my ($ds, $drive) = @_;
2348
2349         my $volid = $drive->{file};
2350         return if !$volid;
2351
2352         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2353         if ($storeid) {
2354             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2355
2356             if ($scfg->{disable}) {
2357                 foreach my $node (keys %$nodehash) {
2358                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2359                 }
2360             } elsif (my $avail = $scfg->{nodes}) {
2361                 foreach my $node (keys %$nodehash) {
2362                     if (!$avail->{$node}) {
2363                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2364                     }
2365                 }
2366             }
2367         }
2368     });
2369
2370     foreach my $node (values %$nodehash) {
2371         if (my $unavail = $node->{unavailable_storages}) {
2372             $node->{unavailable_storages} = [ sort keys %$unavail ];
2373         }
2374     }
2375
2376     return $nodehash
2377 }
2378
2379 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2380 sub check_running {
2381     my ($vmid, $nocheck, $node) = @_;
2382
2383     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2384     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2385 }
2386
2387 sub vzlist {
2388
2389     my $vzlist = config_list();
2390
2391     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2392
2393     while (defined(my $de = $fd->read)) {
2394         next if $de !~ m/^(\d+)\.pid$/;
2395         my $vmid = $1;
2396         next if !defined($vzlist->{$vmid});
2397         if (my $pid = check_running($vmid)) {
2398             $vzlist->{$vmid}->{pid} = $pid;
2399         }
2400     }
2401
2402     return $vzlist;
2403 }
2404
2405 our $vmstatus_return_properties = {
2406     vmid => get_standard_option('pve-vmid'),
2407     status => {
2408         description => "Qemu process status.",
2409         type => 'string',
2410         enum => ['stopped', 'running'],
2411     },
2412     maxmem => {
2413         description => "Maximum memory in bytes.",
2414         type => 'integer',
2415         optional => 1,
2416         renderer => 'bytes',
2417     },
2418     maxdisk => {
2419         description => "Root disk size in bytes.",
2420         type => 'integer',
2421         optional => 1,
2422         renderer => 'bytes',
2423     },
2424     name => {
2425         description => "VM name.",
2426         type => 'string',
2427         optional => 1,
2428     },
2429     qmpstatus => {
2430         description => "Qemu QMP agent status.",
2431         type => 'string',
2432         optional => 1,
2433     },
2434     pid => {
2435         description => "PID of running qemu process.",
2436         type => 'integer',
2437         optional => 1,
2438     },
2439     uptime => {
2440         description => "Uptime.",
2441         type => 'integer',
2442         optional => 1,
2443         renderer => 'duration',
2444     },
2445     cpus => {
2446         description => "Maximum usable CPUs.",
2447         type => 'number',
2448         optional => 1,
2449     },
2450     lock => {
2451         description => "The current config lock, if any.",
2452         type => 'string',
2453         optional => 1,
2454     },
2455     tags => {
2456         description  => "The current configured tags, if any",
2457         type => 'string',
2458         optional => 1,
2459     },
2460 };
2461
2462 my $last_proc_pid_stat;
2463
2464 # get VM status information
2465 # This must be fast and should not block ($full == false)
2466 # We only query KVM using QMP if $full == true (this can be slow)
2467 sub vmstatus {
2468     my ($opt_vmid, $full) = @_;
2469
2470     my $res = {};
2471
2472     my $storecfg = PVE::Storage::config();
2473
2474     my $list = vzlist();
2475     my $defaults = load_defaults();
2476
2477     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2478
2479     my $cpucount = $cpuinfo->{cpus} || 1;
2480
2481     foreach my $vmid (keys %$list) {
2482         next if $opt_vmid && ($vmid ne $opt_vmid);
2483
2484         my $conf = PVE::QemuConfig->load_config($vmid);
2485
2486         my $d = { vmid => $vmid };
2487         $d->{pid} = $list->{$vmid}->{pid};
2488
2489         # fixme: better status?
2490         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2491
2492         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2493         if (defined($size)) {
2494             $d->{disk} = 0; # no info available
2495             $d->{maxdisk} = $size;
2496         } else {
2497             $d->{disk} = 0;
2498             $d->{maxdisk} = 0;
2499         }
2500
2501         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2502             * ($conf->{cores} || $defaults->{cores});
2503         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2504         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2505
2506         $d->{name} = $conf->{name} || "VM $vmid";
2507         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2508             : $defaults->{memory}*(1024*1024);
2509
2510         if ($conf->{balloon}) {
2511             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2512             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2513                 : $defaults->{shares};
2514         }
2515
2516         $d->{uptime} = 0;
2517         $d->{cpu} = 0;
2518         $d->{mem} = 0;
2519
2520         $d->{netout} = 0;
2521         $d->{netin} = 0;
2522
2523         $d->{diskread} = 0;
2524         $d->{diskwrite} = 0;
2525
2526         $d->{template} = PVE::QemuConfig->is_template($conf);
2527
2528         $d->{serial} = 1 if conf_has_serial($conf);
2529         $d->{lock} = $conf->{lock} if $conf->{lock};
2530         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2531
2532         $res->{$vmid} = $d;
2533     }
2534
2535     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2536     foreach my $dev (keys %$netdev) {
2537         next if $dev !~ m/^tap([1-9]\d*)i/;
2538         my $vmid = $1;
2539         my $d = $res->{$vmid};
2540         next if !$d;
2541
2542         $d->{netout} += $netdev->{$dev}->{receive};
2543         $d->{netin} += $netdev->{$dev}->{transmit};
2544
2545         if ($full) {
2546             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2547             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2548         }
2549
2550     }
2551
2552     my $ctime = gettimeofday;
2553
2554     foreach my $vmid (keys %$list) {
2555
2556         my $d = $res->{$vmid};
2557         my $pid = $d->{pid};
2558         next if !$pid;
2559
2560         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2561         next if !$pstat; # not running
2562
2563         my $used = $pstat->{utime} + $pstat->{stime};
2564
2565         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2566
2567         if ($pstat->{vsize}) {
2568             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2569         }
2570
2571         my $old = $last_proc_pid_stat->{$pid};
2572         if (!$old) {
2573             $last_proc_pid_stat->{$pid} = {
2574                 time => $ctime,
2575                 used => $used,
2576                 cpu => 0,
2577             };
2578             next;
2579         }
2580
2581         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2582
2583         if ($dtime > 1000) {
2584             my $dutime = $used -  $old->{used};
2585
2586             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2587             $last_proc_pid_stat->{$pid} = {
2588                 time => $ctime,
2589                 used => $used,
2590                 cpu => $d->{cpu},
2591             };
2592         } else {
2593             $d->{cpu} = $old->{cpu};
2594         }
2595     }
2596
2597     return $res if !$full;
2598
2599     my $qmpclient = PVE::QMPClient->new();
2600
2601     my $ballooncb = sub {
2602         my ($vmid, $resp) = @_;
2603
2604         my $info = $resp->{'return'};
2605         return if !$info->{max_mem};
2606
2607         my $d = $res->{$vmid};
2608
2609         # use memory assigned to VM
2610         $d->{maxmem} = $info->{max_mem};
2611         $d->{balloon} = $info->{actual};
2612
2613         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2614             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2615             $d->{freemem} = $info->{free_mem};
2616         }
2617
2618         $d->{ballooninfo} = $info;
2619     };
2620
2621     my $blockstatscb = sub {
2622         my ($vmid, $resp) = @_;
2623         my $data = $resp->{'return'} || [];
2624         my $totalrdbytes = 0;
2625         my $totalwrbytes = 0;
2626
2627         for my $blockstat (@$data) {
2628             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2629             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2630
2631             $blockstat->{device} =~ s/drive-//;
2632             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2633         }
2634         $res->{$vmid}->{diskread} = $totalrdbytes;
2635         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2636     };
2637
2638     my $statuscb = sub {
2639         my ($vmid, $resp) = @_;
2640
2641         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2642         # this fails if ballon driver is not loaded, so this must be
2643         # the last commnand (following command are aborted if this fails).
2644         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2645
2646         my $status = 'unknown';
2647         if (!defined($status = $resp->{'return'}->{status})) {
2648             warn "unable to get VM status\n";
2649             return;
2650         }
2651
2652         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2653     };
2654
2655     foreach my $vmid (keys %$list) {
2656         next if $opt_vmid && ($vmid ne $opt_vmid);
2657         next if !$res->{$vmid}->{pid}; # not running
2658         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2659     }
2660
2661     $qmpclient->queue_execute(undef, 2);
2662
2663     foreach my $vmid (keys %$list) {
2664         next if $opt_vmid && ($vmid ne $opt_vmid);
2665         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2666     }
2667
2668     return $res;
2669 }
2670
2671 sub conf_has_serial {
2672     my ($conf) = @_;
2673
2674     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2675         if ($conf->{"serial$i"}) {
2676             return 1;
2677         }
2678     }
2679
2680     return 0;
2681 }
2682
2683 sub conf_has_audio {
2684     my ($conf, $id) = @_;
2685
2686     $id //= 0;
2687     my $audio = $conf->{"audio$id"};
2688     return undef if !defined($audio);
2689
2690     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2691     my $audiodriver = $audioproperties->{driver} // 'spice';
2692
2693     return {
2694         dev => $audioproperties->{device},
2695         dev_id => "audiodev$id",
2696         backend => $audiodriver,
2697         backend_id => "$audiodriver-backend${id}",
2698     };
2699 }
2700
2701 sub audio_devs {
2702     my ($audio, $audiopciaddr, $machine_version) = @_;
2703
2704     my $devs = [];
2705
2706     my $id = $audio->{dev_id};
2707     my $audiodev = "";
2708     if (min_version($machine_version, 4, 2)) {
2709         $audiodev = ",audiodev=$audio->{backend_id}";
2710     }
2711
2712     if ($audio->{dev} eq 'AC97') {
2713         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2714     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2715         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2716         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2717         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2718     } else {
2719         die "unkown audio device '$audio->{dev}', implement me!";
2720     }
2721
2722     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2723
2724     return $devs;
2725 }
2726
2727 sub vga_conf_has_spice {
2728     my ($vga) = @_;
2729
2730     my $vgaconf = parse_vga($vga);
2731     my $vgatype = $vgaconf->{type};
2732     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2733
2734     return $1 || 1;
2735 }
2736
2737 sub is_native($) {
2738     my ($arch) = @_;
2739     return get_host_arch() eq $arch;
2740 }
2741
2742 sub get_vm_arch {
2743     my ($conf) = @_;
2744     return $conf->{arch} // get_host_arch();
2745 }
2746
2747 my $default_machines = {
2748     x86_64 => 'pc',
2749     aarch64 => 'virt',
2750 };
2751
2752 sub get_vm_machine {
2753     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2754
2755     my $machine = $forcemachine || $conf->{machine};
2756
2757     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2758         $arch //= 'x86_64';
2759         $machine ||= $default_machines->{$arch};
2760         if ($add_pve_version) {
2761             $kvmversion //= kvm_user_version();
2762             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2763             $machine .= "+pve$pvever";
2764         }
2765     }
2766
2767     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2768         # for version-pinned machines that do not include a pve-version (e.g.
2769         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2770         $machine .= '+pve0';
2771     }
2772
2773     return $machine;
2774 }
2775
2776 sub get_ovmf_files($) {
2777     my ($arch) = @_;
2778
2779     my $ovmf = $OVMF->{$arch}
2780         or die "no OVMF images known for architecture '$arch'\n";
2781
2782     return @$ovmf;
2783 }
2784
2785 my $Arch2Qemu = {
2786     aarch64 => '/usr/bin/qemu-system-aarch64',
2787     x86_64 => '/usr/bin/qemu-system-x86_64',
2788 };
2789 sub get_command_for_arch($) {
2790     my ($arch) = @_;
2791     return '/usr/bin/kvm' if is_native($arch);
2792
2793     my $cmd = $Arch2Qemu->{$arch}
2794         or die "don't know how to emulate architecture '$arch'\n";
2795     return $cmd;
2796 }
2797
2798 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2799 # to use in a QEMU command line (-cpu element), first array_intersect the result
2800 # of query_supported_ with query_understood_. This is necessary because:
2801 #
2802 # a) query_understood_ returns flags the host cannot use and
2803 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2804 #    flags, but CPU settings - with most of them being flags. Those settings
2805 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2806 #
2807 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2808 # expensive. If you need the value returned from this, you can get it much
2809 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2810 # $accel being 'kvm' or 'tcg'.
2811 #
2812 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2813 # changes, automatically populating pmxcfs.
2814 #
2815 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2816 # since kvm and tcg machines support different flags
2817 #
2818 sub query_supported_cpu_flags {
2819     my ($arch) = @_;
2820
2821     $arch //= get_host_arch();
2822     my $default_machine = $default_machines->{$arch};
2823
2824     my $flags = {};
2825
2826     # FIXME: Once this is merged, the code below should work for ARM as well:
2827     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2828     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2829         $arch eq "aarch64";
2830
2831     my $kvm_supported = defined(kvm_version());
2832     my $qemu_cmd = get_command_for_arch($arch);
2833     my $fakevmid = -1;
2834     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2835
2836     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2837     my $query_supported_run_qemu = sub {
2838         my ($kvm) = @_;
2839
2840         my $flags = {};
2841         my $cmd = [
2842             $qemu_cmd,
2843             '-machine', $default_machine,
2844             '-display', 'none',
2845             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2846             '-mon', 'chardev=qmp,mode=control',
2847             '-pidfile', $pidfile,
2848             '-S', '-daemonize'
2849         ];
2850
2851         if (!$kvm) {
2852             push @$cmd, '-accel', 'tcg';
2853         }
2854
2855         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2856         die "QEMU flag querying VM exited with code " . $rc if $rc;
2857
2858         eval {
2859             my $cmd_result = mon_cmd(
2860                 $fakevmid,
2861                 'query-cpu-model-expansion',
2862                 type => 'full',
2863                 model => { name => 'host' }
2864             );
2865
2866             my $props = $cmd_result->{model}->{props};
2867             foreach my $prop (keys %$props) {
2868                 next if $props->{$prop} ne '1';
2869                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2870                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2871                 # We only keep those with underscores, to match /proc/cpuinfo
2872                 $prop =~ s/\.|-/_/g;
2873                 $flags->{$prop} = 1;
2874             }
2875         };
2876         my $err = $@;
2877
2878         # force stop with 10 sec timeout and 'nocheck'
2879         # always stop, even if QMP failed
2880         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2881
2882         die $err if $err;
2883
2884         return [ sort keys %$flags ];
2885     };
2886
2887     # We need to query QEMU twice, since KVM and TCG have different supported flags
2888     PVE::QemuConfig->lock_config($fakevmid, sub {
2889         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2890         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2891
2892         if ($kvm_supported) {
2893             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2894             warn "warning: failed querying supported kvm flags: $@\n" if $@;
2895         }
2896     });
2897
2898     return $flags;
2899 }
2900
2901 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2902 my $understood_cpu_flag_dir = "/usr/share/kvm";
2903 sub query_understood_cpu_flags {
2904     my $arch = get_host_arch();
2905     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2906
2907     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2908         if ! -e $filepath;
2909
2910     my $raw = file_get_contents($filepath);
2911     $raw =~ s/^\s+|\s+$//g;
2912     my @flags = split(/\s+/, $raw);
2913
2914     return \@flags;
2915 }
2916
2917 sub config_to_command {
2918     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2919
2920     my $cmd = [];
2921     my $globalFlags = [];
2922     my $machineFlags = [];
2923     my $rtcFlags = [];
2924     my $devices = [];
2925     my $pciaddr = '';
2926     my $bridges = {};
2927     my $ostype = $conf->{ostype};
2928     my $winversion = windows_version($ostype);
2929     my $kvm = $conf->{kvm};
2930     my $nodename = nodename();
2931
2932     my $arch = get_vm_arch($conf);
2933     my $kvm_binary = get_command_for_arch($arch);
2934     my $kvmver = kvm_user_version($kvm_binary);
2935
2936     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2937         $kvmver //= "undefined";
2938         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2939     }
2940
2941     my $add_pve_version = min_version($kvmver, 4, 1);
2942
2943     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2944     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2945     $kvm //= 1 if is_native($arch);
2946
2947     $machine_version =~ m/(\d+)\.(\d+)/;
2948     my ($machine_major, $machine_minor) = ($1, $2);
2949
2950     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2951         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2952     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
2953         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2954     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2955         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2956         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2957             ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2958     }
2959
2960     # if a specific +pve version is required for a feature, use $version_guard
2961     # instead of min_version to allow machines to be run with the minimum
2962     # required version
2963     my $required_pve_version = 0;
2964     my $version_guard = sub {
2965         my ($major, $minor, $pve) = @_;
2966         return 0 if !min_version($machine_version, $major, $minor, $pve);
2967         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
2968         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
2969         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2970         return 1;
2971     };
2972
2973     if ($kvm) {
2974         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2975             if !defined kvm_version();
2976     }
2977
2978     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2979     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2980     my $use_old_bios_files = undef;
2981     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2982
2983     my $cpuunits = defined($conf->{cpuunits}) ?
2984             $conf->{cpuunits} : $defaults->{cpuunits};
2985
2986     push @$cmd, $kvm_binary;
2987
2988     push @$cmd, '-id', $vmid;
2989
2990     my $vmname = $conf->{name} || "vm$vmid";
2991
2992     push @$cmd, '-name', $vmname;
2993
2994     my $use_virtio = 0;
2995
2996     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2997     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2998     push @$cmd, '-mon', "chardev=qmp,mode=control";
2999
3000     if (min_version($machine_version, 2, 12)) {
3001         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3002         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3003     }
3004
3005     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3006
3007     push @$cmd, '-daemonize';
3008
3009     if ($conf->{smbios1}) {
3010         my $smbios_conf = parse_smbios1($conf->{smbios1});
3011         if ($smbios_conf->{base64}) {
3012             # Do not pass base64 flag to qemu
3013             delete $smbios_conf->{base64};
3014             my $smbios_string = "";
3015             foreach my $key (keys %$smbios_conf) {
3016                 my $value;
3017                 if ($key eq "uuid") {
3018                     $value = $smbios_conf->{uuid}
3019                 } else {
3020                     $value = decode_base64($smbios_conf->{$key});
3021                 }
3022                 # qemu accepts any binary data, only commas need escaping by double comma
3023                 $value =~ s/,/,,/g;
3024                 $smbios_string .= "," . $key . "=" . $value if $value;
3025             }
3026             push @$cmd, '-smbios', "type=1" . $smbios_string;
3027         } else {
3028             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3029         }
3030     }
3031
3032     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3033         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3034         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3035
3036         my ($path, $format);
3037         if (my $efidisk = $conf->{efidisk0}) {
3038             my $d = parse_drive('efidisk0', $efidisk);
3039             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3040             $format = $d->{format};
3041             if ($storeid) {
3042                 $path = PVE::Storage::path($storecfg, $d->{file});
3043                 if (!defined($format)) {
3044                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3045                     $format = qemu_img_format($scfg, $volname);
3046                 }
3047             } else {
3048                 $path = $d->{file};
3049                 die "efidisk format must be specified\n"
3050                     if !defined($format);
3051             }
3052         } else {
3053             warn "no efidisk configured! Using temporary efivars disk.\n";
3054             $path = "/tmp/$vmid-ovmf.fd";
3055             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3056             $format = 'raw';
3057         }
3058
3059         my $size_str = "";
3060
3061         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3062             $size_str = ",size=" . (-s $ovmf_vars);
3063         }
3064
3065         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3066         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3067     }
3068
3069     # load q35 config
3070     if ($q35) {
3071         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3072         if (min_version($machine_version, 4, 0)) {
3073             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3074         } else {
3075             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3076         }
3077     }
3078
3079     if ($conf->{vmgenid}) {
3080         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3081     }
3082
3083     # add usb controllers
3084     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3085     push @$devices, @usbcontrollers if @usbcontrollers;
3086     my $vga = parse_vga($conf->{vga});
3087
3088     my $qxlnum = vga_conf_has_spice($conf->{vga});
3089     $vga->{type} = 'qxl' if $qxlnum;
3090
3091     if (!$vga->{type}) {
3092         if ($arch eq 'aarch64') {
3093             $vga->{type} = 'virtio';
3094         } elsif (min_version($machine_version, 2, 9)) {
3095             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3096         } else {
3097             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3098         }
3099     }
3100
3101     # enable absolute mouse coordinates (needed by vnc)
3102     my $tablet;
3103     if (defined($conf->{tablet})) {
3104         $tablet = $conf->{tablet};
3105     } else {
3106         $tablet = $defaults->{tablet};
3107         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3108         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3109     }
3110
3111     if ($tablet) {
3112         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3113         my $kbd = print_keyboarddevice_full($conf, $arch);
3114         push @$devices, '-device', $kbd if defined($kbd);
3115     }
3116
3117     # host pci device passthrough
3118     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3119         $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3120
3121     # usb devices
3122     my $usb_dev_features = {};
3123     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3124
3125     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3126     push @$devices, @usbdevices if @usbdevices;
3127     # serial devices
3128     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3129         if (my $path = $conf->{"serial$i"}) {
3130             if ($path eq 'socket') {
3131                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3132                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3133                 # On aarch64, serial0 is the UART device. Qemu only allows
3134                 # connecting UART devices via the '-serial' command line, as
3135                 # the device has a fixed slot on the hardware...
3136                 if ($arch eq 'aarch64' && $i == 0) {
3137                     push @$devices, '-serial', "chardev:serial$i";
3138                 } else {
3139                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3140                 }
3141             } else {
3142                 die "no such serial device\n" if ! -c $path;
3143                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3144                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3145             }
3146         }
3147     }
3148
3149     # parallel devices
3150     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3151         if (my $path = $conf->{"parallel$i"}) {
3152             die "no such parallel device\n" if ! -c $path;
3153             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3154             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3155             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3156         }
3157     }
3158
3159     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3160         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3161         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3162         push @$devices, @$audio_devs;
3163     }
3164
3165     my $sockets = 1;
3166     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3167     $sockets = $conf->{sockets} if  $conf->{sockets};
3168
3169     my $cores = $conf->{cores} || 1;
3170
3171     my $maxcpus = $sockets * $cores;
3172
3173     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3174
3175     my $allowed_vcpus = $cpuinfo->{cpus};
3176
3177     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3178         if ($allowed_vcpus < $maxcpus);
3179
3180     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3181
3182         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3183         for (my $i = 2; $i <= $vcpus; $i++)  {
3184             my $cpustr = print_cpu_device($conf,$i);
3185             push @$cmd, '-device', $cpustr;
3186         }
3187
3188     } else {
3189
3190         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3191     }
3192     push @$cmd, '-nodefaults';
3193
3194     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3195
3196     my $bootindex_hash = {};
3197     my $i = 1;
3198     foreach my $o (split(//, $bootorder)) {
3199         $bootindex_hash->{$o} = $i*100;
3200         $i++;
3201     }
3202
3203     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3204
3205     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3206
3207     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3208
3209     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3210         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3211         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3212         push @$cmd,  '-vnc', "unix:$socket,password";
3213     } else {
3214         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3215         push @$cmd, '-nographic';
3216     }
3217
3218     # time drift fix
3219     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3220     my $useLocaltime = $conf->{localtime};
3221
3222     if ($winversion >= 5) { # windows
3223         $useLocaltime = 1 if !defined($conf->{localtime});
3224
3225         # use time drift fix when acpi is enabled
3226         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3227             $tdf = 1 if !defined($conf->{tdf});
3228         }
3229     }
3230
3231     if ($winversion >= 6) {
3232         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3233         push @$cmd, '-no-hpet';
3234     }
3235
3236     push @$rtcFlags, 'driftfix=slew' if $tdf;
3237
3238     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3239         push @$rtcFlags, "base=$conf->{startdate}";
3240     } elsif ($useLocaltime) {
3241         push @$rtcFlags, 'base=localtime';
3242     }
3243
3244     if ($forcecpu) {
3245         push @$cmd, '-cpu', $forcecpu;
3246     } else {
3247         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3248     }
3249
3250     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3251
3252     push @$cmd, '-S' if $conf->{freeze};
3253
3254     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3255
3256     my $guest_agent = parse_guest_agent($conf);
3257
3258     if ($guest_agent->{enabled}) {
3259         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3260         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3261
3262         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3263             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3264             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3265             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3266         } elsif ($guest_agent->{type} eq 'isa') {
3267             push @$devices, '-device', "isa-serial,chardev=qga0";
3268         }
3269     }
3270
3271     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3272     if ($rng && &$version_guard(4, 1, 2)) {
3273         check_rng_source($rng->{source});
3274
3275         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3276         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3277         my $limiter_str = "";
3278         if ($max_bytes) {
3279             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3280         }
3281
3282         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3283         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3284         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3285     }
3286
3287     my $spice_port;
3288
3289     if ($qxlnum) {
3290         if ($qxlnum > 1) {
3291             if ($winversion){
3292                 for (my $i = 1; $i < $qxlnum; $i++){
3293                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3294                 }
3295             } else {
3296                 # assume other OS works like Linux
3297                 my ($ram, $vram) = ("134217728", "67108864");
3298                 if ($vga->{memory}) {
3299                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3300                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3301                 }
3302                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3303                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3304             }
3305         }
3306
3307         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3308
3309         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3310         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3311         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3312
3313         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3314         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3315         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3316
3317         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3318         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3319
3320         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3321         if ($spice_enhancement->{foldersharing}) {
3322             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3323             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3324         }
3325
3326         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3327         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3328         push @$devices, '-spice', "$spice_opts";
3329     }
3330
3331     # enable balloon by default, unless explicitly disabled
3332     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3333         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3334         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3335     }
3336
3337     if ($conf->{watchdog}) {
3338         my $wdopts = parse_watchdog($conf->{watchdog});
3339         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3340         my $watchdog = $wdopts->{model} || 'i6300esb';
3341         push @$devices, '-device', "$watchdog$pciaddr";
3342         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3343     }
3344
3345     my $vollist = [];
3346     my $scsicontroller = {};
3347     my $ahcicontroller = {};
3348     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3349
3350     # Add iscsi initiator name if available
3351     if (my $initiator = get_initiator_name()) {
3352         push @$devices, '-iscsi', "initiator-name=$initiator";
3353     }
3354
3355     PVE::QemuConfig->foreach_volume($conf, sub {
3356         my ($ds, $drive) = @_;
3357
3358         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3359             push @$vollist, $drive->{file};
3360         }
3361
3362         # ignore efidisk here, already added in bios/fw handling code above
3363         return if $drive->{interface} eq 'efidisk';
3364
3365         $use_virtio = 1 if $ds =~ m/^virtio/;
3366
3367         if (drive_is_cdrom ($drive)) {
3368             if ($bootindex_hash->{d}) {
3369                 $drive->{bootindex} = $bootindex_hash->{d};
3370                 $bootindex_hash->{d} += 1;
3371             }
3372         } else {
3373             if ($bootindex_hash->{c}) {
3374                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3375                 $bootindex_hash->{c} += 1;
3376             }
3377         }
3378
3379         if ($drive->{interface} eq 'virtio'){
3380            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3381         }
3382
3383         if ($drive->{interface} eq 'scsi') {
3384
3385             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3386
3387             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3388                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3389
3390             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3391             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3392
3393             my $iothread = '';
3394             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3395                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3396                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3397             } elsif ($drive->{iothread}) {
3398                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3399             }
3400
3401             my $queues = '';
3402             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3403                 $queues = ",num_queues=$drive->{queues}";
3404             }
3405
3406             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3407             $scsicontroller->{$controller}=1;
3408         }
3409
3410         if ($drive->{interface} eq 'sata') {
3411             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3412             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3413             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3414             $ahcicontroller->{$controller}=1;
3415         }
3416
3417         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3418         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3419
3420         push @$devices, '-drive',$drive_cmd;
3421         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3422     });
3423
3424     for (my $i = 0; $i < $MAX_NETS; $i++) {
3425          next if !$conf->{"net$i"};
3426          my $d = parse_net($conf->{"net$i"});
3427          next if !$d;
3428
3429          $use_virtio = 1 if $d->{model} eq 'virtio';
3430
3431          if ($bootindex_hash->{n}) {
3432             $d->{bootindex} = $bootindex_hash->{n};
3433             $bootindex_hash->{n} += 1;
3434          }
3435
3436          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3437          push @$devices, '-netdev', $netdevfull;
3438
3439          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3440          push @$devices, '-device', $netdevicefull;
3441     }
3442
3443     if ($conf->{ivshmem}) {
3444         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3445
3446         my $bus;
3447         if ($q35) {
3448             $bus = print_pcie_addr("ivshmem");
3449         } else {
3450             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3451         }
3452
3453         my $ivshmem_name = $ivshmem->{name} // $vmid;
3454         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3455
3456         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3457         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3458     }
3459
3460     # pci.4 is nested in pci.1
3461     $bridges->{1} = 1 if $bridges->{4};
3462
3463     if (!$q35) {
3464         # add pci bridges
3465         if (min_version($machine_version, 2, 3)) {
3466            $bridges->{1} = 1;
3467            $bridges->{2} = 1;
3468         }
3469
3470         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3471
3472     }
3473
3474     for my $k (sort {$b cmp $a} keys %$bridges) {
3475         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3476
3477         my $k_name = $k;
3478         if ($k == 2 && $legacy_igd) {
3479             $k_name = "$k-igd";
3480         }
3481         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3482
3483         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3484         if ($q35) {
3485             # add after -readconfig pve-q35.cfg
3486             splice @$devices, 2, 0, '-device', $devstr;
3487         } else {
3488             unshift @$devices, '-device', $devstr if $k > 0;
3489         }
3490     }
3491
3492     if (!$kvm) {
3493         push @$machineFlags, 'accel=tcg';
3494     }
3495
3496     my $machine_type_min = $machine_type;
3497     if ($add_pve_version) {
3498         $machine_type_min =~ s/\+pve\d+$//;
3499         $machine_type_min .= "+pve$required_pve_version";
3500     }
3501     push @$machineFlags, "type=${machine_type_min}";
3502
3503     push @$cmd, @$devices;
3504     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3505     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3506     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3507
3508     if (my $vmstate = $conf->{vmstate}) {
3509         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3510         push @$vollist, $vmstate;
3511         push @$cmd, '-loadstate', $statepath;
3512         print "activating and using '$vmstate' as vmstate\n";
3513     }
3514
3515     # add custom args
3516     if ($conf->{args}) {
3517         my $aa = PVE::Tools::split_args($conf->{args});
3518         push @$cmd, @$aa;
3519     }
3520
3521     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3522 }
3523
3524 sub check_rng_source {
3525     my ($source) = @_;
3526
3527     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3528     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3529         if ! -e $source;
3530
3531     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3532     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3533         # Needs to abort, otherwise QEMU crashes on first rng access.
3534         # Note that rng_current cannot be changed to 'none' manually, so
3535         # once the VM is past this point, it is no longer an issue.
3536         die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3537             . " exists, but '$rng_current' is set to 'none'. Ensure that"
3538             . " a compatible hardware-RNG is attached to the host.\n";
3539     }
3540 }
3541
3542 sub spice_port {
3543     my ($vmid) = @_;
3544
3545     my $res = mon_cmd($vmid, 'query-spice');
3546
3547     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3548 }
3549
3550 sub vm_devices_list {
3551     my ($vmid) = @_;
3552
3553     my $res = mon_cmd($vmid, 'query-pci');
3554     my $devices_to_check = [];
3555     my $devices = {};
3556     foreach my $pcibus (@$res) {
3557         push @$devices_to_check, @{$pcibus->{devices}},
3558     }
3559
3560     while (@$devices_to_check) {
3561         my $to_check = [];
3562         for my $d (@$devices_to_check) {
3563             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3564             next if !$d->{'pci_bridge'};
3565
3566             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3567             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3568         }
3569         $devices_to_check = $to_check;
3570     }
3571
3572     my $resblock = mon_cmd($vmid, 'query-block');
3573     foreach my $block (@$resblock) {
3574         if($block->{device} =~ m/^drive-(\S+)/){
3575                 $devices->{$1} = 1;
3576         }
3577     }
3578
3579     my $resmice = mon_cmd($vmid, 'query-mice');
3580     foreach my $mice (@$resmice) {
3581         if ($mice->{name} eq 'QEMU HID Tablet') {
3582             $devices->{tablet} = 1;
3583             last;
3584         }
3585     }
3586
3587     # for usb devices there is no query-usb
3588     # but we can iterate over the entries in
3589     # qom-list path=/machine/peripheral
3590     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3591     foreach my $per (@$resperipheral) {
3592         if ($per->{name} =~ m/^usb\d+$/) {
3593             $devices->{$per->{name}} = 1;
3594         }
3595     }
3596
3597     return $devices;
3598 }
3599
3600 sub vm_deviceplug {
3601     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3602
3603     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3604
3605     my $devices_list = vm_devices_list($vmid);
3606     return 1 if defined($devices_list->{$deviceid});
3607
3608     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3609
3610     if ($deviceid eq 'tablet') {
3611
3612         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3613
3614     } elsif ($deviceid eq 'keyboard') {
3615
3616         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3617
3618     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3619
3620         die "usb hotplug currently not reliable\n";
3621         # since we can't reliably hot unplug all added usb devices
3622         # and usb passthrough disables live migration
3623         # we disable usb hotplugging for now
3624         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3625
3626     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3627
3628         qemu_iothread_add($vmid, $deviceid, $device);
3629
3630         qemu_driveadd($storecfg, $vmid, $device);
3631         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3632
3633         qemu_deviceadd($vmid, $devicefull);
3634         eval { qemu_deviceaddverify($vmid, $deviceid); };
3635         if (my $err = $@) {
3636             eval { qemu_drivedel($vmid, $deviceid); };
3637             warn $@ if $@;
3638             die $err;
3639         }
3640
3641     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3642
3643
3644         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3645         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3646         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3647
3648         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3649
3650         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3651             qemu_iothread_add($vmid, $deviceid, $device);
3652             $devicefull .= ",iothread=iothread-$deviceid";
3653         }
3654
3655         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3656             $devicefull .= ",num_queues=$device->{queues}";
3657         }
3658
3659         qemu_deviceadd($vmid, $devicefull);
3660         qemu_deviceaddverify($vmid, $deviceid);
3661
3662     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3663
3664         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3665         qemu_driveadd($storecfg, $vmid, $device);
3666
3667         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3668         eval { qemu_deviceadd($vmid, $devicefull); };
3669         if (my $err = $@) {
3670             eval { qemu_drivedel($vmid, $deviceid); };
3671             warn $@ if $@;
3672             die $err;
3673         }
3674
3675     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3676
3677         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3678
3679         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3680         my $use_old_bios_files = undef;
3681         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3682
3683         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3684         qemu_deviceadd($vmid, $netdevicefull);
3685         eval {
3686             qemu_deviceaddverify($vmid, $deviceid);
3687             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3688         };
3689         if (my $err = $@) {
3690             eval { qemu_netdevdel($vmid, $deviceid); };
3691             warn $@ if $@;
3692             die $err;
3693         }
3694
3695     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3696
3697         my $bridgeid = $2;
3698         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3699         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3700
3701         qemu_deviceadd($vmid, $devicefull);
3702         qemu_deviceaddverify($vmid, $deviceid);
3703
3704     } else {
3705         die "can't hotplug device '$deviceid'\n";
3706     }
3707
3708     return 1;
3709 }
3710
3711 # fixme: this should raise exceptions on error!
3712 sub vm_deviceunplug {
3713     my ($vmid, $conf, $deviceid) = @_;
3714
3715     my $devices_list = vm_devices_list($vmid);
3716     return 1 if !defined($devices_list->{$deviceid});
3717
3718     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3719
3720     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3721
3722         qemu_devicedel($vmid, $deviceid);
3723
3724     } elsif ($deviceid =~ m/^usb\d+$/) {
3725
3726         die "usb hotplug currently not reliable\n";
3727         # when unplugging usb devices this way,
3728         # there may be remaining usb controllers/hubs
3729         # so we disable it for now
3730         qemu_devicedel($vmid, $deviceid);
3731         qemu_devicedelverify($vmid, $deviceid);
3732
3733     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3734
3735         qemu_devicedel($vmid, $deviceid);
3736         qemu_devicedelverify($vmid, $deviceid);
3737         qemu_drivedel($vmid, $deviceid);
3738         qemu_iothread_del($conf, $vmid, $deviceid);
3739
3740     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3741
3742         qemu_devicedel($vmid, $deviceid);
3743         qemu_devicedelverify($vmid, $deviceid);
3744         qemu_iothread_del($conf, $vmid, $deviceid);
3745
3746     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3747
3748         qemu_devicedel($vmid, $deviceid);
3749         qemu_drivedel($vmid, $deviceid);
3750         qemu_deletescsihw($conf, $vmid, $deviceid);
3751
3752     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3753
3754         qemu_devicedel($vmid, $deviceid);
3755         qemu_devicedelverify($vmid, $deviceid);
3756         qemu_netdevdel($vmid, $deviceid);
3757
3758     } else {
3759         die "can't unplug device '$deviceid'\n";
3760     }
3761
3762     return 1;
3763 }
3764
3765 sub qemu_deviceadd {
3766     my ($vmid, $devicefull) = @_;
3767
3768     $devicefull = "driver=".$devicefull;
3769     my %options =  split(/[=,]/, $devicefull);
3770
3771     mon_cmd($vmid, "device_add" , %options);
3772 }
3773
3774 sub qemu_devicedel {
3775     my ($vmid, $deviceid) = @_;
3776
3777     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3778 }
3779
3780 sub qemu_iothread_add {
3781     my($vmid, $deviceid, $device) = @_;
3782
3783     if ($device->{iothread}) {
3784         my $iothreads = vm_iothreads_list($vmid);
3785         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3786     }
3787 }
3788
3789 sub qemu_iothread_del {
3790     my($conf, $vmid, $deviceid) = @_;
3791
3792     my $confid = $deviceid;
3793     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3794         $confid = 'scsi' . $1;
3795     }
3796     my $device = parse_drive($confid, $conf->{$confid});
3797     if ($device->{iothread}) {
3798         my $iothreads = vm_iothreads_list($vmid);
3799         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3800     }
3801 }
3802
3803 sub qemu_objectadd {
3804     my($vmid, $objectid, $qomtype) = @_;
3805
3806     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3807
3808     return 1;
3809 }
3810
3811 sub qemu_objectdel {
3812     my($vmid, $objectid) = @_;
3813
3814     mon_cmd($vmid, "object-del", id => $objectid);
3815
3816     return 1;
3817 }
3818
3819 sub qemu_driveadd {
3820     my ($storecfg, $vmid, $device) = @_;
3821
3822     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3823     $drive =~ s/\\/\\\\/g;
3824     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3825
3826     # If the command succeeds qemu prints: "OK"
3827     return 1 if $ret =~ m/OK/s;
3828
3829     die "adding drive failed: $ret\n";
3830 }
3831
3832 sub qemu_drivedel {
3833     my($vmid, $deviceid) = @_;
3834
3835     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3836     $ret =~ s/^\s+//;
3837
3838     return 1 if $ret eq "";
3839
3840     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3841     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3842
3843     die "deleting drive $deviceid failed : $ret\n";
3844 }
3845
3846 sub qemu_deviceaddverify {
3847     my ($vmid, $deviceid) = @_;
3848
3849     for (my $i = 0; $i <= 5; $i++) {
3850          my $devices_list = vm_devices_list($vmid);
3851          return 1 if defined($devices_list->{$deviceid});
3852          sleep 1;
3853     }
3854
3855     die "error on hotplug device '$deviceid'\n";
3856 }
3857
3858
3859 sub qemu_devicedelverify {
3860     my ($vmid, $deviceid) = @_;
3861
3862     # need to verify that the device is correctly removed as device_del
3863     # is async and empty return is not reliable
3864
3865     for (my $i = 0; $i <= 5; $i++) {
3866          my $devices_list = vm_devices_list($vmid);
3867          return 1 if !defined($devices_list->{$deviceid});
3868          sleep 1;
3869     }
3870
3871     die "error on hot-unplugging device '$deviceid'\n";
3872 }
3873
3874 sub qemu_findorcreatescsihw {
3875     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3876
3877     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3878
3879     my $scsihwid="$controller_prefix$controller";
3880     my $devices_list = vm_devices_list($vmid);
3881
3882     if(!defined($devices_list->{$scsihwid})) {
3883         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3884     }
3885
3886     return 1;
3887 }
3888
3889 sub qemu_deletescsihw {
3890     my ($conf, $vmid, $opt) = @_;
3891
3892     my $device = parse_drive($opt, $conf->{$opt});
3893
3894     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3895         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3896         return 1;
3897     }
3898
3899     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3900
3901     my $devices_list = vm_devices_list($vmid);
3902     foreach my $opt (keys %{$devices_list}) {
3903         if (is_valid_drivename($opt)) {
3904             my $drive = parse_drive($opt, $conf->{$opt});
3905             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3906                 return 1;
3907             }
3908         }
3909     }
3910
3911     my $scsihwid="scsihw$controller";
3912
3913     vm_deviceunplug($vmid, $conf, $scsihwid);
3914
3915     return 1;
3916 }
3917
3918 sub qemu_add_pci_bridge {
3919     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3920
3921     my $bridges = {};
3922
3923     my $bridgeid;
3924
3925     print_pci_addr($device, $bridges, $arch, $machine_type);
3926
3927     while (my ($k, $v) = each %$bridges) {
3928         $bridgeid = $k;
3929     }
3930     return 1 if !defined($bridgeid) || $bridgeid < 1;
3931
3932     my $bridge = "pci.$bridgeid";
3933     my $devices_list = vm_devices_list($vmid);
3934
3935     if (!defined($devices_list->{$bridge})) {
3936         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3937     }
3938
3939     return 1;
3940 }
3941
3942 sub qemu_set_link_status {
3943     my ($vmid, $device, $up) = @_;
3944
3945     mon_cmd($vmid, "set_link", name => $device,
3946                up => $up ? JSON::true : JSON::false);
3947 }
3948
3949 sub qemu_netdevadd {
3950     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3951
3952     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3953     my %options =  split(/[=,]/, $netdev);
3954
3955     if (defined(my $vhost = $options{vhost})) {
3956         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3957     }
3958
3959     if (defined(my $queues = $options{queues})) {
3960         $options{queues} = $queues + 0;
3961     }
3962
3963     mon_cmd($vmid, "netdev_add",  %options);
3964     return 1;
3965 }
3966
3967 sub qemu_netdevdel {
3968     my ($vmid, $deviceid) = @_;
3969
3970     mon_cmd($vmid, "netdev_del", id => $deviceid);
3971 }
3972
3973 sub qemu_usb_hotplug {
3974     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3975
3976     return if !$device;
3977
3978     # remove the old one first
3979     vm_deviceunplug($vmid, $conf, $deviceid);
3980
3981     # check if xhci controller is necessary and available
3982     if ($device->{usb3}) {
3983
3984         my $devicelist = vm_devices_list($vmid);
3985
3986         if (!$devicelist->{xhci}) {
3987             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
3988             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3989         }
3990     }
3991     my $d = parse_usb_device($device->{host});
3992     $d->{usb3} = $device->{usb3};
3993
3994     # add the new one
3995     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3996 }
3997
3998 sub qemu_cpu_hotplug {
3999     my ($vmid, $conf, $vcpus) = @_;
4000
4001     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4002
4003     my $sockets = 1;
4004     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4005     $sockets = $conf->{sockets} if  $conf->{sockets};
4006     my $cores = $conf->{cores} || 1;
4007     my $maxcpus = $sockets * $cores;
4008
4009     $vcpus = $maxcpus if !$vcpus;
4010
4011     die "you can't add more vcpus than maxcpus\n"
4012         if $vcpus > $maxcpus;
4013
4014     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4015
4016     if ($vcpus < $currentvcpus) {
4017
4018         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4019
4020             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4021                 qemu_devicedel($vmid, "cpu$i");
4022                 my $retry = 0;
4023                 my $currentrunningvcpus = undef;
4024                 while (1) {
4025                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4026                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4027                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4028                     $retry++;
4029                     sleep 1;
4030                 }
4031                 #update conf after each succesfull cpu unplug
4032                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4033                 PVE::QemuConfig->write_config($vmid, $conf);
4034             }
4035         } else {
4036             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4037         }
4038
4039         return;
4040     }
4041
4042     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4043     die "vcpus in running vm does not match its configuration\n"
4044         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4045
4046     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4047
4048         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4049             my $cpustr = print_cpu_device($conf, $i);
4050             qemu_deviceadd($vmid, $cpustr);
4051
4052             my $retry = 0;
4053             my $currentrunningvcpus = undef;
4054             while (1) {
4055                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4056                 last if scalar(@{$currentrunningvcpus}) == $i;
4057                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4058                 sleep 1;
4059                 $retry++;
4060             }
4061             #update conf after each succesfull cpu hotplug
4062             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4063             PVE::QemuConfig->write_config($vmid, $conf);
4064         }
4065     } else {
4066
4067         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4068             mon_cmd($vmid, "cpu-add", id => int($i));
4069         }
4070     }
4071 }
4072
4073 sub qemu_block_set_io_throttle {
4074     my ($vmid, $deviceid,
4075         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4076         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4077         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4078         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4079
4080     return if !check_running($vmid) ;
4081
4082     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4083         bps => int($bps),
4084         bps_rd => int($bps_rd),
4085         bps_wr => int($bps_wr),
4086         iops => int($iops),
4087         iops_rd => int($iops_rd),
4088         iops_wr => int($iops_wr),
4089         bps_max => int($bps_max),
4090         bps_rd_max => int($bps_rd_max),
4091         bps_wr_max => int($bps_wr_max),
4092         iops_max => int($iops_max),
4093         iops_rd_max => int($iops_rd_max),
4094         iops_wr_max => int($iops_wr_max),
4095         bps_max_length => int($bps_max_length),
4096         bps_rd_max_length => int($bps_rd_max_length),
4097         bps_wr_max_length => int($bps_wr_max_length),
4098         iops_max_length => int($iops_max_length),
4099         iops_rd_max_length => int($iops_rd_max_length),
4100         iops_wr_max_length => int($iops_wr_max_length),
4101     );
4102
4103 }
4104
4105 # old code, only used to shutdown old VM after update
4106 sub __read_avail {
4107     my ($fh, $timeout) = @_;
4108
4109     my $sel = new IO::Select;
4110     $sel->add($fh);
4111
4112     my $res = '';
4113     my $buf;
4114
4115     my @ready;
4116     while (scalar (@ready = $sel->can_read($timeout))) {
4117         my $count;
4118         if ($count = $fh->sysread($buf, 8192)) {
4119             if ($buf =~ /^(.*)\(qemu\) $/s) {
4120                 $res .= $1;
4121                 last;
4122             } else {
4123                 $res .= $buf;
4124             }
4125         } else {
4126             if (!defined($count)) {
4127                 die "$!\n";
4128             }
4129             last;
4130         }
4131     }
4132
4133     die "monitor read timeout\n" if !scalar(@ready);
4134
4135     return $res;
4136 }
4137
4138 sub qemu_block_resize {
4139     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4140
4141     my $running = check_running($vmid);
4142
4143     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4144
4145     return if !$running;
4146
4147     my $padding = (1024 - $size % 1024) % 1024;
4148     $size = $size + $padding;
4149
4150     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4151
4152 }
4153
4154 sub qemu_volume_snapshot {
4155     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4156
4157     my $running = check_running($vmid);
4158
4159     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4160         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4161     } else {
4162         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4163     }
4164 }
4165
4166 sub qemu_volume_snapshot_delete {
4167     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4168
4169     my $running = check_running($vmid);
4170
4171     if($running) {
4172
4173         $running = undef;
4174         my $conf = PVE::QemuConfig->load_config($vmid);
4175         PVE::QemuConfig->foreach_volume($conf, sub {
4176             my ($ds, $drive) = @_;
4177             $running = 1 if $drive->{file} eq $volid;
4178         });
4179     }
4180
4181     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4182         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4183     } else {
4184         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4185     }
4186 }
4187
4188 sub set_migration_caps {
4189     my ($vmid) = @_;
4190
4191     my $cap_ref = [];
4192
4193     my $enabled_cap = {
4194         "auto-converge" => 1,
4195         "xbzrle" => 1,
4196         "x-rdma-pin-all" => 0,
4197         "zero-blocks" => 0,
4198         "compress" => 0
4199     };
4200
4201     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4202
4203     for my $supported_capability (@$supported_capabilities) {
4204         push @$cap_ref, {
4205             capability => $supported_capability->{capability},
4206             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4207         };
4208     }
4209
4210     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4211 }
4212
4213 sub foreach_volid {
4214     my ($conf, $func, @param) = @_;
4215
4216     my $volhash = {};
4217
4218     my $test_volid = sub {
4219         my ($key, $drive, $snapname) = @_;
4220
4221         my $volid = $drive->{file};
4222         return if !$volid;
4223
4224         $volhash->{$volid}->{cdrom} //= 1;
4225         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4226
4227         my $replicate = $drive->{replicate} // 1;
4228         $volhash->{$volid}->{replicate} //= 0;
4229         $volhash->{$volid}->{replicate} = 1 if $replicate;
4230
4231         $volhash->{$volid}->{shared} //= 0;
4232         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4233
4234         $volhash->{$volid}->{referenced_in_config} //= 0;
4235         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4236
4237         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4238             if defined($snapname);
4239
4240         my $size = $drive->{size};
4241         $volhash->{$volid}->{size} //= $size if $size;
4242
4243         $volhash->{$volid}->{is_vmstate} //= 0;
4244         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4245
4246         $volhash->{$volid}->{is_unused} //= 0;
4247         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4248     };
4249
4250     my $include_opts = {
4251         extra_keys => ['vmstate'],
4252         include_unused => 1,
4253     };
4254
4255     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4256     foreach my $snapname (keys %{$conf->{snapshots}}) {
4257         my $snap = $conf->{snapshots}->{$snapname};
4258         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4259     }
4260
4261     foreach my $volid (keys %$volhash) {
4262         &$func($volid, $volhash->{$volid}, @param);
4263     }
4264 }
4265
4266 my $fast_plug_option = {
4267     'lock' => 1,
4268     'name' => 1,
4269     'onboot' => 1,
4270     'shares' => 1,
4271     'startup' => 1,
4272     'description' => 1,
4273     'protection' => 1,
4274     'vmstatestorage' => 1,
4275     'hookscript' => 1,
4276     'tags' => 1,
4277 };
4278
4279 # hotplug changes in [PENDING]
4280 # $selection hash can be used to only apply specified options, for
4281 # example: { cores => 1 } (only apply changed 'cores')
4282 # $errors ref is used to return error messages
4283 sub vmconfig_hotplug_pending {
4284     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4285
4286     my $defaults = load_defaults();
4287     my $arch = get_vm_arch($conf);
4288     my $machine_type = get_vm_machine($conf, undef, $arch);
4289
4290     # commit values which do not have any impact on running VM first
4291     # Note: those option cannot raise errors, we we do not care about
4292     # $selection and always apply them.
4293
4294     my $add_error = sub {
4295         my ($opt, $msg) = @_;
4296         $errors->{$opt} = "hotplug problem - $msg";
4297     };
4298
4299     my $changes = 0;
4300     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4301         if ($fast_plug_option->{$opt}) {
4302             $conf->{$opt} = $conf->{pending}->{$opt};
4303             delete $conf->{pending}->{$opt};
4304             $changes = 1;
4305         }
4306     }
4307
4308     if ($changes) {
4309         PVE::QemuConfig->write_config($vmid, $conf);
4310     }
4311
4312     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4313
4314     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4315     foreach my $opt (sort keys %$pending_delete_hash) {
4316         next if $selection && !$selection->{$opt};
4317         my $force = $pending_delete_hash->{$opt}->{force};
4318         eval {
4319             if ($opt eq 'hotplug') {
4320                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4321             } elsif ($opt eq 'tablet') {
4322                 die "skip\n" if !$hotplug_features->{usb};
4323                 if ($defaults->{tablet}) {
4324                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4325                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4326                         if $arch eq 'aarch64';
4327                 } else {
4328                     vm_deviceunplug($vmid, $conf, 'tablet');
4329                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4330                 }
4331             } elsif ($opt =~ m/^usb\d+/) {
4332                 die "skip\n";
4333                 # since we cannot reliably hot unplug usb devices
4334                 # we are disabling it
4335                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4336                 vm_deviceunplug($vmid, $conf, $opt);
4337             } elsif ($opt eq 'vcpus') {
4338                 die "skip\n" if !$hotplug_features->{cpu};
4339                 qemu_cpu_hotplug($vmid, $conf, undef);
4340             } elsif ($opt eq 'balloon') {
4341                 # enable balloon device is not hotpluggable
4342                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4343                 # here we reset the ballooning value to memory
4344                 my $balloon = $conf->{memory} || $defaults->{memory};
4345                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4346             } elsif ($fast_plug_option->{$opt}) {
4347                 # do nothing
4348             } elsif ($opt =~ m/^net(\d+)$/) {
4349                 die "skip\n" if !$hotplug_features->{network};
4350                 vm_deviceunplug($vmid, $conf, $opt);
4351             } elsif (is_valid_drivename($opt)) {
4352                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4353                 vm_deviceunplug($vmid, $conf, $opt);
4354                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4355             } elsif ($opt =~ m/^memory$/) {
4356                 die "skip\n" if !$hotplug_features->{memory};
4357                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4358             } elsif ($opt eq 'cpuunits') {
4359                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4360             } elsif ($opt eq 'cpulimit') {
4361                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4362             } else {
4363                 die "skip\n";
4364             }
4365         };
4366         if (my $err = $@) {
4367             &$add_error($opt, $err) if $err ne "skip\n";
4368         } else {
4369             delete $conf->{$opt};
4370             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4371         }
4372     }
4373
4374     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4375     $apply_pending_cloudinit = sub {
4376         return if $apply_pending_cloudinit_done; # once is enough
4377         $apply_pending_cloudinit_done = 1; # once is enough
4378
4379         my ($key, $value) = @_;
4380
4381         my @cloudinit_opts = keys %$confdesc_cloudinit;
4382         foreach my $opt (keys %{$conf->{pending}}) {
4383             next if !grep { $_ eq $opt } @cloudinit_opts;
4384             $conf->{$opt} = delete $conf->{pending}->{$opt};
4385         }
4386
4387         my $new_conf = { %$conf };
4388         $new_conf->{$key} = $value;
4389         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4390     };
4391
4392     foreach my $opt (keys %{$conf->{pending}}) {
4393         next if $selection && !$selection->{$opt};
4394         my $value = $conf->{pending}->{$opt};
4395         eval {
4396             if ($opt eq 'hotplug') {
4397                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4398             } elsif ($opt eq 'tablet') {
4399                 die "skip\n" if !$hotplug_features->{usb};
4400                 if ($value == 1) {
4401                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4402                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4403                         if $arch eq 'aarch64';
4404                 } elsif ($value == 0) {
4405                     vm_deviceunplug($vmid, $conf, 'tablet');
4406                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4407                 }
4408             } elsif ($opt =~ m/^usb\d+$/) {
4409                 die "skip\n";
4410                 # since we cannot reliably hot unplug usb devices
4411                 # we are disabling it
4412                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4413                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4414                 die "skip\n" if !$d;
4415                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4416             } elsif ($opt eq 'vcpus') {
4417                 die "skip\n" if !$hotplug_features->{cpu};
4418                 qemu_cpu_hotplug($vmid, $conf, $value);
4419             } elsif ($opt eq 'balloon') {
4420                 # enable/disable balloning device is not hotpluggable
4421                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4422                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4423                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4424
4425                 # allow manual ballooning if shares is set to zero
4426                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4427                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4428                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4429                 }
4430             } elsif ($opt =~ m/^net(\d+)$/) {
4431                 # some changes can be done without hotplug
4432                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4433                                     $vmid, $opt, $value, $arch, $machine_type);
4434             } elsif (is_valid_drivename($opt)) {
4435                 die "skip\n" if $opt eq 'efidisk0';
4436                 # some changes can be done without hotplug
4437                 my $drive = parse_drive($opt, $value);
4438                 if (drive_is_cloudinit($drive)) {
4439                     &$apply_pending_cloudinit($opt, $value);
4440                 }
4441                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4442                                      $vmid, $opt, $value, $arch, $machine_type);
4443             } elsif ($opt =~ m/^memory$/) { #dimms
4444                 die "skip\n" if !$hotplug_features->{memory};
4445                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4446             } elsif ($opt eq 'cpuunits') {
4447                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4448             } elsif ($opt eq 'cpulimit') {
4449                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4450                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4451             } else {
4452                 die "skip\n";  # skip non-hot-pluggable options
4453             }
4454         };
4455         if (my $err = $@) {
4456             &$add_error($opt, $err) if $err ne "skip\n";
4457         } else {
4458             $conf->{$opt} = $value;
4459             delete $conf->{pending}->{$opt};
4460         }
4461     }
4462
4463     PVE::QemuConfig->write_config($vmid, $conf);
4464 }
4465
4466 sub try_deallocate_drive {
4467     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4468
4469     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4470         my $volid = $drive->{file};
4471         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4472             my $sid = PVE::Storage::parse_volume_id($volid);
4473             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4474
4475             # check if the disk is really unused
4476             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4477                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4478             PVE::Storage::vdisk_free($storecfg, $volid);
4479             return 1;
4480         } else {
4481             # If vm is not owner of this disk remove from config
4482             return 1;
4483         }
4484     }
4485
4486     return undef;
4487 }
4488
4489 sub vmconfig_delete_or_detach_drive {
4490     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4491
4492     my $drive = parse_drive($opt, $conf->{$opt});
4493
4494     my $rpcenv = PVE::RPCEnvironment::get();
4495     my $authuser = $rpcenv->get_user();
4496
4497     if ($force) {
4498         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4499         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4500     } else {
4501         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4502     }
4503 }
4504
4505
4506
4507 sub vmconfig_apply_pending {
4508     my ($vmid, $conf, $storecfg, $errors) = @_;
4509
4510     my $add_apply_error = sub {
4511         my ($opt, $msg) = @_;
4512         my $err_msg = "unable to apply pending change $opt : $msg";
4513         $errors->{$opt} = $err_msg;
4514         warn $err_msg;
4515     };
4516
4517     # cold plug
4518
4519     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4520     foreach my $opt (sort keys %$pending_delete_hash) {
4521         my $force = $pending_delete_hash->{$opt}->{force};
4522         eval {
4523             if ($opt =~ m/^unused/) {
4524                 die "internal error";
4525             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4526                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4527             }
4528         };
4529         if (my $err = $@) {
4530             $add_apply_error->($opt, $err);
4531         } else {
4532             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4533             delete $conf->{$opt};
4534         }
4535     }
4536
4537     PVE::QemuConfig->cleanup_pending($conf);
4538
4539     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4540         next if $opt eq 'delete'; # just to be sure
4541         eval {
4542             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4543                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4544             }
4545         };
4546         if (my $err = $@) {
4547             $add_apply_error->($opt, $err);
4548         } else {
4549             $conf->{$opt} = delete $conf->{pending}->{$opt};
4550         }
4551     }
4552
4553     # write all changes at once to avoid unnecessary i/o
4554     PVE::QemuConfig->write_config($vmid, $conf);
4555 }
4556
4557 sub vmconfig_update_net {
4558     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4559
4560     my $newnet = parse_net($value);
4561
4562     if ($conf->{$opt}) {
4563         my $oldnet = parse_net($conf->{$opt});
4564
4565         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4566             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4567             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4568             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4569
4570             # for non online change, we try to hot-unplug
4571             die "skip\n" if !$hotplug;
4572             vm_deviceunplug($vmid, $conf, $opt);
4573         } else {
4574
4575             die "internal error" if $opt !~ m/net(\d+)/;
4576             my $iface = "tap${vmid}i$1";
4577
4578             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4579                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4580                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4581                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4582                 PVE::Network::tap_unplug($iface);
4583
4584                 if ($have_sdn) {
4585                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4586                 } else {
4587                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4588                 }
4589             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4590                 # Rate can be applied on its own but any change above needs to
4591                 # include the rate in tap_plug since OVS resets everything.
4592                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4593             }
4594
4595             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4596                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4597             }
4598
4599             return 1;
4600         }
4601     }
4602
4603     if ($hotplug) {
4604         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4605     } else {
4606         die "skip\n";
4607     }
4608 }
4609
4610 sub vmconfig_update_disk {
4611     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4612
4613     my $drive = parse_drive($opt, $value);
4614
4615     if ($conf->{$opt}) {
4616
4617         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4618
4619             my $media = $drive->{media} || 'disk';
4620             my $oldmedia = $old_drive->{media} || 'disk';
4621             die "unable to change media type\n" if $media ne $oldmedia;
4622
4623             if (!drive_is_cdrom($old_drive)) {
4624
4625                 if ($drive->{file} ne $old_drive->{file}) {
4626
4627                     die "skip\n" if !$hotplug;
4628
4629                     # unplug and register as unused
4630                     vm_deviceunplug($vmid, $conf, $opt);
4631                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4632
4633                 } else {
4634                     # update existing disk
4635
4636                     # skip non hotpluggable value
4637                     if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4638                         safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4639                         safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4640                         safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4641                         safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4642                         die "skip\n";
4643                     }
4644
4645                     # apply throttle
4646                     if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4647                         safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4648                         safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4649                         safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4650                         safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4651                         safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4652                         safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4653                         safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4654                         safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4655                         safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4656                         safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4657                         safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4658                         safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4659                         safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4660                         safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4661                         safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4662                         safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4663                         safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4664
4665                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4666                                                    ($drive->{mbps} || 0)*1024*1024,
4667                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4668                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4669                                                    $drive->{iops} || 0,
4670                                                    $drive->{iops_rd} || 0,
4671                                                    $drive->{iops_wr} || 0,
4672                                                    ($drive->{mbps_max} || 0)*1024*1024,
4673                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4674                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4675                                                    $drive->{iops_max} || 0,
4676                                                    $drive->{iops_rd_max} || 0,
4677                                                    $drive->{iops_wr_max} || 0,
4678                                                    $drive->{bps_max_length} || 1,
4679                                                    $drive->{bps_rd_max_length} || 1,
4680                                                    $drive->{bps_wr_max_length} || 1,
4681                                                    $drive->{iops_max_length} || 1,
4682                                                    $drive->{iops_rd_max_length} || 1,
4683                                                    $drive->{iops_wr_max_length} || 1);
4684
4685                     }
4686
4687                     return 1;
4688                 }
4689
4690             } else { # cdrom
4691
4692                 if ($drive->{file} eq 'none') {
4693                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4694                     if (drive_is_cloudinit($old_drive)) {
4695                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4696                     }
4697                 } else {
4698                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4699
4700                     # force eject if locked
4701                     mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4702
4703                     if ($path) {
4704                         mon_cmd($vmid, "blockdev-change-medium",
4705                             id => "$opt", filename => "$path");
4706                     }
4707                 }
4708
4709                 return 1;
4710             }
4711         }
4712     }
4713
4714     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4715     # hotplug new disks
4716     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4717     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4718 }
4719
4720 # called in locked context by incoming migration
4721 sub vm_migrate_get_nbd_disks {
4722     my ($storecfg, $conf, $replicated_volumes) = @_;
4723
4724     my $local_volumes = {};
4725     PVE::QemuConfig->foreach_volume($conf, sub {
4726         my ($ds, $drive) = @_;
4727
4728         return if drive_is_cdrom($drive);
4729
4730         my $volid = $drive->{file};
4731
4732         return if !$volid;
4733
4734         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4735
4736         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4737         return if $scfg->{shared};
4738
4739         # replicated disks re-use existing state via bitmap
4740         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4741         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4742     });
4743     return $local_volumes;
4744 }
4745
4746 # called in locked context by incoming migration
4747 sub vm_migrate_alloc_nbd_disks {
4748     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4749
4750     my $format = undef;
4751
4752     my $nbd = {};
4753     foreach my $opt (sort keys %$source_volumes) {
4754         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4755
4756         if ($use_existing) {
4757             $nbd->{$opt}->{drivestr} = print_drive($drive);
4758             $nbd->{$opt}->{volid} = $volid;
4759             $nbd->{$opt}->{replicated} = 1;
4760             next;
4761         }
4762
4763         # If a remote storage is specified and the format of the original
4764         # volume is not available there, fall back to the default format.
4765         # Otherwise use the same format as the original.
4766         if (!$storagemap->{identity}) {
4767             $storeid = map_storage($storagemap, $storeid);
4768             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4769             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4770             my $fileFormat = qemu_img_format($scfg, $volname);
4771             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4772         } else {
4773             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4774             $format = qemu_img_format($scfg, $volname);
4775         }
4776
4777         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4778         my $newdrive = $drive;
4779         $newdrive->{format} = $format;
4780         $newdrive->{file} = $newvolid;
4781         my $drivestr = print_drive($newdrive);
4782         $nbd->{$opt}->{drivestr} = $drivestr;
4783         $nbd->{$opt}->{volid} = $newvolid;
4784     }
4785
4786     return $nbd;
4787 }
4788
4789 # see vm_start_nolock for parameters, additionally:
4790 # migrate_opts:
4791 #   storagemap = parsed storage map for allocating NBD disks
4792 sub vm_start {
4793     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4794
4795     return PVE::QemuConfig->lock_config($vmid, sub {
4796         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4797
4798         die "you can't start a vm if it's a template\n"
4799             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4800
4801         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4802
4803         PVE::QemuConfig->check_lock($conf)
4804             if !($params->{skiplock} || $has_suspended_lock);
4805
4806         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4807
4808         die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4809
4810         if (my $storagemap = $migrate_opts->{storagemap}) {
4811             my $replicated = $migrate_opts->{replicated_volumes};
4812             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4813             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4814
4815             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4816                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4817             }
4818         }
4819
4820         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4821     });
4822 }
4823
4824
4825 # params:
4826 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4827 #   skiplock => 0/1, skip checking for config lock
4828 #   skiptemplate => 0/1, skip checking whether VM is template
4829 #   forcemachine => to force Qemu machine (rollback/migration)
4830 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4831 #   timeout => in seconds
4832 #   paused => start VM in paused state (backup)
4833 #   resume => resume from hibernation
4834 # migrate_opts:
4835 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4836 #   migratedfrom => source node
4837 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4838 #   network => CIDR of migration network
4839 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4840 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4841 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4842 sub vm_start_nolock {
4843     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4844
4845     my $statefile = $params->{statefile};
4846     my $resume = $params->{resume};
4847
4848     my $migratedfrom = $migrate_opts->{migratedfrom};
4849     my $migration_type = $migrate_opts->{type};
4850
4851     my $res = {};
4852
4853     # clean up leftover reboot request files
4854     eval { clear_reboot_request($vmid); };
4855     warn $@ if $@;
4856
4857     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4858         vmconfig_apply_pending($vmid, $conf, $storecfg);
4859         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4860     }
4861
4862     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4863
4864     my $defaults = load_defaults();
4865
4866     # set environment variable useful inside network script
4867     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4868
4869     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4870
4871     my $forcemachine = $params->{forcemachine};
4872     my $forcecpu = $params->{forcecpu};
4873     if ($resume) {
4874         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4875         $forcemachine = $conf->{runningmachine};
4876         $forcecpu = $conf->{runningcpu};
4877         print "Resuming suspended VM\n";
4878     }
4879
4880     my ($cmd, $vollist, $spice_port) =
4881         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4882
4883     my $migration_ip;
4884     my $get_migration_ip = sub {
4885         my ($nodename) = @_;
4886
4887         return $migration_ip if defined($migration_ip);
4888
4889         my $cidr = $migrate_opts->{network};
4890
4891         if (!defined($cidr)) {
4892             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4893             $cidr = $dc_conf->{migration}->{network};
4894         }
4895
4896         if (defined($cidr)) {
4897             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4898
4899             die "could not get IP: no address configured on local " .
4900                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4901
4902             die "could not get IP: multiple addresses configured on local " .
4903                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4904
4905             $migration_ip = @$ips[0];
4906         }
4907
4908         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4909             if !defined($migration_ip);
4910
4911         return $migration_ip;
4912     };
4913
4914     my $migrate_uri;
4915     if ($statefile) {
4916         if ($statefile eq 'tcp') {
4917             my $localip = "localhost";
4918             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4919             my $nodename = nodename();
4920
4921             if (!defined($migration_type)) {
4922                 if (defined($datacenterconf->{migration}->{type})) {
4923                     $migration_type = $datacenterconf->{migration}->{type};
4924                 } else {
4925                     $migration_type = 'secure';
4926                 }
4927             }
4928
4929             if ($migration_type eq 'insecure') {
4930                 $localip = $get_migration_ip->($nodename);
4931                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4932             }
4933
4934             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4935             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4936             $migrate_uri = "tcp:${localip}:${migrate_port}";
4937             push @$cmd, '-incoming', $migrate_uri;
4938             push @$cmd, '-S';
4939
4940         } elsif ($statefile eq 'unix') {
4941             # should be default for secure migrations as a ssh TCP forward
4942             # tunnel is not deterministic reliable ready and fails regurarly
4943             # to set up in time, so use UNIX socket forwards
4944             my $socket_addr = "/run/qemu-server/$vmid.migrate";
4945             unlink $socket_addr;
4946
4947             $migrate_uri = "unix:$socket_addr";
4948
4949             push @$cmd, '-incoming', $migrate_uri;
4950             push @$cmd, '-S';
4951
4952         } elsif (-e $statefile) {
4953             push @$cmd, '-loadstate', $statefile;
4954         } else {
4955             my $statepath = PVE::Storage::path($storecfg, $statefile);
4956             push @$vollist, $statefile;
4957             push @$cmd, '-loadstate', $statepath;
4958         }
4959     } elsif ($params->{paused}) {
4960         push @$cmd, '-S';
4961     }
4962
4963     # host pci devices
4964     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
4965       my $d = parse_hostpci($conf->{"hostpci$i"});
4966       next if !$d;
4967       my $pcidevices = $d->{pciid};
4968       foreach my $pcidevice (@$pcidevices) {
4969             my $pciid = $pcidevice->{id};
4970
4971             my $info = PVE::SysFSTools::pci_device_info("$pciid");
4972             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4973             die "no pci device info for device '$pciid'\n" if !$info;
4974
4975             if ($d->{mdev}) {
4976                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4977                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4978             } else {
4979                 die "can't unbind/bind pci group to vfio '$pciid'\n"
4980                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4981                 die "can't reset pci device '$pciid'\n"
4982                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4983             }
4984       }
4985     }
4986
4987     PVE::Storage::activate_volumes($storecfg, $vollist);
4988
4989     eval {
4990         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4991             outfunc => sub {}, errfunc => sub {});
4992     };
4993     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4994     # timeout should be more than enough here...
4995     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4996
4997     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4998                                               : $defaults->{cpuunits};
4999
5000     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5001     my %run_params = (
5002         timeout => $statefile ? undef : $start_timeout,
5003         umask => 0077,
5004         noerr => 1,
5005     );
5006
5007     # when migrating, prefix QEMU output so other side can pick up any
5008     # errors that might occur and show the user
5009     if ($migratedfrom) {
5010         $run_params{quiet} = 1;
5011         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5012     }
5013
5014     my %properties = (
5015         Slice => 'qemu.slice',
5016         KillMode => 'none',
5017         CPUShares => $cpuunits
5018     );
5019
5020     if (my $cpulimit = $conf->{cpulimit}) {
5021         $properties{CPUQuota} = int($cpulimit * 100);
5022     }
5023     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5024
5025     my $run_qemu = sub {
5026         PVE::Tools::run_fork sub {
5027             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5028
5029             my $exitcode = run_command($cmd, %run_params);
5030             die "QEMU exited with code $exitcode\n" if $exitcode;
5031         };
5032     };
5033
5034     if ($conf->{hugepages}) {
5035
5036         my $code = sub {
5037             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5038             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5039
5040             PVE::QemuServer::Memory::hugepages_mount();
5041             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5042
5043             eval { $run_qemu->() };
5044             if (my $err = $@) {
5045                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5046                     if !$conf->{keephugepages};
5047                 die $err;
5048             }
5049
5050             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5051                 if !$conf->{keephugepages};
5052         };
5053         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5054
5055     } else {
5056         eval { $run_qemu->() };
5057     }
5058
5059     if (my $err = $@) {
5060         # deactivate volumes if start fails
5061         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5062         die "start failed: $err";
5063     }
5064
5065     print "migration listens on $migrate_uri\n" if $migrate_uri;
5066     $res->{migrate_uri} = $migrate_uri;
5067
5068     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5069         eval { mon_cmd($vmid, "cont"); };
5070         warn $@ if $@;
5071     }
5072
5073     #start nbd server for storage migration
5074     if (my $nbd = $migrate_opts->{nbd}) {
5075         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5076
5077         my $migrate_storage_uri;
5078         # nbd_protocol_version > 0 for unix socket support
5079         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5080             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5081             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5082             $migrate_storage_uri = "nbd:unix:$socket_path";
5083         } else {
5084             my $nodename = nodename();
5085             my $localip = $get_migration_ip->($nodename);
5086             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5087             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5088
5089             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5090             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5091             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5092         }
5093
5094         $res->{migrate_storage_uri} = $migrate_storage_uri;
5095
5096         foreach my $opt (sort keys %$nbd) {
5097             my $drivestr = $nbd->{$opt}->{drivestr};
5098             my $volid = $nbd->{$opt}->{volid};
5099             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5100             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5101             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5102             print "re-using replicated volume: $opt - $volid\n"
5103                 if $nbd->{$opt}->{replicated};
5104
5105             $res->{drives}->{$opt} = $nbd->{$opt};
5106             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5107         }
5108     }
5109
5110     if ($migratedfrom) {
5111         eval {
5112             set_migration_caps($vmid);
5113         };
5114         warn $@ if $@;
5115
5116         if ($spice_port) {
5117             print "spice listens on port $spice_port\n";
5118             $res->{spice_port} = $spice_port;
5119             if ($migrate_opts->{spice_ticket}) {
5120                 mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
5121                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5122             }
5123         }
5124
5125     } else {
5126         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5127             if !$statefile && $conf->{balloon};
5128
5129         foreach my $opt (keys %$conf) {
5130             next if $opt !~  m/^net\d+$/;
5131             my $nicconf = parse_net($conf->{$opt});
5132             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5133         }
5134     }
5135
5136     mon_cmd($vmid, 'qom-set',
5137                 path => "machine/peripheral/balloon0",
5138                 property => "guest-stats-polling-interval",
5139                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5140
5141     if ($resume) {
5142         print "Resumed VM, removing state\n";
5143         if (my $vmstate = $conf->{vmstate}) {
5144             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5145             PVE::Storage::vdisk_free($storecfg, $vmstate);
5146         }
5147         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5148         PVE::QemuConfig->write_config($vmid, $conf);
5149     }
5150
5151     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5152
5153     return $res;
5154 }
5155
5156 sub vm_commandline {
5157     my ($storecfg, $vmid, $snapname) = @_;
5158
5159     my $conf = PVE::QemuConfig->load_config($vmid);
5160     my $forcemachine;
5161     my $forcecpu;
5162
5163     if ($snapname) {
5164         my $snapshot = $conf->{snapshots}->{$snapname};
5165         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5166
5167         # check for machine or CPU overrides in snapshot
5168         $forcemachine = $snapshot->{runningmachine};
5169         $forcecpu = $snapshot->{runningcpu};
5170
5171         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5172
5173         $conf = $snapshot;
5174     }
5175
5176     my $defaults = load_defaults();
5177
5178     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5179         $forcemachine, $forcecpu);
5180
5181     return PVE::Tools::cmd2string($cmd);
5182 }
5183
5184 sub vm_reset {
5185     my ($vmid, $skiplock) = @_;
5186
5187     PVE::QemuConfig->lock_config($vmid, sub {
5188
5189         my $conf = PVE::QemuConfig->load_config($vmid);
5190
5191         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5192
5193         mon_cmd($vmid, "system_reset");
5194     });
5195 }
5196
5197 sub get_vm_volumes {
5198     my ($conf) = @_;
5199
5200     my $vollist = [];
5201     foreach_volid($conf, sub {
5202         my ($volid, $attr) = @_;
5203
5204         return if $volid =~ m|^/|;
5205
5206         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5207         return if !$sid;
5208
5209         push @$vollist, $volid;
5210     });
5211
5212     return $vollist;
5213 }
5214
5215 sub vm_stop_cleanup {
5216     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5217
5218     eval {
5219
5220         if (!$keepActive) {
5221             my $vollist = get_vm_volumes($conf);
5222             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5223         }
5224
5225         foreach my $ext (qw(mon qmp pid vnc qga)) {
5226             unlink "/var/run/qemu-server/${vmid}.$ext";
5227         }
5228
5229         if ($conf->{ivshmem}) {
5230             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5231             # just delete it for now, VMs which have this already open do not
5232             # are affected, but new VMs will get a separated one. If this
5233             # becomes an issue we either add some sort of ref-counting or just
5234             # add a "don't delete on stop" flag to the ivshmem format.
5235             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5236         }
5237
5238         foreach my $key (keys %$conf) {
5239             next if $key !~ m/^hostpci(\d+)$/;
5240             my $hostpciindex = $1;
5241             my $d = parse_hostpci($conf->{$key});
5242             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5243
5244             foreach my $pci (@{$d->{pciid}}) {
5245                 my $pciid = $pci->{id};
5246                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5247             }
5248         }
5249
5250         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5251     };
5252     warn $@ if $@; # avoid errors - just warn
5253 }
5254
5255 # call only in locked context
5256 sub _do_vm_stop {
5257     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5258
5259     my $pid = check_running($vmid, $nocheck);
5260     return if !$pid;
5261
5262     my $conf;
5263     if (!$nocheck) {
5264         $conf = PVE::QemuConfig->load_config($vmid);
5265         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5266         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5267             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5268             $timeout = $opts->{down} if $opts->{down};
5269         }
5270         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5271     }
5272
5273     eval {
5274         if ($shutdown) {
5275             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5276                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5277             } else {
5278                 mon_cmd($vmid, "system_powerdown");
5279             }
5280         } else {
5281             mon_cmd($vmid, "quit");
5282         }
5283     };
5284     my $err = $@;
5285
5286     if (!$err) {
5287         $timeout = 60 if !defined($timeout);
5288
5289         my $count = 0;
5290         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5291             $count++;
5292             sleep 1;
5293         }
5294
5295         if ($count >= $timeout) {
5296             if ($force) {
5297                 warn "VM still running - terminating now with SIGTERM\n";
5298                 kill 15, $pid;
5299             } else {
5300                 die "VM quit/powerdown failed - got timeout\n";
5301             }
5302         } else {
5303             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5304             return;
5305         }
5306     } else {
5307        if (!check_running($vmid, $nocheck)) {
5308             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5309             return;
5310        }
5311        if ($force) {
5312             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5313             kill 15, $pid;
5314         } else {
5315             die "VM quit/powerdown failed\n";
5316         }
5317     }
5318
5319     # wait again
5320     $timeout = 10;
5321
5322     my $count = 0;
5323     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5324         $count++;
5325         sleep 1;
5326     }
5327
5328     if ($count >= $timeout) {
5329         warn "VM still running - terminating now with SIGKILL\n";
5330         kill 9, $pid;
5331         sleep 1;
5332     }
5333
5334     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5335 }
5336
5337 # Note: use $nocheck to skip tests if VM configuration file exists.
5338 # We need that when migration VMs to other nodes (files already moved)
5339 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5340 sub vm_stop {
5341     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5342
5343     $force = 1 if !defined($force) && !$shutdown;
5344
5345     if ($migratedfrom){
5346         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5347         kill 15, $pid if $pid;
5348         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5349         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5350         return;
5351     }
5352
5353     PVE::QemuConfig->lock_config($vmid, sub {
5354         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5355    });
5356 }
5357
5358 sub vm_reboot {
5359     my ($vmid, $timeout) = @_;
5360
5361     PVE::QemuConfig->lock_config($vmid, sub {
5362         eval {
5363
5364             # only reboot if running, as qmeventd starts it again on a stop event
5365             return if !check_running($vmid);
5366
5367             create_reboot_request($vmid);
5368
5369             my $storecfg = PVE::Storage::config();
5370             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5371
5372         };
5373         if (my $err = $@) {
5374             # avoid that the next normal shutdown will be confused for a reboot
5375             clear_reboot_request($vmid);
5376             die $err;
5377         }
5378    });
5379 }
5380
5381 # note: if using the statestorage parameter, the caller has to check privileges
5382 sub vm_suspend {
5383     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5384
5385     my $conf;
5386     my $path;
5387     my $storecfg;
5388     my $vmstate;
5389
5390     PVE::QemuConfig->lock_config($vmid, sub {
5391
5392         $conf = PVE::QemuConfig->load_config($vmid);
5393
5394         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5395         PVE::QemuConfig->check_lock($conf)
5396             if !($skiplock || $is_backing_up);
5397
5398         die "cannot suspend to disk during backup\n"
5399             if $is_backing_up && $includestate;
5400
5401         if ($includestate) {
5402             $conf->{lock} = 'suspending';
5403             my $date = strftime("%Y-%m-%d", localtime(time()));
5404             $storecfg = PVE::Storage::config();
5405             if (!$statestorage) {
5406                 $statestorage = find_vmstate_storage($conf, $storecfg);
5407                 # check permissions for the storage
5408                 my $rpcenv = PVE::RPCEnvironment::get();
5409                 if ($rpcenv->{type} ne 'cli') {
5410                     my $authuser = $rpcenv->get_user();
5411                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5412                 }
5413             }
5414
5415
5416             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5417             $path = PVE::Storage::path($storecfg, $vmstate);
5418             PVE::QemuConfig->write_config($vmid, $conf);
5419         } else {
5420             mon_cmd($vmid, "stop");
5421         }
5422     });
5423
5424     if ($includestate) {
5425         # save vm state
5426         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5427
5428         eval {
5429             mon_cmd($vmid, "savevm-start", statefile => $path);
5430             for(;;) {
5431                 my $state = mon_cmd($vmid, "query-savevm");
5432                 if (!$state->{status}) {
5433                     die "savevm not active\n";
5434                 } elsif ($state->{status} eq 'active') {
5435                     sleep(1);
5436                     next;
5437                 } elsif ($state->{status} eq 'completed') {
5438                     print "State saved, quitting\n";
5439                     last;
5440                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5441                     die "query-savevm failed with error '$state->{error}'\n"
5442                 } else {
5443                     die "query-savevm returned status '$state->{status}'\n";
5444                 }
5445             }
5446         };
5447         my $err = $@;
5448
5449         PVE::QemuConfig->lock_config($vmid, sub {
5450             $conf = PVE::QemuConfig->load_config($vmid);
5451             if ($err) {
5452                 # cleanup, but leave suspending lock, to indicate something went wrong
5453                 eval {
5454                     mon_cmd($vmid, "savevm-end");
5455                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5456                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5457                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5458                     PVE::QemuConfig->write_config($vmid, $conf);
5459                 };
5460                 warn $@ if $@;
5461                 die $err;
5462             }
5463
5464             die "lock changed unexpectedly\n"
5465                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5466
5467             mon_cmd($vmid, "quit");
5468             $conf->{lock} = 'suspended';
5469             PVE::QemuConfig->write_config($vmid, $conf);
5470         });
5471     }
5472 }
5473
5474 sub vm_resume {
5475     my ($vmid, $skiplock, $nocheck) = @_;
5476
5477     PVE::QemuConfig->lock_config($vmid, sub {
5478         my $res = mon_cmd($vmid, 'query-status');
5479         my $resume_cmd = 'cont';
5480
5481         if ($res->{status} && $res->{status} eq 'suspended') {
5482             $resume_cmd = 'system_wakeup';
5483         }
5484
5485         if (!$nocheck) {
5486
5487             my $conf = PVE::QemuConfig->load_config($vmid);
5488
5489             PVE::QemuConfig->check_lock($conf)
5490                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5491         }
5492
5493         mon_cmd($vmid, $resume_cmd);
5494     });
5495 }
5496
5497 sub vm_sendkey {
5498     my ($vmid, $skiplock, $key) = @_;
5499
5500     PVE::QemuConfig->lock_config($vmid, sub {
5501
5502         my $conf = PVE::QemuConfig->load_config($vmid);
5503
5504         # there is no qmp command, so we use the human monitor command
5505         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5506         die $res if $res ne '';
5507     });
5508 }
5509
5510 # vzdump restore implementaion
5511
5512 sub tar_archive_read_firstfile {
5513     my $archive = shift;
5514
5515     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5516
5517     # try to detect archive type first
5518     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5519         die "unable to open file '$archive'\n";
5520     my $firstfile = <$fh>;
5521     kill 15, $pid;
5522     close $fh;
5523
5524     die "ERROR: archive contaions no data\n" if !$firstfile;
5525     chomp $firstfile;
5526
5527     return $firstfile;
5528 }
5529
5530 sub tar_restore_cleanup {
5531     my ($storecfg, $statfile) = @_;
5532
5533     print STDERR "starting cleanup\n";
5534
5535     if (my $fd = IO::File->new($statfile, "r")) {
5536         while (defined(my $line = <$fd>)) {
5537             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5538                 my $volid = $2;
5539                 eval {
5540                     if ($volid =~ m|^/|) {
5541                         unlink $volid || die 'unlink failed\n';
5542                     } else {
5543                         PVE::Storage::vdisk_free($storecfg, $volid);
5544                     }
5545                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5546                 };
5547                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5548             } else {
5549                 print STDERR "unable to parse line in statfile - $line";
5550             }
5551         }
5552         $fd->close();
5553     }
5554 }
5555
5556 sub restore_file_archive {
5557     my ($archive, $vmid, $user, $opts) = @_;
5558
5559     return restore_vma_archive($archive, $vmid, $user, $opts)
5560         if $archive eq '-';
5561
5562     my $info = PVE::Storage::archive_info($archive);
5563     my $format = $opts->{format} // $info->{format};
5564     my $comp = $info->{compression};
5565
5566     # try to detect archive format
5567     if ($format eq 'tar') {
5568         return restore_tar_archive($archive, $vmid, $user, $opts);
5569     } else {
5570         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5571     }
5572 }
5573
5574 # hepler to remove disks that will not be used after restore
5575 my $restore_cleanup_oldconf = sub {
5576     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5577
5578     PVE::QemuConfig->foreach_volume($oldconf, sub {
5579         my ($ds, $drive) = @_;
5580
5581         return if drive_is_cdrom($drive, 1);
5582
5583         my $volid = $drive->{file};
5584         return if !$volid || $volid =~ m|^/|;
5585
5586         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5587         return if !$path || !$owner || ($owner != $vmid);
5588
5589         # Note: only delete disk we want to restore
5590         # other volumes will become unused
5591         if ($virtdev_hash->{$ds}) {
5592             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5593             if (my $err = $@) {
5594                 warn $err;
5595             }
5596         }
5597     });
5598
5599     # delete vmstate files, after the restore we have no snapshots anymore
5600     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5601         my $snap = $oldconf->{snapshots}->{$snapname};
5602         if ($snap->{vmstate}) {
5603             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5604             if (my $err = $@) {
5605                 warn $err;
5606             }
5607         }
5608     }
5609 };
5610
5611 # Helper to parse vzdump backup device hints
5612 #
5613 # $rpcenv: Environment, used to ckeck storage permissions
5614 # $user: User ID, to check storage permissions
5615 # $storecfg: Storage configuration
5616 # $fh: the file handle for reading the configuration
5617 # $devinfo: should contain device sizes for all backu-up'ed devices
5618 # $options: backup options (pool, default storage)
5619 #
5620 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5621 my $parse_backup_hints = sub {
5622     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5623
5624     my $virtdev_hash = {};
5625
5626     while (defined(my $line = <$fh>)) {
5627         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5628             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5629             die "archive does not contain data for drive '$virtdev'\n"
5630                 if !$devinfo->{$devname};
5631
5632             if (defined($options->{storage})) {
5633                 $storeid = $options->{storage} || 'local';
5634             } elsif (!$storeid) {
5635                 $storeid = 'local';
5636             }
5637             $format = 'raw' if !$format;
5638             $devinfo->{$devname}->{devname} = $devname;
5639             $devinfo->{$devname}->{virtdev} = $virtdev;
5640             $devinfo->{$devname}->{format} = $format;
5641             $devinfo->{$devname}->{storeid} = $storeid;
5642
5643             # check permission on storage
5644             my $pool = $options->{pool}; # todo: do we need that?
5645             if ($user ne 'root@pam') {
5646                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5647             }
5648
5649             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5650         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5651             my $virtdev = $1;
5652             my $drive = parse_drive($virtdev, $2);
5653             if (drive_is_cloudinit($drive)) {
5654                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5655                 $storeid = $options->{storage} if defined ($options->{storage});
5656                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5657                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5658
5659                 $virtdev_hash->{$virtdev} = {
5660                     format => $format,
5661                     storeid => $storeid,
5662                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5663                     is_cloudinit => 1,
5664                 };
5665             }
5666         }
5667     }
5668
5669     return $virtdev_hash;
5670 };
5671
5672 # Helper to allocate and activate all volumes required for a restore
5673 #
5674 # $storecfg: Storage configuration
5675 # $virtdev_hash: as returned by parse_backup_hints()
5676 #
5677 # Returns: { $virtdev => $volid }
5678 my $restore_allocate_devices = sub {
5679     my ($storecfg, $virtdev_hash, $vmid) = @_;
5680
5681     my $map = {};
5682     foreach my $virtdev (sort keys %$virtdev_hash) {
5683         my $d = $virtdev_hash->{$virtdev};
5684         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5685         my $storeid = $d->{storeid};
5686         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5687
5688         # test if requested format is supported
5689         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5690         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5691         $d->{format} = $defFormat if !$supported;
5692
5693         my $name;
5694         if ($d->{is_cloudinit}) {
5695             $name = "vm-$vmid-cloudinit";
5696             $name .= ".$d->{format}" if $d->{format} ne 'raw';
5697         }
5698
5699         my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5700
5701         print STDERR "new volume ID is '$volid'\n";
5702         $d->{volid} = $volid;
5703
5704         PVE::Storage::activate_volumes($storecfg, [$volid]);
5705
5706         $map->{$virtdev} = $volid;
5707     }
5708
5709     return $map;
5710 };
5711
5712 my $restore_update_config_line = sub {
5713     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5714
5715     return if $line =~ m/^\#qmdump\#/;
5716     return if $line =~ m/^\#vzdump\#/;
5717     return if $line =~ m/^lock:/;
5718     return if $line =~ m/^unused\d+:/;
5719     return if $line =~ m/^parent:/;
5720
5721     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5722     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5723         # try to convert old 1.X settings
5724         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5725         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5726             my ($model, $macaddr) = split(/\=/, $devconfig);
5727             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5728             my $net = {
5729                 model => $model,
5730                 bridge => "vmbr$ind",
5731                 macaddr => $macaddr,
5732             };
5733             my $netstr = print_net($net);
5734
5735             print $outfd "net$cookie->{netcount}: $netstr\n";
5736             $cookie->{netcount}++;
5737         }
5738     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5739         my ($id, $netstr) = ($1, $2);
5740         my $net = parse_net($netstr);
5741         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5742         $netstr = print_net($net);
5743         print $outfd "$id: $netstr\n";
5744     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5745         my $virtdev = $1;
5746         my $value = $3;
5747         my $di = parse_drive($virtdev, $value);
5748         if (defined($di->{backup}) && !$di->{backup}) {
5749             print $outfd "#$line";
5750         } elsif ($map->{$virtdev}) {
5751             delete $di->{format}; # format can change on restore
5752             $di->{file} = $map->{$virtdev};
5753             $value = print_drive($di);
5754             print $outfd "$virtdev: $value\n";
5755         } else {
5756             print $outfd $line;
5757         }
5758     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5759         my $vmgenid = $1;
5760         if ($vmgenid ne '0') {
5761             # always generate a new vmgenid if there was a valid one setup
5762             $vmgenid = generate_uuid();
5763         }
5764         print $outfd "vmgenid: $vmgenid\n";
5765     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5766         my ($uuid, $uuid_str);
5767         UUID::generate($uuid);
5768         UUID::unparse($uuid, $uuid_str);
5769         my $smbios1 = parse_smbios1($2);
5770         $smbios1->{uuid} = $uuid_str;
5771         print $outfd $1.print_smbios1($smbios1)."\n";
5772     } else {
5773         print $outfd $line;
5774     }
5775 };
5776
5777 my $restore_deactivate_volumes = sub {
5778     my ($storecfg, $devinfo) = @_;
5779
5780     my $vollist = [];
5781     foreach my $devname (keys %$devinfo) {
5782         my $volid = $devinfo->{$devname}->{volid};
5783         push @$vollist, $volid if $volid;
5784     }
5785
5786     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5787 };
5788
5789 my $restore_destroy_volumes = sub {
5790     my ($storecfg, $devinfo) = @_;
5791
5792     foreach my $devname (keys %$devinfo) {
5793         my $volid = $devinfo->{$devname}->{volid};
5794         next if !$volid;
5795         eval {
5796             if ($volid =~ m|^/|) {
5797                 unlink $volid || die 'unlink failed\n';
5798             } else {
5799                 PVE::Storage::vdisk_free($storecfg, $volid);
5800             }
5801             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5802         };
5803         print STDERR "unable to cleanup '$volid' - $@" if $@;
5804     }
5805 };
5806
5807 sub scan_volids {
5808     my ($cfg, $vmid) = @_;
5809
5810     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5811
5812     my $volid_hash = {};
5813     foreach my $storeid (keys %$info) {
5814         foreach my $item (@{$info->{$storeid}}) {
5815             next if !($item->{volid} && $item->{size});
5816             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5817             $volid_hash->{$item->{volid}} = $item;
5818         }
5819     }
5820
5821     return $volid_hash;
5822 }
5823
5824 sub update_disk_config {
5825     my ($vmid, $conf, $volid_hash) = @_;
5826
5827     my $changes;
5828     my $prefix = "VM $vmid";
5829
5830     # used and unused disks
5831     my $referenced = {};
5832
5833     # Note: it is allowed to define multiple storages with same path (alias), so
5834     # we need to check both 'volid' and real 'path' (two different volid can point
5835     # to the same path).
5836
5837     my $referencedpath = {};
5838
5839     # update size info
5840     PVE::QemuConfig->foreach_volume($conf, sub {
5841         my ($opt, $drive) = @_;
5842
5843         my $volid = $drive->{file};
5844         return if !$volid;
5845
5846         # mark volid as "in-use" for next step
5847         $referenced->{$volid} = 1;
5848         if ($volid_hash->{$volid} &&
5849             (my $path = $volid_hash->{$volid}->{path})) {
5850             $referencedpath->{$path} = 1;
5851         }
5852
5853         return if drive_is_cdrom($drive);
5854         return if !$volid_hash->{$volid};
5855
5856         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size});
5857         if (defined($updated)) {
5858             $changes = 1;
5859             $conf->{$opt} = print_drive($updated);
5860             print "$prefix ($opt): $msg\n";
5861         }
5862     });
5863
5864     # remove 'unusedX' entry if volume is used
5865     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5866         my ($opt, $drive) = @_;
5867
5868         my $volid = $drive->{file};
5869         return if !$volid;
5870
5871         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5872         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5873             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5874             $changes = 1;
5875             delete $conf->{$opt};
5876         }
5877
5878         $referenced->{$volid} = 1;
5879         $referencedpath->{$path} = 1 if $path;
5880     });
5881
5882     foreach my $volid (sort keys %$volid_hash) {
5883         next if $volid =~ m/vm-$vmid-state-/;
5884         next if $referenced->{$volid};
5885         my $path = $volid_hash->{$volid}->{path};
5886         next if !$path; # just to be sure
5887         next if $referencedpath->{$path};
5888         $changes = 1;
5889         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5890         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5891         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5892     }
5893
5894     return $changes;
5895 }
5896
5897 sub rescan {
5898     my ($vmid, $nolock, $dryrun) = @_;
5899
5900     my $cfg = PVE::Storage::config();
5901
5902     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5903     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5904     foreach my $stor (keys %{$cfg->{ids}}) {
5905         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5906     }
5907
5908     print "rescan volumes...\n";
5909     my $volid_hash = scan_volids($cfg, $vmid);
5910
5911     my $updatefn =  sub {
5912         my ($vmid) = @_;
5913
5914         my $conf = PVE::QemuConfig->load_config($vmid);
5915
5916         PVE::QemuConfig->check_lock($conf);
5917
5918         my $vm_volids = {};
5919         foreach my $volid (keys %$volid_hash) {
5920             my $info = $volid_hash->{$volid};
5921             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5922         }
5923
5924         my $changes = update_disk_config($vmid, $conf, $vm_volids);
5925
5926         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5927     };
5928
5929     if (defined($vmid)) {
5930         if ($nolock) {
5931             &$updatefn($vmid);
5932         } else {
5933             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5934         }
5935     } else {
5936         my $vmlist = config_list();
5937         foreach my $vmid (keys %$vmlist) {
5938             if ($nolock) {
5939                 &$updatefn($vmid);
5940             } else {
5941                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5942             }
5943         }
5944     }
5945 }
5946
5947 sub restore_proxmox_backup_archive {
5948     my ($archive, $vmid, $user, $options) = @_;
5949
5950     my $storecfg = PVE::Storage::config();
5951
5952     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5953     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5954
5955     my $server = $scfg->{server};
5956     my $datastore = $scfg->{datastore};
5957     my $username = $scfg->{username} // 'root@pam';
5958     my $fingerprint = $scfg->{fingerprint};
5959     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
5960
5961     my $repo = "$username\@$server:$datastore";
5962
5963     # This is only used for `pbs-restore`!
5964     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5965     local $ENV{PBS_PASSWORD} = $password;
5966     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5967
5968     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5969         PVE::Storage::parse_volname($storecfg, $archive);
5970
5971     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5972
5973     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5974
5975     my $tmpdir = "/var/tmp/vzdumptmp$$";
5976     rmtree $tmpdir;
5977     mkpath $tmpdir;
5978
5979     my $conffile = PVE::QemuConfig->config_file($vmid);
5980     my $tmpfn = "$conffile.$$.tmp";
5981      # disable interrupts (always do cleanups)
5982     local $SIG{INT} =
5983         local $SIG{TERM} =
5984         local $SIG{QUIT} =
5985         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5986
5987     # Note: $oldconf is undef if VM does not exists
5988     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5989     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5990
5991     my $rpcenv = PVE::RPCEnvironment::get();
5992     my $devinfo = {};
5993
5994     eval {
5995         # enable interrupts
5996         local $SIG{INT} =
5997             local $SIG{TERM} =
5998             local $SIG{QUIT} =
5999             local $SIG{HUP} =
6000             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6001
6002         my $cfgfn = "$tmpdir/qemu-server.conf";
6003         my $firewall_config_fn = "$tmpdir/fw.conf";
6004         my $index_fn = "$tmpdir/index.json";
6005
6006         my $cmd = "restore";
6007
6008         my $param = [$pbs_backup_name, "index.json", $index_fn];
6009         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6010         my $index = PVE::Tools::file_get_contents($index_fn);
6011         $index = decode_json($index);
6012
6013         # print Dumper($index);
6014         foreach my $info (@{$index->{files}}) {
6015             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6016                 my $devname = $1;
6017                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6018                     $devinfo->{$devname}->{size} = $1;
6019                 } else {
6020                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6021                 }
6022             }
6023         }
6024
6025         my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
6026         if (!$is_qemu_server_backup) {
6027             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6028         }
6029         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6030
6031         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6032         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6033
6034         if ($has_firewall_config) {
6035             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6036             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6037
6038             my $pve_firewall_dir = '/etc/pve/firewall';
6039             mkdir $pve_firewall_dir; # make sure the dir exists
6040             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6041         }
6042
6043         my $fh = IO::File->new($cfgfn, "r") ||
6044             die "unable to read qemu-server.conf - $!\n";
6045
6046         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6047
6048         # fixme: rate limit?
6049
6050         # create empty/temp config
6051         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6052
6053         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6054
6055         # allocate volumes
6056         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6057
6058         foreach my $virtdev (sort keys %$virtdev_hash) {
6059             my $d = $virtdev_hash->{$virtdev};
6060             next if $d->{is_cloudinit}; # no need to restore cloudinit
6061
6062             my $volid = $d->{volid};
6063
6064             my $path = PVE::Storage::path($storecfg, $volid);
6065
6066             # This is the ONLY user of the PBS_ env vars set on top of this function!
6067             my $pbs_restore_cmd = [
6068                 '/usr/bin/pbs-restore',
6069                 '--repository', $repo,
6070                 $pbs_backup_name,
6071                 "$d->{devname}.img.fidx",
6072                 $path,
6073                 '--verbose',
6074                 ];
6075
6076             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6077             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6078
6079             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6080                 push @$pbs_restore_cmd, '--skip-zero';
6081             }
6082
6083             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6084             print "restore proxmox backup image: $dbg_cmdstring\n";
6085             run_command($pbs_restore_cmd);
6086         }
6087
6088         $fh->seek(0, 0) || die "seek failed - $!\n";
6089
6090         my $outfd = new IO::File ($tmpfn, "w") ||
6091             die "unable to write config for VM $vmid\n";
6092
6093         my $cookie = { netcount => 0 };
6094         while (defined(my $line = <$fh>)) {
6095             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6096         }
6097
6098         $fh->close();
6099         $outfd->close();
6100     };
6101     my $err = $@;
6102
6103     $restore_deactivate_volumes->($storecfg, $devinfo);
6104
6105     rmtree $tmpdir;
6106
6107     if ($err) {
6108         unlink $tmpfn;
6109         $restore_destroy_volumes->($storecfg, $devinfo);
6110         die $err;
6111     }
6112
6113     rename($tmpfn, $conffile) ||
6114         die "unable to commit configuration file '$conffile'\n";
6115
6116     PVE::Cluster::cfs_update(); # make sure we read new file
6117
6118     eval { rescan($vmid, 1); };
6119     warn $@ if $@;
6120 }
6121
6122 sub restore_vma_archive {
6123     my ($archive, $vmid, $user, $opts, $comp) = @_;
6124
6125     my $readfrom = $archive;
6126
6127     my $cfg = PVE::Storage::config();
6128     my $commands = [];
6129     my $bwlimit = $opts->{bwlimit};
6130
6131     my $dbg_cmdstring = '';
6132     my $add_pipe = sub {
6133         my ($cmd) = @_;
6134         push @$commands, $cmd;
6135         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6136         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6137         $readfrom = '-';
6138     };
6139
6140     my $input = undef;
6141     if ($archive eq '-') {
6142         $input = '<&STDIN';
6143     } else {
6144         # If we use a backup from a PVE defined storage we also consider that
6145         # storage's rate limit:
6146         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6147         if (defined($volid)) {
6148             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6149             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6150             if ($readlimit) {
6151                 print STDERR "applying read rate limit: $readlimit\n";
6152                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6153                 $add_pipe->($cstream);
6154             }
6155         }
6156     }
6157
6158     if ($comp) {
6159         my $info = PVE::Storage::decompressor_info('vma', $comp);
6160         my $cmd = $info->{decompressor};
6161         push @$cmd, $readfrom;
6162         $add_pipe->($cmd);
6163     }
6164
6165     my $tmpdir = "/var/tmp/vzdumptmp$$";
6166     rmtree $tmpdir;
6167
6168     # disable interrupts (always do cleanups)
6169     local $SIG{INT} =
6170         local $SIG{TERM} =
6171         local $SIG{QUIT} =
6172         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6173
6174     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6175     POSIX::mkfifo($mapfifo, 0600);
6176     my $fifofh;
6177
6178     my $openfifo = sub {
6179         open($fifofh, '>', $mapfifo) || die $!;
6180     };
6181
6182     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6183
6184     my $oldtimeout;
6185     my $timeout = 5;
6186
6187     my $devinfo = {};
6188
6189     my $rpcenv = PVE::RPCEnvironment::get();
6190
6191     my $conffile = PVE::QemuConfig->config_file($vmid);
6192     my $tmpfn = "$conffile.$$.tmp";
6193
6194     # Note: $oldconf is undef if VM does not exist
6195     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6196     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6197
6198     my %storage_limits;
6199
6200     my $print_devmap = sub {
6201         my $cfgfn = "$tmpdir/qemu-server.conf";
6202
6203         # we can read the config - that is already extracted
6204         my $fh = IO::File->new($cfgfn, "r") ||
6205             die "unable to read qemu-server.conf - $!\n";
6206
6207         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6208         if (-f $fwcfgfn) {
6209             my $pve_firewall_dir = '/etc/pve/firewall';
6210             mkdir $pve_firewall_dir; # make sure the dir exists
6211             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6212         }
6213
6214         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6215
6216         foreach my $key (keys %storage_limits) {
6217             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6218             next if !$limit;
6219             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6220             $storage_limits{$key} = $limit * 1024;
6221         }
6222
6223         foreach my $devname (keys %$devinfo) {
6224             die "found no device mapping information for device '$devname'\n"
6225                 if !$devinfo->{$devname}->{virtdev};
6226         }
6227
6228         # create empty/temp config
6229         if ($oldconf) {
6230             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6231             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6232         }
6233
6234         # allocate volumes
6235         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6236
6237         # print restore information to $fifofh
6238         foreach my $virtdev (sort keys %$virtdev_hash) {
6239             my $d = $virtdev_hash->{$virtdev};
6240             next if $d->{is_cloudinit}; # no need to restore cloudinit
6241
6242             my $storeid = $d->{storeid};
6243             my $volid = $d->{volid};
6244
6245             my $map_opts = '';
6246             if (my $limit = $storage_limits{$storeid}) {
6247                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6248             }
6249
6250             my $write_zeros = 1;
6251             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6252                 $write_zeros = 0;
6253             }
6254
6255             my $path = PVE::Storage::path($cfg, $volid);
6256
6257             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6258
6259             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6260         }
6261
6262         $fh->seek(0, 0) || die "seek failed - $!\n";
6263
6264         my $outfd = new IO::File ($tmpfn, "w") ||
6265             die "unable to write config for VM $vmid\n";
6266
6267         my $cookie = { netcount => 0 };
6268         while (defined(my $line = <$fh>)) {
6269             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6270         }
6271
6272         $fh->close();
6273         $outfd->close();
6274     };
6275
6276     eval {
6277         # enable interrupts
6278         local $SIG{INT} =
6279             local $SIG{TERM} =
6280             local $SIG{QUIT} =
6281             local $SIG{HUP} =
6282             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6283         local $SIG{ALRM} = sub { die "got timeout\n"; };
6284
6285         $oldtimeout = alarm($timeout);
6286
6287         my $parser = sub {
6288             my $line = shift;
6289
6290             print "$line\n";
6291
6292             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6293                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6294                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6295             } elsif ($line =~ m/^CTIME: /) {
6296                 # we correctly received the vma config, so we can disable
6297                 # the timeout now for disk allocation (set to 10 minutes, so
6298                 # that we always timeout if something goes wrong)
6299                 alarm(600);
6300                 &$print_devmap();
6301                 print $fifofh "done\n";
6302                 my $tmp = $oldtimeout || 0;
6303                 $oldtimeout = undef;
6304                 alarm($tmp);
6305                 close($fifofh);
6306             }
6307         };
6308
6309         print "restore vma archive: $dbg_cmdstring\n";
6310         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6311     };
6312     my $err = $@;
6313
6314     alarm($oldtimeout) if $oldtimeout;
6315
6316     $restore_deactivate_volumes->($cfg, $devinfo);
6317
6318     unlink $mapfifo;
6319     rmtree $tmpdir;
6320
6321     if ($err) {
6322         unlink $tmpfn;
6323         $restore_destroy_volumes->($cfg, $devinfo);
6324         die $err;
6325     }
6326
6327     rename($tmpfn, $conffile) ||
6328         die "unable to commit configuration file '$conffile'\n";
6329
6330     PVE::Cluster::cfs_update(); # make sure we read new file
6331
6332     eval { rescan($vmid, 1); };
6333     warn $@ if $@;
6334 }
6335
6336 sub restore_tar_archive {
6337     my ($archive, $vmid, $user, $opts) = @_;
6338
6339     if ($archive ne '-') {
6340         my $firstfile = tar_archive_read_firstfile($archive);
6341         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6342             if $firstfile ne 'qemu-server.conf';
6343     }
6344
6345     my $storecfg = PVE::Storage::config();
6346
6347     # avoid zombie disks when restoring over an existing VM -> cleanup first
6348     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6349     # skiplock=1 because qmrestore has set the 'create' lock itself already
6350     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6351     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6352
6353     my $tocmd = "/usr/lib/qemu-server/qmextract";
6354
6355     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6356     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6357     $tocmd .= ' --prealloc' if $opts->{prealloc};
6358     $tocmd .= ' --info' if $opts->{info};
6359
6360     # tar option "xf" does not autodetect compression when read from STDIN,
6361     # so we pipe to zcat
6362     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6363         PVE::Tools::shellquote("--to-command=$tocmd");
6364
6365     my $tmpdir = "/var/tmp/vzdumptmp$$";
6366     mkpath $tmpdir;
6367
6368     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6369     local $ENV{VZDUMP_VMID} = $vmid;
6370     local $ENV{VZDUMP_USER} = $user;
6371
6372     my $conffile = PVE::QemuConfig->config_file($vmid);
6373     my $tmpfn = "$conffile.$$.tmp";
6374
6375     # disable interrupts (always do cleanups)
6376     local $SIG{INT} =
6377         local $SIG{TERM} =
6378         local $SIG{QUIT} =
6379         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6380
6381     eval {
6382         # enable interrupts
6383         local $SIG{INT} =
6384             local $SIG{TERM} =
6385             local $SIG{QUIT} =
6386             local $SIG{HUP} =
6387             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6388
6389         if ($archive eq '-') {
6390             print "extracting archive from STDIN\n";
6391             run_command($cmd, input => "<&STDIN");
6392         } else {
6393             print "extracting archive '$archive'\n";
6394             run_command($cmd);
6395         }
6396
6397         return if $opts->{info};
6398
6399         # read new mapping
6400         my $map = {};
6401         my $statfile = "$tmpdir/qmrestore.stat";
6402         if (my $fd = IO::File->new($statfile, "r")) {
6403             while (defined (my $line = <$fd>)) {
6404                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6405                     $map->{$1} = $2 if $1;
6406                 } else {
6407                     print STDERR "unable to parse line in statfile - $line\n";
6408                 }
6409             }
6410             $fd->close();
6411         }
6412
6413         my $confsrc = "$tmpdir/qemu-server.conf";
6414
6415         my $srcfd = new IO::File($confsrc, "r") ||
6416             die "unable to open file '$confsrc'\n";
6417
6418         my $outfd = new IO::File ($tmpfn, "w") ||
6419             die "unable to write config for VM $vmid\n";
6420
6421         my $cookie = { netcount => 0 };
6422         while (defined (my $line = <$srcfd>)) {
6423             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6424         }
6425
6426         $srcfd->close();
6427         $outfd->close();
6428     };
6429     if (my $err = $@) {
6430         unlink $tmpfn;
6431         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6432         die $err;
6433     }
6434
6435     rmtree $tmpdir;
6436
6437     rename $tmpfn, $conffile ||
6438         die "unable to commit configuration file '$conffile'\n";
6439
6440     PVE::Cluster::cfs_update(); # make sure we read new file
6441
6442     eval { rescan($vmid, 1); };
6443     warn $@ if $@;
6444 };
6445
6446 sub foreach_storage_used_by_vm {
6447     my ($conf, $func) = @_;
6448
6449     my $sidhash = {};
6450
6451     PVE::QemuConfig->foreach_volume($conf, sub {
6452         my ($ds, $drive) = @_;
6453         return if drive_is_cdrom($drive);
6454
6455         my $volid = $drive->{file};
6456
6457         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6458         $sidhash->{$sid} = $sid if $sid;
6459     });
6460
6461     foreach my $sid (sort keys %$sidhash) {
6462         &$func($sid);
6463     }
6464 }
6465
6466 my $qemu_snap_storage = {
6467     rbd => 1,
6468 };
6469 sub do_snapshots_with_qemu {
6470     my ($storecfg, $volid) = @_;
6471
6472     my $storage_name = PVE::Storage::parse_volume_id($volid);
6473     my $scfg = $storecfg->{ids}->{$storage_name};
6474
6475     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6476         return 1;
6477     }
6478
6479     if ($volid =~ m/\.(qcow2|qed)$/){
6480         return 1;
6481     }
6482
6483     return undef;
6484 }
6485
6486 sub qga_check_running {
6487     my ($vmid, $nowarn) = @_;
6488
6489     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6490     if ($@) {
6491         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6492         return 0;
6493     }
6494     return 1;
6495 }
6496
6497 sub template_create {
6498     my ($vmid, $conf, $disk) = @_;
6499
6500     my $storecfg = PVE::Storage::config();
6501
6502     PVE::QemuConfig->foreach_volume($conf, sub {
6503         my ($ds, $drive) = @_;
6504
6505         return if drive_is_cdrom($drive);
6506         return if $disk && $ds ne $disk;
6507
6508         my $volid = $drive->{file};
6509         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6510
6511         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6512         $drive->{file} = $voliddst;
6513         $conf->{$ds} = print_drive($drive);
6514         PVE::QemuConfig->write_config($vmid, $conf);
6515     });
6516 }
6517
6518 sub convert_iscsi_path {
6519     my ($path) = @_;
6520
6521     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6522         my $portal = $1;
6523         my $target = $2;
6524         my $lun = $3;
6525
6526         my $initiator_name = get_initiator_name();
6527
6528         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6529                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6530     }
6531
6532     die "cannot convert iscsi path '$path', unkown format\n";
6533 }
6534
6535 sub qemu_img_convert {
6536     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6537
6538     my $storecfg = PVE::Storage::config();
6539     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6540     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6541
6542     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6543
6544     my $cachemode;
6545     my $src_path;
6546     my $src_is_iscsi = 0;
6547     my $src_format;
6548
6549     if ($src_storeid) {
6550         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6551         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6552         $src_format = qemu_img_format($src_scfg, $src_volname);
6553         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6554         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6555         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6556     } elsif (-f $src_volid) {
6557         $src_path = $src_volid;
6558         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6559             $src_format = $1;
6560         }
6561     }
6562
6563     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6564
6565     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6566     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6567     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6568     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6569
6570     my $cmd = [];
6571     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6572     push @$cmd, '-l', "snapshot.name=$snapname"
6573         if $snapname && $src_format && $src_format eq "qcow2";
6574     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6575     push @$cmd, '-T', $cachemode if defined($cachemode);
6576
6577     if ($src_is_iscsi) {
6578         push @$cmd, '--image-opts';
6579         $src_path = convert_iscsi_path($src_path);
6580     } elsif ($src_format) {
6581         push @$cmd, '-f', $src_format;
6582     }
6583
6584     if ($dst_is_iscsi) {
6585         push @$cmd, '--target-image-opts';
6586         $dst_path = convert_iscsi_path($dst_path);
6587     } else {
6588         push @$cmd, '-O', $dst_format;
6589     }
6590
6591     push @$cmd, $src_path;
6592
6593     if (!$dst_is_iscsi && $is_zero_initialized) {
6594         push @$cmd, "zeroinit:$dst_path";
6595     } else {
6596         push @$cmd, $dst_path;
6597     }
6598
6599     my $parser = sub {
6600         my $line = shift;
6601         if($line =~ m/\((\S+)\/100\%\)/){
6602             my $percent = $1;
6603             my $transferred = int($size * $percent / 100);
6604             my $remaining = $size - $transferred;
6605
6606             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6607         }
6608
6609     };
6610
6611     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6612     my $err = $@;
6613     die "copy failed: $err" if $err;
6614 }
6615
6616 sub qemu_img_format {
6617     my ($scfg, $volname) = @_;
6618
6619     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6620         return $1;
6621     } else {
6622         return "raw";
6623     }
6624 }
6625
6626 sub qemu_drive_mirror {
6627     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6628
6629     $jobs = {} if !$jobs;
6630
6631     my $qemu_target;
6632     my $format;
6633     $jobs->{"drive-$drive"} = {};
6634
6635     if ($dst_volid =~ /^nbd:/) {
6636         $qemu_target = $dst_volid;
6637         $format = "nbd";
6638     } else {
6639         my $storecfg = PVE::Storage::config();
6640         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6641
6642         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6643
6644         $format = qemu_img_format($dst_scfg, $dst_volname);
6645
6646         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6647
6648         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6649     }
6650
6651     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6652     $opts->{format} = $format if $format;
6653
6654     if (defined($src_bitmap)) {
6655         $opts->{sync} = 'incremental';
6656         $opts->{bitmap} = $src_bitmap;
6657         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6658     }
6659
6660     if (defined($bwlimit)) {
6661         $opts->{speed} = $bwlimit * 1024;
6662         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6663     } else {
6664         print "drive mirror is starting for drive-$drive\n";
6665     }
6666
6667     # if a job already runs for this device we get an error, catch it for cleanup
6668     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6669     if (my $err = $@) {
6670         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6671         warn "$@\n" if $@;
6672         die "mirroring error: $err\n";
6673     }
6674
6675     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6676 }
6677
6678 # $completion can be either
6679 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6680 # 'cancel': wait until all jobs are ready, block-job-cancel them
6681 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6682 sub qemu_drive_mirror_monitor {
6683     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6684
6685     $completion //= 'complete';
6686
6687     eval {
6688         my $err_complete = 0;
6689
6690         while (1) {
6691             die "storage migration timed out\n" if $err_complete > 300;
6692
6693             my $stats = mon_cmd($vmid, "query-block-jobs");
6694
6695             my $running_mirror_jobs = {};
6696             foreach my $stat (@$stats) {
6697                 next if $stat->{type} ne 'mirror';
6698                 $running_mirror_jobs->{$stat->{device}} = $stat;
6699             }
6700
6701             my $readycounter = 0;
6702
6703             foreach my $job (keys %$jobs) {
6704
6705                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6706                     print "$job : finished\n";
6707                     delete $jobs->{$job};
6708                     next;
6709                 }
6710
6711                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6712
6713                 my $busy = $running_mirror_jobs->{$job}->{busy};
6714                 my $ready = $running_mirror_jobs->{$job}->{ready};
6715                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6716                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6717                     my $remaining = $total - $transferred;
6718                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6719
6720                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6721                 }
6722
6723                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6724             }
6725
6726             last if scalar(keys %$jobs) == 0;
6727
6728             if ($readycounter == scalar(keys %$jobs)) {
6729                 print "all mirroring jobs are ready \n";
6730                 last if $completion eq 'skip'; #do the complete later
6731
6732                 if ($vmiddst && $vmiddst != $vmid) {
6733                     my $agent_running = $qga && qga_check_running($vmid);
6734                     if ($agent_running) {
6735                         print "freeze filesystem\n";
6736                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6737                     } else {
6738                         print "suspend vm\n";
6739                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6740                     }
6741
6742                     # if we clone a disk for a new target vm, we don't switch the disk
6743                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6744
6745                     if ($agent_running) {
6746                         print "unfreeze filesystem\n";
6747                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6748                     } else {
6749                         print "resume vm\n";
6750                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6751                     }
6752
6753                     last;
6754                 } else {
6755
6756                     foreach my $job (keys %$jobs) {
6757                         # try to switch the disk if source and destination are on the same guest
6758                         print "$job: Completing block job...\n";
6759
6760                         my $op;
6761                         if ($completion eq 'complete') {
6762                             $op = 'block-job-complete';
6763                         } elsif ($completion eq 'cancel') {
6764                             $op = 'block-job-cancel';
6765                         } else {
6766                             die "invalid completion value: $completion\n";
6767                         }
6768                         eval { mon_cmd($vmid, $op, device => $job) };
6769                         if ($@ =~ m/cannot be completed/) {
6770                             print "$job: Block job cannot be completed, try again.\n";
6771                             $err_complete++;
6772                         }else {
6773                             print "$job: Completed successfully.\n";
6774                             $jobs->{$job}->{complete} = 1;
6775                         }
6776                     }
6777                 }
6778             }
6779             sleep 1;
6780         }
6781     };
6782     my $err = $@;
6783
6784     if ($err) {
6785         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6786         die "mirroring error: $err";
6787     }
6788
6789 }
6790
6791 sub qemu_blockjobs_cancel {
6792     my ($vmid, $jobs) = @_;
6793
6794     foreach my $job (keys %$jobs) {
6795         print "$job: Cancelling block job\n";
6796         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6797         $jobs->{$job}->{cancel} = 1;
6798     }
6799
6800     while (1) {
6801         my $stats = mon_cmd($vmid, "query-block-jobs");
6802
6803         my $running_jobs = {};
6804         foreach my $stat (@$stats) {
6805             $running_jobs->{$stat->{device}} = $stat;
6806         }
6807
6808         foreach my $job (keys %$jobs) {
6809
6810             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6811                 print "$job: Done.\n";
6812                 delete $jobs->{$job};
6813             }
6814         }
6815
6816         last if scalar(keys %$jobs) == 0;
6817
6818         sleep 1;
6819     }
6820 }
6821
6822 sub clone_disk {
6823     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6824         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6825
6826     my $newvolid;
6827
6828     if (!$full) {
6829         print "create linked clone of drive $drivename ($drive->{file})\n";
6830         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6831         push @$newvollist, $newvolid;
6832     } else {
6833
6834         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6835         $storeid = $storage if $storage;
6836
6837         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6838         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6839
6840         print "create full clone of drive $drivename ($drive->{file})\n";
6841         my $name = undef;
6842         if (drive_is_cloudinit($drive)) {
6843             $name = "vm-$newvmid-cloudinit";
6844             $name .= ".$dst_format" if $dst_format ne 'raw';
6845             $snapname = undef;
6846             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6847         } elsif ($drivename eq 'efidisk0') {
6848             $size = get_efivars_size($conf);
6849         }
6850         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6851         push @$newvollist, $newvolid;
6852
6853         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6854
6855         if (drive_is_cloudinit($drive)) {
6856             goto no_data_clone;
6857         }
6858
6859         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6860         if (!$running || $snapname) {
6861             # TODO: handle bwlimits
6862             if ($drivename eq 'efidisk0') {
6863                 # the relevant data on the efidisk may be smaller than the source
6864                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6865                 # that is given by the OVMF_VARS.fd
6866                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6867                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6868                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6869             } else {
6870                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6871             }
6872         } else {
6873
6874             my $kvmver = get_running_qemu_version ($vmid);
6875             if (!min_version($kvmver, 2, 7)) {
6876                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6877                     if $drive->{iothread};
6878             }
6879
6880             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6881         }
6882     }
6883
6884 no_data_clone:
6885     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6886
6887     my $disk = $drive;
6888     $disk->{format} = undef;
6889     $disk->{file} = $newvolid;
6890     $disk->{size} = $size;
6891
6892     return $disk;
6893 }
6894
6895 sub get_running_qemu_version {
6896     my ($vmid) = @_;
6897     my $res = mon_cmd($vmid, "query-version");
6898     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6899 }
6900
6901 sub qemu_use_old_bios_files {
6902     my ($machine_type) = @_;
6903
6904     return if !$machine_type;
6905
6906     my $use_old_bios_files = undef;
6907
6908     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6909         $machine_type = $1;
6910         $use_old_bios_files = 1;
6911     } else {
6912         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6913         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6914         # load new efi bios files on migration. So this hack is required to allow
6915         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6916         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6917         $use_old_bios_files = !min_version($version, 2, 4);
6918     }
6919
6920     return ($use_old_bios_files, $machine_type);
6921 }
6922
6923 sub get_efivars_size {
6924     my ($conf) = @_;
6925     my $arch = get_vm_arch($conf);
6926     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6927     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
6928     return -s $ovmf_vars;
6929 }
6930
6931 sub update_efidisk_size {
6932     my ($conf) = @_;
6933
6934     return if !defined($conf->{efidisk0});
6935
6936     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
6937     $disk->{size} = get_efivars_size($conf);
6938     $conf->{efidisk0} = print_drive($disk);
6939
6940     return;
6941 }
6942
6943 sub create_efidisk($$$$$) {
6944     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6945
6946     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6947     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6948
6949     my $vars_size_b = -s $ovmf_vars;
6950     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6951     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6952     PVE::Storage::activate_volumes($storecfg, [$volid]);
6953
6954     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6955     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6956
6957     return ($volid, $size/1024);
6958 }
6959
6960 sub vm_iothreads_list {
6961     my ($vmid) = @_;
6962
6963     my $res = mon_cmd($vmid, 'query-iothreads');
6964
6965     my $iothreads = {};
6966     foreach my $iothread (@$res) {
6967         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6968     }
6969
6970     return $iothreads;
6971 }
6972
6973 sub scsihw_infos {
6974     my ($conf, $drive) = @_;
6975
6976     my $maxdev = 0;
6977
6978     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6979         $maxdev = 7;
6980     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6981         $maxdev = 1;
6982     } else {
6983         $maxdev = 256;
6984     }
6985
6986     my $controller = int($drive->{index} / $maxdev);
6987     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6988
6989     return ($maxdev, $controller, $controller_prefix);
6990 }
6991
6992 sub windows_version {
6993     my ($ostype) = @_;
6994
6995     return 0 if !$ostype;
6996
6997     my $winversion = 0;
6998
6999     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7000         $winversion = 5;
7001     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7002         $winversion = 6;
7003     } elsif ($ostype =~ m/^win(\d+)$/) {
7004         $winversion = $1;
7005     }
7006
7007     return $winversion;
7008 }
7009
7010 sub resolve_dst_disk_format {
7011         my ($storecfg, $storeid, $src_volname, $format) = @_;
7012         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7013
7014         if (!$format) {
7015             # if no target format is specified, use the source disk format as hint
7016             if ($src_volname) {
7017                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7018                 $format = qemu_img_format($scfg, $src_volname);
7019             } else {
7020                 return $defFormat;
7021             }
7022         }
7023
7024         # test if requested format is supported - else use default
7025         my $supported = grep { $_ eq $format } @$validFormats;
7026         $format = $defFormat if !$supported;
7027         return $format;
7028 }
7029
7030 # NOTE: if this logic changes, please update docs & possibly gui logic
7031 sub find_vmstate_storage {
7032     my ($conf, $storecfg) = @_;
7033
7034     # first, return storage from conf if set
7035     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7036
7037     my ($target, $shared, $local);
7038
7039     foreach_storage_used_by_vm($conf, sub {
7040         my ($sid) = @_;
7041         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7042         my $dst = $scfg->{shared} ? \$shared : \$local;
7043         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7044     });
7045
7046     # second, use shared storage where VM has at least one disk
7047     # third, use local storage where VM has at least one disk
7048     # fall back to local storage
7049     $target = $shared // $local // 'local';
7050
7051     return $target;
7052 }
7053
7054 sub generate_uuid {
7055     my ($uuid, $uuid_str);
7056     UUID::generate($uuid);
7057     UUID::unparse($uuid, $uuid_str);
7058     return $uuid_str;
7059 }
7060
7061 sub generate_smbios1_uuid {
7062     return "uuid=".generate_uuid();
7063 }
7064
7065 sub nbd_stop {
7066     my ($vmid) = @_;
7067
7068     mon_cmd($vmid, 'nbd-server-stop');
7069 }
7070
7071 sub create_reboot_request {
7072     my ($vmid) = @_;
7073     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7074         or die "failed to create reboot trigger file: $!\n";
7075     close($fh);
7076 }
7077
7078 sub clear_reboot_request {
7079     my ($vmid) = @_;
7080     my $path = "/run/qemu-server/$vmid.reboot";
7081     my $res = 0;
7082
7083     $res = unlink($path);
7084     die "could not remove reboot request for $vmid: $!"
7085         if !$res && $! != POSIX::ENOENT;
7086
7087     return $res;
7088 }
7089
7090 # bash completion helper
7091
7092 sub complete_backup_archives {
7093     my ($cmdname, $pname, $cvalue) = @_;
7094
7095     my $cfg = PVE::Storage::config();
7096
7097     my $storeid;
7098
7099     if ($cvalue =~ m/^([^:]+):/) {
7100         $storeid = $1;
7101     }
7102
7103     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7104
7105     my $res = [];
7106     foreach my $id (keys %$data) {
7107         foreach my $item (@{$data->{$id}}) {
7108             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7109             push @$res, $item->{volid} if defined($item->{volid});
7110         }
7111     }
7112
7113     return $res;
7114 }
7115
7116 my $complete_vmid_full = sub {
7117     my ($running) = @_;
7118
7119     my $idlist = vmstatus();
7120
7121     my $res = [];
7122
7123     foreach my $id (keys %$idlist) {
7124         my $d = $idlist->{$id};
7125         if (defined($running)) {
7126             next if $d->{template};
7127             next if $running && $d->{status} ne 'running';
7128             next if !$running && $d->{status} eq 'running';
7129         }
7130         push @$res, $id;
7131
7132     }
7133     return $res;
7134 };
7135
7136 sub complete_vmid {
7137     return &$complete_vmid_full();
7138 }
7139
7140 sub complete_vmid_stopped {
7141     return &$complete_vmid_full(0);
7142 }
7143
7144 sub complete_vmid_running {
7145     return &$complete_vmid_full(1);
7146 }
7147
7148 sub complete_storage {
7149
7150     my $cfg = PVE::Storage::config();
7151     my $ids = $cfg->{ids};
7152
7153     my $res = [];
7154     foreach my $sid (keys %$ids) {
7155         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7156         next if !$ids->{$sid}->{content}->{images};
7157         push @$res, $sid;
7158     }
7159
7160     return $res;
7161 }
7162
7163 sub complete_migration_storage {
7164     my ($cmd, $param, $current_value, $all_args) = @_;
7165
7166     my $targetnode = @$all_args[1];
7167
7168     my $cfg = PVE::Storage::config();
7169     my $ids = $cfg->{ids};
7170
7171     my $res = [];
7172     foreach my $sid (keys %$ids) {
7173         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7174         next if !$ids->{$sid}->{content}->{images};
7175         push @$res, $sid;
7176     }
7177
7178     return $res;
7179 }
7180
7181 1;