PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename_cache;
 112 sub nodename {
 113     $nodename_cache //= PVE::INotify::nodename();
 114     return $nodename_cache;
 115 }
 116
 117 my $cpu_vendor_list = {
 118     # Intel CPUs
 119     486 => 'GenuineIntel',
 120     pentium => 'GenuineIntel',
 121     pentium2  => 'GenuineIntel',
 122     pentium3  => 'GenuineIntel',
 123     coreduo => 'GenuineIntel',
 124     core2duo => 'GenuineIntel',
 125     Conroe  => 'GenuineIntel',
 126     Penryn  => 'GenuineIntel',
 127     Nehalem  => 'GenuineIntel',
 128     'Nehalem-IBRS'  => 'GenuineIntel',
 129     Westmere => 'GenuineIntel',
 130     'Westmere-IBRS' => 'GenuineIntel',
 131     SandyBridge => 'GenuineIntel',
 132     'SandyBridge-IBRS' => 'GenuineIntel',
 133     IvyBridge => 'GenuineIntel',
 134     'IvyBridge-IBRS' => 'GenuineIntel',
 135     Haswell => 'GenuineIntel',
 136     'Haswell-IBRS' => 'GenuineIntel',
 137     'Haswell-noTSX' => 'GenuineIntel',
 138     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 139     Broadwell => 'GenuineIntel',
 140     'Broadwell-IBRS' => 'GenuineIntel',
 141     'Broadwell-noTSX' => 'GenuineIntel',
 142     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 143     'Skylake-Client' => 'GenuineIntel',
 144     'Skylake-Client-IBRS' => 'GenuineIntel',
 145     'Skylake-Server' => 'GenuineIntel',
 146     'Skylake-Server-IBRS' => 'GenuineIntel',
 147     'Cascadelake-Server' => 'GenuineIntel',
 148     KnightsMill => 'GenuineIntel',
 149
 150
 151     # AMD CPUs
 152     athlon => 'AuthenticAMD',
 153     phenom  => 'AuthenticAMD',
 154     Opteron_G1  => 'AuthenticAMD',
 155     Opteron_G2  => 'AuthenticAMD',
 156     Opteron_G3  => 'AuthenticAMD',
 157     Opteron_G4  => 'AuthenticAMD',
 158     Opteron_G5  => 'AuthenticAMD',
 159     EPYC => 'AuthenticAMD',
 160     'EPYC-IBPB' => 'AuthenticAMD',
 161
 162     # generic types, use vendor from host node
 163     host => 'default',
 164     kvm32 => 'default',
 165     kvm64 => 'default',
 166     qemu32 => 'default',
 167     qemu64 => 'default',
 168     max => 'default',
 169 };
 170
 171 my @supported_cpu_flags = (
 172     'pcid',
 173     'spec-ctrl',
 174     'ibpb',
 175     'ssbd',
 176     'virt-ssbd',
 177     'amd-ssbd',
 178     'amd-no-ssb',
 179     'pdpe1gb',
 180     'md-clear',
 181     'hv-tlbflush',
 182     'hv-evmcs',
 183     'aes'
 184 );
 185 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 186
 187 my $cpu_fmt = {
 188     cputype => {
 189         description => "Emulated CPU type.",
 190         type => 'string',
 191         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 192         default => 'kvm64',
 193         default_key => 1,
 194     },
 195     hidden => {
 196         description => "Do not identify as a KVM virtual machine.",
 197         type => 'boolean',
 198         optional => 1,
 199         default => 0
 200     },
 201     'hv-vendor-id' => {
 202         type => 'string',
 203         pattern => qr/[a-zA-Z0-9]{1,12}/,
 204         format_description => 'vendor-id',
 205         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 206         optional => 1,
 207     },
 208     flags => {
 209         description => "List of additional CPU flags separated by ';'."
 210                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 211                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 212         format_description => '+FLAG[;-FLAG...]',
 213         type => 'string',
 214         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 215         optional => 1,
 216     },
 217 };
 218
 219 my $watchdog_fmt = {
 220     model => {
 221         default_key => 1,
 222         type => 'string',
 223         enum => [qw(i6300esb ib700)],
 224         description => "Watchdog type to emulate.",
 225         default => 'i6300esb',
 226         optional => 1,
 227     },
 228     action => {
 229         type => 'string',
 230         enum => [qw(reset shutdown poweroff pause debug none)],
 231         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 232         optional => 1,
 233     },
 234 };
 235 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 236
 237 my $agent_fmt = {
 238     enabled => {
 239         description => "Enable/disable Qemu GuestAgent.",
 240         type => 'boolean',
 241         default => 0,
 242         default_key => 1,
 243     },
 244     fstrim_cloned_disks => {
 245         description => "Run fstrim after cloning/moving a disk.",
 246         type => 'boolean',
 247         optional => 1,
 248         default => 0
 249     },
 250     type => {
 251         description => "Select the agent type",
 252         type => 'string',
 253         default => 'virtio',
 254         optional => 1,
 255         enum => [qw(virtio isa)],
 256     },
 257 };
 258
 259 my $vga_fmt = {
 260     type => {
 261         description => "Select the VGA type.",
 262         type => 'string',
 263         default => 'std',
 264         optional => 1,
 265         default_key => 1,
 266         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 267     },
 268     memory => {
 269         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 270         type => 'integer',
 271         optional => 1,
 272         minimum => 4,
 273         maximum => 512,
 274     },
 275 };
 276
 277 my $ivshmem_fmt = {
 278     size => {
 279         type => 'integer',
 280         minimum => 1,
 281         description => "The size of the file in MB.",
 282     },
 283     name => {
 284         type => 'string',
 285         pattern => '[a-zA-Z0-9\-]+',
 286         optional => 1,
 287         format_description => 'string',
 288         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 289     },
 290 };
 291
 292 my $audio_fmt = {
 293     device => {
 294         type => 'string',
 295         enum => [qw(ich9-intel-hda intel-hda AC97)],
 296         description =>  "Configure an audio device."
 297     },
 298     driver =>  {
 299         type => 'string',
 300         enum => ['spice'],
 301         default => 'spice',
 302         optional => 1,
 303         description => "Driver backend for the audio device."
 304     },
 305 };
 306
 307 my $spice_enhancements_fmt = {
 308     foldersharing => {
 309         type => 'boolean',
 310         optional => 1,
 311         default => '0',
 312         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 313     },
 314     videostreaming =>  {
 315         type => 'string',
 316         enum => ['off', 'all', 'filter'],
 317         default => 'off',
 318         optional => 1,
 319         description => "Enable video streaming. Uses compression for detected video streams."
 320     },
 321 };
 322
 323 my $confdesc = {
 324     onboot => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Specifies whether a VM will be started during system bootup.",
 328         default => 0,
 329     },
 330     autostart => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Automatic restart after crash (currently ignored).",
 334         default => 0,
 335     },
 336     hotplug => {
 337         optional => 1,
 338         type => 'string', format => 'pve-hotplug-features',
 339         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 340         default => 'network,disk,usb',
 341     },
 342     reboot => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 346         default => 1,
 347     },
 348     lock => {
 349         optional => 1,
 350         type => 'string',
 351         description => "Lock/unlock the VM.",
 352         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 353     },
 354     cpulimit => {
 355         optional => 1,
 356         type => 'number',
 357         description => "Limit of CPU usage.",
 358         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 359         minimum => 0,
 360         maximum => 128,
 361         default => 0,
 362     },
 363     cpuunits => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "CPU weight for a VM.",
 367         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 368         minimum => 2,
 369         maximum => 262144,
 370         default => 1024,
 371     },
 372     memory => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 376         minimum => 16,
 377         default => 512,
 378     },
 379     balloon => {
 380         optional => 1,
 381         type => 'integer',
 382         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 383         minimum => 0,
 384     },
 385     shares => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 389         minimum => 0,
 390         maximum => 50000,
 391         default => 1000,
 392     },
 393     keyboard => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 397                        "It should not be necessary to set it.",
 398         enum => PVE::Tools::kvmkeymaplist(),
 399         default => undef,
 400     },
 401     name => {
 402         optional => 1,
 403         type => 'string', format => 'dns-name',
 404         description => "Set a name for the VM. Only used on the configuration web interface.",
 405     },
 406     scsihw => {
 407         optional => 1,
 408         type => 'string',
 409         description => "SCSI controller model",
 410         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 411         default => 'lsi',
 412     },
 413     description => {
 414         optional => 1,
 415         type => 'string',
 416         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 417     },
 418     ostype => {
 419         optional => 1,
 420         type => 'string',
 421         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 422         description => "Specify guest operating system.",
 423         verbose_description => <<EODESC,
 424 Specify guest operating system. This is used to enable special
 425 optimization/features for specific operating systems:
 426
 427 [horizontal]
 428 other;; unspecified OS
 429 wxp;; Microsoft Windows XP
 430 w2k;; Microsoft Windows 2000
 431 w2k3;; Microsoft Windows 2003
 432 w2k8;; Microsoft Windows 2008
 433 wvista;; Microsoft Windows Vista
 434 win7;; Microsoft Windows 7
 435 win8;; Microsoft Windows 8/2012/2012r2
 436 win10;; Microsoft Windows 10/2016
 437 l24;; Linux 2.4 Kernel
 438 l26;; Linux 2.6 - 5.X Kernel
 439 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 440 EODESC
 441     },
 442     boot => {
 443         optional => 1,
 444         type => 'string',
 445         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 446         pattern => '[acdn]{1,4}',
 447         default => 'cdn',
 448     },
 449     bootdisk => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-bootdisk',
 452         description => "Enable booting from specified disk.",
 453         pattern => '(ide|sata|scsi|virtio)\d+',
 454     },
 455     smp => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPUs. Please use option -sockets instead.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     sockets => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of CPU sockets.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     cores => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "The number of cores per socket.",
 473         minimum => 1,
 474         default => 1,
 475     },
 476     numa => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable NUMA.",
 480         default => 0,
 481     },
 482     hugepages => {
 483         optional => 1,
 484         type => 'string',
 485         description => "Enable/disable hugepages memory.",
 486         enum => [qw(any 2 1024)],
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable Qemu GuestAgent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 523     },
 524     freeze => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 528     },
 529     vga => {
 530         optional => 1,
 531         type => 'string', format => $vga_fmt,
 532         description => "Configure the VGA hardware.",
 533         verbose_description => "Configure the VGA Hardware. If you want to use ".
 534             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 535             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 536             "is 'std' for all OS types besides some Windows versions (XP and " .
 537             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 538             "display server. For win* OS you can select how many independent " .
 539             "displays you want, Linux guests can add displays them self.\n".
 540             "You can also run without any graphic card, using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 547             " (by a guest action), the watchdog must be periodically polled " .
 548             "by an agent inside the guest or else the watchdog will reset " .
 549             "the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -no-hpet
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is " .
 584             "usually needed to allow absolute mouse positioning with VNC. " .
 585             "Else the mouse runs out of sync with normal VNC clients. " .
 586             "If you're running lots of console-only guests on one host, " .
 587             "you may consider disabling this to save some context switches. " .
 588             "This is turned off by default if you use spice (-vga=qxl).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => $cpu_fmt,
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 637     }),
 638     machine => get_standard_option('pve-qemu-machine'),
 639     arch => {
 640         description => "Virtual processor architecture. Defaults to the host.",
 641         optional => 1,
 642         type => 'string',
 643         enum => [qw(x86_64 aarch64)],
 644     },
 645     smbios1 => {
 646         description => "Specify SMBIOS type 1 fields.",
 647         type => 'string', format => 'pve-qm-smbios1',
 648         maxLength => 512,
 649         optional => 1,
 650     },
 651     protection => {
 652         optional => 1,
 653         type => 'boolean',
 654         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 669         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 670             " 128-bit integer value identifier to the guest OS. This allows to".
 671             " notify the guest operating system when the virtual machine is".
 672             " executed with a different configuration (e.g. snapshot execution".
 673             " or creation from a template). The guest operating system notices".
 674             " the change, and is then able to react as appropriate by marking".
 675             " its copies of distributed databases as dirty, re-initializing its".
 676             " random number generator, etc.\n".
 677             "Note that auto-creation only works when done throug API/CLI create".
 678             " or update methods, but not when manually editing the config file.",
 679         default => "1 (autogenerated)",
 680         optional => 1,
 681     },
 682     hookscript => {
 683         type => 'string',
 684         format => 'pve-volume-id',
 685         optional => 1,
 686         description => "Script that will be executed during various steps in the vms lifetime.",
 687     },
 688     ivshmem => {
 689         type => 'string',
 690         format => $ivshmem_fmt,
 691         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711 };
 712
 713 my $cicustom_fmt = {
 714     meta => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728     user => {
 729         type => 'string',
 730         optional => 1,
 731         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735 };
 736 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 737
 738 my $confdesc_cloudinit = {
 739     citype => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 743         enum => ['configdrive2', 'nocloud'],
 744     },
 745     ciuser => {
 746         optional => 1,
 747         type => 'string',
 748         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 749     },
 750     cipassword => {
 751         optional => 1,
 752         type => 'string',
 753         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 754     },
 755     cicustom => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 759         format => 'pve-qm-cicustom',
 760     },
 761     searchdomain => {
 762         optional => 1,
 763         type => 'string',
 764         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 765     },
 766     nameserver => {
 767         optional => 1,
 768         type => 'string', format => 'address-list',
 769         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 770     },
 771     sshkeys => {
 772         optional => 1,
 773         type => 'string',
 774         format => 'urlencoded',
 775         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 776     },
 777 };
 778
 779 # what about other qemu settings ?
 780 #cpu => 'string',
 781 #machine => 'string',
 782 #fda => 'file',
 783 #fdb => 'file',
 784 #mtdblock => 'file',
 785 #sd => 'file',
 786 #pflash => 'file',
 787 #snapshot => 'bool',
 788 #bootp => 'file',
 789 ##tftp => 'dir',
 790 ##smb => 'dir',
 791 #kernel => 'file',
 792 #append => 'string',
 793 #initrd => 'file',
 794 ##soundhw => 'string',
 795
 796 while (my ($k, $v) = each %$confdesc) {
 797     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 798 }
 799
 800 my $MAX_IDE_DISKS = 4;
 801 my $MAX_SCSI_DISKS = 14;
 802 my $MAX_VIRTIO_DISKS = 16;
 803 my $MAX_SATA_DISKS = 6;
 804 my $MAX_USB_DEVICES = 5;
 805 my $MAX_NETS = 32;
 806 my $MAX_UNUSED_DISKS = 256;
 807 my $MAX_HOSTPCI_DEVICES = 16;
 808 my $MAX_SERIAL_PORTS = 4;
 809 my $MAX_PARALLEL_PORTS = 3;
 810 my $MAX_NUMA = 8;
 811
 812 my $numa_fmt = {
 813     cpus => {
 814         type => "string",
 815         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 816         description => "CPUs accessing this NUMA node.",
 817         format_description => "id[-id];...",
 818     },
 819     memory => {
 820         type => "number",
 821         description => "Amount of memory this NUMA node provides.",
 822         optional => 1,
 823     },
 824     hostnodes => {
 825         type => "string",
 826         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 827         description => "Host NUMA nodes to use.",
 828         format_description => "id[-id];...",
 829         optional => 1,
 830     },
 831     policy => {
 832         type => 'string',
 833         enum => [qw(preferred bind interleave)],
 834         description => "NUMA allocation policy.",
 835         optional => 1,
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 839 my $numadesc = {
 840     optional => 1,
 841     type => 'string', format => $numa_fmt,
 842     description => "NUMA topology.",
 843 };
 844 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 845
 846 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 847     $confdesc->{"numa$i"} = $numadesc;
 848 }
 849
 850 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 851                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 852                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 853 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 854
 855 my $net_fmt_bridge_descr = <<__EOD__;
 856 Bridge to attach the network device to. The Proxmox VE standard bridge
 857 is called 'vmbr0'.
 858
 859 If you do not specify a bridge, we create a kvm user (NATed) network
 860 device, which provides DHCP and DNS services. The following addresses
 861 are used:
 862
 863  10.0.2.2   Gateway
 864  10.0.2.3   DNS Server
 865  10.0.2.4   SMB Server
 866
 867 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 868 __EOD__
 869
 870 my $net_fmt = {
 871     macaddr  => get_standard_option('mac-addr', {
 872         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 873     }),
 874     model => {
 875         type => 'string',
 876         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 877         enum => $nic_model_list,
 878         default_key => 1,
 879     },
 880     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 881     bridge => {
 882         type => 'string',
 883         description => $net_fmt_bridge_descr,
 884         format_description => 'bridge',
 885         optional => 1,
 886     },
 887     queues => {
 888         type => 'integer',
 889         minimum => 0, maximum => 16,
 890         description => 'Number of packet queues to be used on the device.',
 891         optional => 1,
 892     },
 893     rate => {
 894         type => 'number',
 895         minimum => 0,
 896         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 897         optional => 1,
 898     },
 899     tag => {
 900         type => 'integer',
 901         minimum => 1, maximum => 4094,
 902         description => 'VLAN tag to apply to packets on this interface.',
 903         optional => 1,
 904     },
 905     trunks => {
 906         type => 'string',
 907         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 908         description => 'VLAN trunks to pass through this interface.',
 909         format_description => 'vlanid[;vlanid...]',
 910         optional => 1,
 911     },
 912     firewall => {
 913         type => 'boolean',
 914         description => 'Whether this interface should be protected by the firewall.',
 915         optional => 1,
 916     },
 917     link_down => {
 918         type => 'boolean',
 919         description => 'Whether this interface should be disconnected (like pulling the plug).',
 920         optional => 1,
 921     },
 922 };
 923
 924 my $netdesc = {
 925     optional => 1,
 926     type => 'string', format => $net_fmt,
 927     description => "Specify network devices.",
 928 };
 929
 930 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 931
 932 my $ipconfig_fmt = {
 933     ip => {
 934         type => 'string',
 935         format => 'pve-ipv4-config',
 936         format_description => 'IPv4Format/CIDR',
 937         description => 'IPv4 address in CIDR format.',
 938         optional => 1,
 939         default => 'dhcp',
 940     },
 941     gw => {
 942         type => 'string',
 943         format => 'ipv4',
 944         format_description => 'GatewayIPv4',
 945         description => 'Default gateway for IPv4 traffic.',
 946         optional => 1,
 947         requires => 'ip',
 948     },
 949     ip6 => {
 950         type => 'string',
 951         format => 'pve-ipv6-config',
 952         format_description => 'IPv6Format/CIDR',
 953         description => 'IPv6 address in CIDR format.',
 954         optional => 1,
 955         default => 'dhcp',
 956     },
 957     gw6 => {
 958         type => 'string',
 959         format => 'ipv6',
 960         format_description => 'GatewayIPv6',
 961         description => 'Default gateway for IPv6 traffic.',
 962         optional => 1,
 963         requires => 'ip6',
 964     },
 965 };
 966 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 967 my $ipconfigdesc = {
 968     optional => 1,
 969     type => 'string', format => 'pve-qm-ipconfig',
 970     description => <<'EODESCR',
 971 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 972
 973 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 974
 975 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 976 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 977
 978 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 979 EODESCR
 980 };
 981 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 982
 983 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 984     $confdesc->{"net$i"} = $netdesc;
 985     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 986 }
 987
 988 foreach my $key (keys %$confdesc_cloudinit) {
 989     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 990 }
 991
 992 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 993 sub verify_volume_id_or_qm_path {
 994     my ($volid, $noerr) = @_;
 995
 996     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 997         return $volid;
 998     }
 999
1000     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1001     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1002     if ($@) {
1003         return undef if $noerr;
1004         die $@;
1005     }
1006     return $volid;
1007 }
1008
1009 my $drivename_hash;
1010
1011 my %drivedesc_base = (
1012     volume => { alias => 'file' },
1013     file => {
1014         type => 'string',
1015         format => 'pve-volume-id-or-qm-path',
1016         default_key => 1,
1017         format_description => 'volume',
1018         description => "The drive's backing volume.",
1019     },
1020     media => {
1021         type => 'string',
1022         enum => [qw(cdrom disk)],
1023         description => "The drive's media type.",
1024         default => 'disk',
1025         optional => 1
1026     },
1027     cyls => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific cylinder count.",
1030         optional => 1
1031     },
1032     heads => {
1033         type => 'integer',
1034         description => "Force the drive's physical geometry to have a specific head count.",
1035         optional => 1
1036     },
1037     secs => {
1038         type => 'integer',
1039         description => "Force the drive's physical geometry to have a specific sector count.",
1040         optional => 1
1041     },
1042     trans => {
1043         type => 'string',
1044         enum => [qw(none lba auto)],
1045         description => "Force disk geometry bios translation mode.",
1046         optional => 1,
1047     },
1048     snapshot => {
1049         type => 'boolean',
1050         description => "Controls qemu's snapshot mode feature."
1051             . " If activated, changes made to the disk are temporary and will"
1052             . " be discarded when the VM is shutdown.",
1053         optional => 1,
1054     },
1055     cache => {
1056         type => 'string',
1057         enum => [qw(none writethrough writeback unsafe directsync)],
1058         description => "The drive's cache mode",
1059         optional => 1,
1060     },
1061     format => get_standard_option('pve-qm-image-format'),
1062     size => {
1063         type => 'string',
1064         format => 'disk-size',
1065         format_description => 'DiskSize',
1066         description => "Disk size. This is purely informational and has no effect.",
1067         optional => 1,
1068     },
1069     backup => {
1070         type => 'boolean',
1071         description => "Whether the drive should be included when making backups.",
1072         optional => 1,
1073     },
1074     replicate => {
1075         type => 'boolean',
1076         description => 'Whether the drive should considered for replication jobs.',
1077         optional => 1,
1078         default => 1,
1079     },
1080     rerror => {
1081         type => 'string',
1082         enum => [qw(ignore report stop)],
1083         description => 'Read error action.',
1084         optional => 1,
1085     },
1086     werror => {
1087         type => 'string',
1088         enum => [qw(enospc ignore report stop)],
1089         description => 'Write error action.',
1090         optional => 1,
1091     },
1092     aio => {
1093         type => 'string',
1094         enum => [qw(native threads)],
1095         description => 'AIO type to use.',
1096         optional => 1,
1097     },
1098     discard => {
1099         type => 'string',
1100         enum => [qw(ignore on)],
1101         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1102         optional => 1,
1103     },
1104     detect_zeroes => {
1105         type => 'boolean',
1106         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1107         optional => 1,
1108     },
1109     serial => {
1110         type => 'string',
1111         format => 'urlencoded',
1112         format_description => 'serial',
1113         maxLength => 20*3, # *3 since it's %xx url enoded
1114         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1115         optional => 1,
1116     },
1117     shared => {
1118         type => 'boolean',
1119         description => 'Mark this locally-managed volume as available on all nodes',
1120         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1121         optional => 1,
1122         default => 0,
1123     }
1124 );
1125
1126 my %iothread_fmt = ( iothread => {
1127         type => 'boolean',
1128         description => "Whether to use iothreads for this drive",
1129         optional => 1,
1130 });
1131
1132 my %model_fmt = (
1133     model => {
1134         type => 'string',
1135         format => 'urlencoded',
1136         format_description => 'model',
1137         maxLength => 40*3, # *3 since it's %xx url enoded
1138         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1139         optional => 1,
1140     },
1141 );
1142
1143 my %queues_fmt = (
1144     queues => {
1145         type => 'integer',
1146         description => "Number of queues.",
1147         minimum => 2,
1148         optional => 1
1149     }
1150 );
1151
1152 my %scsiblock_fmt = (
1153     scsiblock => {
1154         type => 'boolean',
1155         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1156         optional => 1,
1157         default => 0,
1158     },
1159 );
1160
1161 my %ssd_fmt = (
1162     ssd => {
1163         type => 'boolean',
1164         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my %wwn_fmt = (
1170     wwn => {
1171         type => 'string',
1172         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1173         format_description => 'wwn',
1174         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1175         optional => 1,
1176     },
1177 );
1178
1179 my $add_throttle_desc = sub {
1180     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1181     my $d = {
1182         type => $type,
1183         format_description => $unit,
1184         description => "Maximum $what in $longunit.",
1185         optional => 1,
1186     };
1187     $d->{minimum} = $minimum if defined($minimum);
1188     $drivedesc_base{$key} = $d;
1189 };
1190 # throughput: (leaky bucket)
1191 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1192 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1193 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1194 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1200
1201 # pools: (pool of IO before throttling starts taking effect)
1202 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1203 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1204 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1205 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1206 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1207 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1208
1209 # burst lengths
1210 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1211 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1212 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1213 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1214 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1215 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1216
1217 # legacy support
1218 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1219 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1220 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1221 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1222
1223 my $ide_fmt = {
1224     %drivedesc_base,
1225     %model_fmt,
1226     %ssd_fmt,
1227     %wwn_fmt,
1228 };
1229 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1230
1231 my $idedesc = {
1232     optional => 1,
1233     type => 'string', format => $ide_fmt,
1234     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1235 };
1236 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1237
1238 my $scsi_fmt = {
1239     %drivedesc_base,
1240     %iothread_fmt,
1241     %queues_fmt,
1242     %scsiblock_fmt,
1243     %ssd_fmt,
1244     %wwn_fmt,
1245 };
1246 my $scsidesc = {
1247     optional => 1,
1248     type => 'string', format => $scsi_fmt,
1249     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1250 };
1251 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1252
1253 my $sata_fmt = {
1254     %drivedesc_base,
1255     %ssd_fmt,
1256     %wwn_fmt,
1257 };
1258 my $satadesc = {
1259     optional => 1,
1260     type => 'string', format => $sata_fmt,
1261     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1262 };
1263 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1264
1265 my $virtio_fmt = {
1266     %drivedesc_base,
1267     %iothread_fmt,
1268 };
1269 my $virtiodesc = {
1270     optional => 1,
1271     type => 'string', format => $virtio_fmt,
1272     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1273 };
1274 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1275
1276 my $alldrive_fmt = {
1277     %drivedesc_base,
1278     %iothread_fmt,
1279     %model_fmt,
1280     %queues_fmt,
1281     %scsiblock_fmt,
1282     %ssd_fmt,
1283     %wwn_fmt,
1284 };
1285
1286 my $efidisk_fmt = {
1287     volume => { alias => 'file' },
1288     file => {
1289         type => 'string',
1290         format => 'pve-volume-id-or-qm-path',
1291         default_key => 1,
1292         format_description => 'volume',
1293         description => "The drive's backing volume.",
1294     },
1295     format => get_standard_option('pve-qm-image-format'),
1296     size => {
1297         type => 'string',
1298         format => 'disk-size',
1299         format_description => 'DiskSize',
1300         description => "Disk size. This is purely informational and has no effect.",
1301         optional => 1,
1302     },
1303 };
1304
1305 my $efidisk_desc = {
1306     optional => 1,
1307     type => 'string', format => $efidisk_fmt,
1308     description => "Configure a Disk for storing EFI vars",
1309 };
1310
1311 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1312
1313 my $usb_fmt = {
1314     host => {
1315         default_key => 1,
1316         type => 'string', format => 'pve-qm-usb-device',
1317         format_description => 'HOSTUSBDEVICE|spice',
1318         description => <<EODESCR,
1319 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1320
1321  'bus-port(.port)*' (decimal numbers) or
1322  'vendor_id:product_id' (hexadeciaml numbers) or
1323  'spice'
1324
1325 You can use the 'lsusb -t' command to list existing usb devices.
1326
1327 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1328
1329 The value 'spice' can be used to add a usb redirection devices for spice.
1330 EODESCR
1331     },
1332     usb3 => {
1333         optional => 1,
1334         type => 'boolean',
1335         description => "Specifies whether if given host option is a USB3 device or port.",
1336         default => 0,
1337     },
1338 };
1339
1340 my $usbdesc = {
1341     optional => 1,
1342     type => 'string', format => $usb_fmt,
1343     description => "Configure an USB device (n is 0 to 4).",
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1346
1347 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1348 my $hostpci_fmt = {
1349     host => {
1350         default_key => 1,
1351         type => 'string',
1352         pattern => qr/$PCIRE(;$PCIRE)*/,
1353         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1354         description => <<EODESCR,
1355 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1356 of PCI virtual functions of the host. HOSTPCIID syntax is:
1357
1358 'bus:dev.func' (hexadecimal numbers)
1359
1360 You can us the 'lspci' command to list existing PCI devices.
1361 EODESCR
1362     },
1363     rombar => {
1364         type => 'boolean',
1365         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1366         optional => 1,
1367         default => 1,
1368     },
1369     romfile => {
1370         type => 'string',
1371         pattern => '[^,;]+',
1372         format_description => 'string',
1373         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1374         optional => 1,
1375     },
1376     pcie => {
1377         type => 'boolean',
1378         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1379         optional => 1,
1380         default => 0,
1381     },
1382     'x-vga' => {
1383         type => 'boolean',
1384         description =>  "Enable vfio-vga device support.",
1385         optional => 1,
1386         default => 0,
1387     },
1388     'mdev' => {
1389         type => 'string',
1390         format_description => 'string',
1391         pattern => '[^/\.:]+',
1392         optional => 1,
1393         description => <<EODESCR
1394 The type of mediated device to use.
1395 An instance of this type will be created on startup of the VM and
1396 will be cleaned up when the VM stops.
1397 EODESCR
1398     }
1399 };
1400 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1401
1402 my $hostpcidesc = {
1403         optional => 1,
1404         type => 'string', format => 'pve-qm-hostpci',
1405         description => "Map host PCI devices into guest.",
1406         verbose_description =>  <<EODESCR,
1407 Map host PCI devices into guest.
1408
1409 NOTE: This option allows direct access to host hardware. So it is no longer
1410 possible to migrate such machines - use with special care.
1411
1412 CAUTION: Experimental! User reported problems with this option.
1413 EODESCR
1414 };
1415 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1416
1417 my $serialdesc = {
1418         optional => 1,
1419         type => 'string',
1420         pattern => '(/dev/.+|socket)',
1421         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1422         verbose_description =>  <<EODESCR,
1423 Create a serial device inside the VM (n is 0 to 3), and pass through a
1424 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1425 host side (use 'qm terminal' to open a terminal connection).
1426
1427 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1428
1429 CAUTION: Experimental! User reported problems with this option.
1430 EODESCR
1431 };
1432
1433 my $paralleldesc= {
1434         optional => 1,
1435         type => 'string',
1436         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1437         description =>  "Map host parallel devices (n is 0 to 2).",
1438         verbose_description =>  <<EODESCR,
1439 Map host parallel devices (n is 0 to 2).
1440
1441 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1442
1443 CAUTION: Experimental! User reported problems with this option.
1444 EODESCR
1445 };
1446
1447 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1448     $confdesc->{"parallel$i"} = $paralleldesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1452     $confdesc->{"serial$i"} = $serialdesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1456     $confdesc->{"hostpci$i"} = $hostpcidesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1460     $drivename_hash->{"ide$i"} = 1;
1461     $confdesc->{"ide$i"} = $idedesc;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1465     $drivename_hash->{"sata$i"} = 1;
1466     $confdesc->{"sata$i"} = $satadesc;
1467 }
1468
1469 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1470     $drivename_hash->{"scsi$i"} = 1;
1471     $confdesc->{"scsi$i"} = $scsidesc ;
1472 }
1473
1474 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1475     $drivename_hash->{"virtio$i"} = 1;
1476     $confdesc->{"virtio$i"} = $virtiodesc;
1477 }
1478
1479 $drivename_hash->{efidisk0} = 1;
1480 $confdesc->{efidisk0} = $efidisk_desc;
1481
1482 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1483     $confdesc->{"usb$i"} = $usbdesc;
1484 }
1485
1486 my $unuseddesc = {
1487     optional => 1,
1488     type => 'string', format => 'pve-volume-id',
1489     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1490 };
1491
1492 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1493     $confdesc->{"unused$i"} = $unuseddesc;
1494 }
1495
1496 my $kvm_api_version = 0;
1497
1498 sub kvm_version {
1499     return $kvm_api_version if $kvm_api_version;
1500
1501     open my $fh, '<', '/dev/kvm'
1502         or return undef;
1503
1504     # 0xae00 => KVM_GET_API_VERSION
1505     $kvm_api_version = ioctl($fh, 0xae00, 0);
1506
1507     return $kvm_api_version;
1508 }
1509
1510 my $kvm_user_version = {};
1511 my $kvm_mtime = {};
1512
1513 sub kvm_user_version {
1514     my ($binary) = @_;
1515
1516     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1517     my $st = stat($binary);
1518
1519     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1520     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1521         $cachedmtime == $st->mtime;
1522
1523     $kvm_user_version->{$binary} = 'unknown';
1524     $kvm_mtime->{$binary} = $st->mtime;
1525
1526     my $code = sub {
1527         my $line = shift;
1528         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1529             $kvm_user_version->{$binary} = $2;
1530         }
1531     };
1532
1533     eval { run_command([$binary, '--version'], outfunc => $code); };
1534     warn $@ if $@;
1535
1536     return $kvm_user_version->{$binary};
1537
1538 }
1539
1540 sub kernel_has_vhost_net {
1541     return -c '/dev/vhost-net';
1542 }
1543
1544 sub valid_drive_names {
1545     # order is important - used to autoselect boot disk
1546     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1547             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1548             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1549             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1550             'efidisk0');
1551 }
1552
1553 sub is_valid_drivename {
1554     my $dev = shift;
1555
1556     return defined($drivename_hash->{$dev});
1557 }
1558
1559 sub option_exists {
1560     my $key = shift;
1561     return defined($confdesc->{$key});
1562 }
1563
1564 my $cdrom_path;
1565 sub get_cdrom_path {
1566
1567     return  $cdrom_path if $cdrom_path;
1568
1569     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1570     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1571     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1572 }
1573
1574 sub get_iso_path {
1575     my ($storecfg, $vmid, $cdrom) = @_;
1576
1577     if ($cdrom eq 'cdrom') {
1578         return get_cdrom_path();
1579     } elsif ($cdrom eq 'none') {
1580         return '';
1581     } elsif ($cdrom =~ m|^/|) {
1582         return $cdrom;
1583     } else {
1584         return PVE::Storage::path($storecfg, $cdrom);
1585     }
1586 }
1587
1588 # try to convert old style file names to volume IDs
1589 sub filename_to_volume_id {
1590     my ($vmid, $file, $media) = @_;
1591
1592      if (!($file eq 'none' || $file eq 'cdrom' ||
1593           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1594
1595         return undef if $file =~ m|/|;
1596
1597         if ($media && $media eq 'cdrom') {
1598             $file = "local:iso/$file";
1599         } else {
1600             $file = "local:$vmid/$file";
1601         }
1602     }
1603
1604     return $file;
1605 }
1606
1607 sub verify_media_type {
1608     my ($opt, $vtype, $media) = @_;
1609
1610     return if !$media;
1611
1612     my $etype;
1613     if ($media eq 'disk') {
1614         $etype = 'images';
1615     } elsif ($media eq 'cdrom') {
1616         $etype = 'iso';
1617     } else {
1618         die "internal error";
1619     }
1620
1621     return if ($vtype eq $etype);
1622
1623     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1624 }
1625
1626 sub cleanup_drive_path {
1627     my ($opt, $storecfg, $drive) = @_;
1628
1629     # try to convert filesystem paths to volume IDs
1630
1631     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1632         ($drive->{file} !~ m|^/dev/.+|) &&
1633         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1634         ($drive->{file} !~ m/^\d+$/)) {
1635         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1636         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1637         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1638         verify_media_type($opt, $vtype, $drive->{media});
1639         $drive->{file} = $volid;
1640     }
1641
1642     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1643 }
1644
1645 sub parse_hotplug_features {
1646     my ($data) = @_;
1647
1648     my $res = {};
1649
1650     return $res if $data eq '0';
1651
1652     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1653
1654     foreach my $feature (PVE::Tools::split_list($data)) {
1655         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1656             $res->{$1} = 1;
1657         } else {
1658             die "invalid hotplug feature '$feature'\n";
1659         }
1660     }
1661     return $res;
1662 }
1663
1664 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1665 sub pve_verify_hotplug_features {
1666     my ($value, $noerr) = @_;
1667
1668     return $value if parse_hotplug_features($value);
1669
1670     return undef if $noerr;
1671
1672     die "unable to parse hotplug option\n";
1673 }
1674
1675 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1676 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1677 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1678 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1679 #        [,iothread=on][,serial=serial][,model=model]
1680
1681 sub parse_drive {
1682     my ($key, $data) = @_;
1683
1684     my ($interface, $index);
1685
1686     if ($key =~ m/^([^\d]+)(\d+)$/) {
1687         $interface = $1;
1688         $index = $2;
1689     } else {
1690         return undef;
1691     }
1692
1693     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1694                                      : $confdesc->{$key}->{format};
1695     if (!$desc) {
1696         warn "invalid drive key: $key\n";
1697         return undef;
1698     }
1699     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1700     return undef if !$res;
1701     $res->{interface} = $interface;
1702     $res->{index} = $index;
1703
1704     my $error = 0;
1705     foreach my $opt (qw(bps bps_rd bps_wr)) {
1706         if (my $bps = defined(delete $res->{$opt})) {
1707             if (defined($res->{"m$opt"})) {
1708                 warn "both $opt and m$opt specified\n";
1709                 ++$error;
1710                 next;
1711             }
1712             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1713         }
1714     }
1715
1716     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1717     for my $requirement (
1718         [mbps_max => 'mbps'],
1719         [mbps_rd_max => 'mbps_rd'],
1720         [mbps_wr_max => 'mbps_wr'],
1721         [miops_max => 'miops'],
1722         [miops_rd_max => 'miops_rd'],
1723         [miops_wr_max => 'miops_wr'],
1724         [bps_max_length => 'mbps_max'],
1725         [bps_rd_max_length => 'mbps_rd_max'],
1726         [bps_wr_max_length => 'mbps_wr_max'],
1727         [iops_max_length => 'iops_max'],
1728         [iops_rd_max_length => 'iops_rd_max'],
1729         [iops_wr_max_length => 'iops_wr_max']) {
1730         my ($option, $requires) = @$requirement;
1731         if ($res->{$option} && !$res->{$requires}) {
1732             warn "$option requires $requires\n";
1733             ++$error;
1734         }
1735     }
1736
1737     return undef if $error;
1738
1739     return undef if $res->{mbps_rd} && $res->{mbps};
1740     return undef if $res->{mbps_wr} && $res->{mbps};
1741     return undef if $res->{iops_rd} && $res->{iops};
1742     return undef if $res->{iops_wr} && $res->{iops};
1743
1744     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1745         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1746         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1747         return undef if $res->{interface} eq 'virtio';
1748     }
1749
1750     if (my $size = $res->{size}) {
1751         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1752     }
1753
1754     return $res;
1755 }
1756
1757 sub print_drive {
1758     my ($drive) = @_;
1759     my $data = { %$drive };
1760     delete $data->{$_} for qw(index interface);
1761     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1762 }
1763
1764 sub scsi_inquiry {
1765     my($fh, $noerr) = @_;
1766
1767     my $SG_IO = 0x2285;
1768     my $SG_GET_VERSION_NUM = 0x2282;
1769
1770     my $versionbuf = "\x00" x 8;
1771     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1772     if (!$ret) {
1773         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1774         return undef;
1775     }
1776     my $version = unpack("I", $versionbuf);
1777     if ($version < 30000) {
1778         die "scsi generic interface too old\n"  if !$noerr;
1779         return undef;
1780     }
1781
1782     my $buf = "\x00" x 36;
1783     my $sensebuf = "\x00" x 8;
1784     my $cmd = pack("C x3 C x1", 0x12, 36);
1785
1786     # see /usr/include/scsi/sg.h
1787     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1788
1789     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1790                       length($sensebuf), 0, length($buf), $buf,
1791                       $cmd, $sensebuf, 6000);
1792
1793     $ret = ioctl($fh, $SG_IO, $packet);
1794     if (!$ret) {
1795         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1796         return undef;
1797     }
1798
1799     my @res = unpack($sg_io_hdr_t, $packet);
1800     if ($res[17] || $res[18]) {
1801         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1802         return undef;
1803     }
1804
1805     my $res = {};
1806     (my $byte0, my $byte1, $res->{vendor},
1807      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1808
1809     $res->{removable} = $byte1 & 128 ? 1 : 0;
1810     $res->{type} = $byte0 & 31;
1811
1812     return $res;
1813 }
1814
1815 sub path_is_scsi {
1816     my ($path) = @_;
1817
1818     my $fh = IO::File->new("+<$path") || return undef;
1819     my $res = scsi_inquiry($fh, 1);
1820     close($fh);
1821
1822     return $res;
1823 }
1824
1825 sub print_tabletdevice_full {
1826     my ($conf, $arch) = @_;
1827
1828     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1829
1830     # we use uhci for old VMs because tablet driver was buggy in older qemu
1831     my $usbbus;
1832     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1833         $usbbus = 'ehci';
1834     } else {
1835         $usbbus = 'uhci';
1836     }
1837
1838     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1839 }
1840
1841 sub print_keyboarddevice_full {
1842     my ($conf, $arch, $machine) = @_;
1843
1844     return undef if $arch ne 'aarch64';
1845
1846     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1847 }
1848
1849 sub print_drivedevice_full {
1850     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1851
1852     my $device = '';
1853     my $maxdev = 0;
1854
1855     if ($drive->{interface} eq 'virtio') {
1856         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1857         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1858         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1859     } elsif ($drive->{interface} eq 'scsi') {
1860
1861         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1862         my $unit = $drive->{index} % $maxdev;
1863         my $devicetype = 'hd';
1864         my $path = '';
1865         if (drive_is_cdrom($drive)) {
1866             $devicetype = 'cd';
1867         } else {
1868             if ($drive->{file} =~ m|^/|) {
1869                 $path = $drive->{file};
1870                 if (my $info = path_is_scsi($path)) {
1871                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1872                         $devicetype = 'block';
1873                     } elsif ($info->{type} == 1) { # tape
1874                         $devicetype = 'generic';
1875                     }
1876                 }
1877             } else {
1878                  $path = PVE::Storage::path($storecfg, $drive->{file});
1879             }
1880
1881             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1882             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1883             if ($path =~ m/^iscsi\:\/\// &&
1884                !min_version($version, 4, 1)) {
1885                 $devicetype = 'generic';
1886             }
1887         }
1888
1889         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1890             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1891         } else {
1892             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1893         }
1894
1895         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1896             $device .= ",rotation_rate=1";
1897         }
1898         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1899
1900     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1901         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1902         my $controller = int($drive->{index} / $maxdev);
1903         my $unit = $drive->{index} % $maxdev;
1904         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1905
1906         $device = "ide-$devicetype";
1907         if ($drive->{interface} eq 'ide') {
1908             $device .= ",bus=ide.$controller,unit=$unit";
1909         } else {
1910             $device .= ",bus=ahci$controller.$unit";
1911         }
1912         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1913
1914         if ($devicetype eq 'hd') {
1915             if (my $model = $drive->{model}) {
1916                 $model = URI::Escape::uri_unescape($model);
1917                 $device .= ",model=$model";
1918             }
1919             if ($drive->{ssd}) {
1920                 $device .= ",rotation_rate=1";
1921             }
1922         }
1923         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1924     } elsif ($drive->{interface} eq 'usb') {
1925         die "implement me";
1926         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1927     } else {
1928         die "unsupported interface type";
1929     }
1930
1931     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1932
1933     if (my $serial = $drive->{serial}) {
1934         $serial = URI::Escape::uri_unescape($serial);
1935         $device .= ",serial=$serial";
1936     }
1937
1938
1939     return $device;
1940 }
1941
1942 sub get_initiator_name {
1943     my $initiator;
1944
1945     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1946     while (defined(my $line = <$fh>)) {
1947         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1948         $initiator = $1;
1949         last;
1950     }
1951     $fh->close();
1952
1953     return $initiator;
1954 }
1955
1956 sub print_drive_full {
1957     my ($storecfg, $vmid, $drive) = @_;
1958
1959     my $path;
1960     my $volid = $drive->{file};
1961     my $format;
1962
1963     if (drive_is_cdrom($drive)) {
1964         $path = get_iso_path($storecfg, $vmid, $volid);
1965     } else {
1966         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1967         if ($storeid) {
1968             $path = PVE::Storage::path($storecfg, $volid);
1969             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1970             $format = qemu_img_format($scfg, $volname);
1971         } else {
1972             $path = $volid;
1973             $format = "raw";
1974         }
1975    }
1976
1977     my $opts = '';
1978     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1979     foreach my $o (@qemu_drive_options) {
1980         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1981     }
1982
1983     # snapshot only accepts on|off
1984     if (defined($drive->{snapshot})) {
1985         my $v = $drive->{snapshot} ? 'on' : 'off';
1986         $opts .= ",snapshot=$v";
1987     }
1988
1989     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1990         my ($dir, $qmpname) = @$type;
1991         if (my $v = $drive->{"mbps$dir"}) {
1992             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1993         }
1994         if (my $v = $drive->{"mbps${dir}_max"}) {
1995             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1996         }
1997         if (my $v = $drive->{"bps${dir}_max_length"}) {
1998             $opts .= ",throttling.bps$qmpname-max-length=$v";
1999         }
2000         if (my $v = $drive->{"iops${dir}"}) {
2001             $opts .= ",throttling.iops$qmpname=$v";
2002         }
2003         if (my $v = $drive->{"iops${dir}_max"}) {
2004             $opts .= ",throttling.iops$qmpname-max=$v";
2005         }
2006         if (my $v = $drive->{"iops${dir}_max_length"}) {
2007             $opts .= ",throttling.iops$qmpname-max-length=$v";
2008         }
2009     }
2010
2011     $opts .= ",format=$format" if $format && !$drive->{format};
2012
2013     my $cache_direct = 0;
2014
2015     if (my $cache = $drive->{cache}) {
2016         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2017     } elsif (!drive_is_cdrom($drive)) {
2018         $opts .= ",cache=none";
2019         $cache_direct = 1;
2020     }
2021
2022     # aio native works only with O_DIRECT
2023     if (!$drive->{aio}) {
2024         if($cache_direct) {
2025             $opts .= ",aio=native";
2026         } else {
2027             $opts .= ",aio=threads";
2028         }
2029     }
2030
2031     if (!drive_is_cdrom($drive)) {
2032         my $detectzeroes;
2033         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2034             $detectzeroes = 'off';
2035         } elsif ($drive->{discard}) {
2036             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2037         } else {
2038             # This used to be our default with discard not being specified:
2039             $detectzeroes = 'on';
2040         }
2041         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2042     }
2043
2044     my $pathinfo = $path ? "file=$path," : '';
2045
2046     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2047 }
2048
2049 sub print_netdevice_full {
2050     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2051
2052     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2053
2054     my $device = $net->{model};
2055     if ($net->{model} eq 'virtio') {
2056          $device = 'virtio-net-pci';
2057      };
2058
2059     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2060     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2061     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2062         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2063         my $vectors = $net->{queues} * 2 + 2;
2064         $tmpstr .= ",vectors=$vectors,mq=on";
2065     }
2066     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2067
2068     if ($use_old_bios_files) {
2069         my $romfile;
2070         if ($device eq 'virtio-net-pci') {
2071             $romfile = 'pxe-virtio.rom';
2072         } elsif ($device eq 'e1000') {
2073             $romfile = 'pxe-e1000.rom';
2074         } elsif ($device eq 'ne2k') {
2075             $romfile = 'pxe-ne2k_pci.rom';
2076         } elsif ($device eq 'pcnet') {
2077             $romfile = 'pxe-pcnet.rom';
2078         } elsif ($device eq 'rtl8139') {
2079             $romfile = 'pxe-rtl8139.rom';
2080         }
2081         $tmpstr .= ",romfile=$romfile" if $romfile;
2082     }
2083
2084     return $tmpstr;
2085 }
2086
2087 sub print_netdev_full {
2088     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2089
2090     my $i = '';
2091     if ($netid =~ m/^net(\d+)$/) {
2092         $i = int($1);
2093     }
2094
2095     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2096
2097     my $ifname = "tap${vmid}i$i";
2098
2099     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2100     die "interface name '$ifname' is too long (max 15 character)\n"
2101         if length($ifname) >= 16;
2102
2103     my $vhostparam = '';
2104     if (is_native($arch)) {
2105         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2106     }
2107
2108     my $vmname = $conf->{name} || "vm$vmid";
2109
2110     my $netdev = "";
2111     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2112
2113     if ($net->{bridge}) {
2114         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2115     } else {
2116         $netdev = "type=user,id=$netid,hostname=$vmname";
2117     }
2118
2119     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2120
2121     return $netdev;
2122 }
2123
2124
2125 sub print_cpu_device {
2126     my ($conf, $id) = @_;
2127
2128     my $kvm = $conf->{kvm} // 1;
2129     my $cpu = $kvm ? "kvm64" : "qemu64";
2130     if (my $cputype = $conf->{cpu}) {
2131         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2132             or die "Cannot parse cpu description: $cputype\n";
2133         $cpu = $cpuconf->{cputype};
2134     }
2135
2136     my $cores = $conf->{cores} || 1;
2137
2138     my $current_core = ($id - 1) % $cores;
2139     my $current_socket = int(($id - 1 - $current_core)/$cores);
2140
2141     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2142 }
2143
2144 my $vga_map = {
2145     'cirrus' => 'cirrus-vga',
2146     'std' => 'VGA',
2147     'vmware' => 'vmware-svga',
2148     'virtio' => 'virtio-vga',
2149 };
2150
2151 sub print_vga_device {
2152     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2153
2154     my $type = $vga_map->{$vga->{type}};
2155     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2156         $type = 'virtio-gpu';
2157     }
2158     my $vgamem_mb = $vga->{memory};
2159
2160     my $max_outputs = '';
2161     if ($qxlnum) {
2162         $type = $id ? 'qxl' : 'qxl-vga';
2163
2164         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2165             # set max outputs so linux can have up to 4 qxl displays with one device
2166             if (min_version($machine_version, 4, 1)) {
2167                 $max_outputs = ",max_outputs=4";
2168             }
2169         }
2170     }
2171
2172     die "no devicetype for $vga->{type}\n" if !$type;
2173
2174     my $memory = "";
2175     if ($vgamem_mb) {
2176         if ($vga->{type} eq 'virtio') {
2177             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2178             $memory = ",max_hostmem=$bytes";
2179         } elsif ($qxlnum) {
2180             # from https://www.spice-space.org/multiple-monitors.html
2181             $memory = ",vgamem_mb=$vga->{memory}";
2182             my $ram = $vgamem_mb * 4;
2183             my $vram = $vgamem_mb * 2;
2184             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2185         } else {
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187         }
2188     } elsif ($qxlnum && $id) {
2189         $memory = ",ram_size=67108864,vram_size=33554432";
2190     }
2191
2192     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2193     my $vgaid = "vga" . ($id // '');
2194     my $pciaddr;
2195
2196     if ($q35 && $vgaid eq 'vga') {
2197         # the first display uses pcie.0 bus on q35 machines
2198         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2199     } else {
2200         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2201     }
2202
2203     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2204 }
2205
2206 sub drive_is_cloudinit {
2207     my ($drive) = @_;
2208     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2209 }
2210
2211 sub drive_is_cdrom {
2212     my ($drive, $exclude_cloudinit) = @_;
2213
2214     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2215
2216     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2217
2218 }
2219
2220 sub parse_number_sets {
2221     my ($set) = @_;
2222     my $res = [];
2223     foreach my $part (split(/;/, $set)) {
2224         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2225             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2226             push @$res, [ $1, $2 ];
2227         } else {
2228             die "invalid range: $part\n";
2229         }
2230     }
2231     return $res;
2232 }
2233
2234 sub parse_numa {
2235     my ($data) = @_;
2236
2237     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2238     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2239     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2240     return $res;
2241 }
2242
2243 sub parse_hostpci {
2244     my ($value) = @_;
2245
2246     return undef if !$value;
2247
2248     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2249
2250     my @idlist = split(/;/, $res->{host});
2251     delete $res->{host};
2252     foreach my $id (@idlist) {
2253         my $devs = PVE::SysFSTools::lspci($id);
2254         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2255         push @{$res->{pciid}}, @$devs;
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812     my $nodename = nodename();
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my @loc_res = ();
2828
2829     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2830     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2831
2832     push @loc_res, "ivshmem" if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2842
2843     return \@loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_local_storage_availability {
2900     my ($conf, $storecfg) = @_;
2901
2902     my $nodelist = PVE::Cluster::get_nodelist();
2903     my $nodehash = { map { $_ => {} } @$nodelist };
2904
2905     foreach_drive($conf, sub {
2906         my ($ds, $drive) = @_;
2907
2908         my $volid = $drive->{file};
2909         return if !$volid;
2910
2911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2912         if ($storeid) {
2913             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2914
2915             if ($scfg->{disable}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                 }
2919             } elsif (my $avail = $scfg->{nodes}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     if (!$avail->{$node}) {
2922                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2923                     }
2924                 }
2925             }
2926         }
2927     });
2928
2929     foreach my $node (values %$nodehash) {
2930         if (my $unavail = $node->{unavailable_storages}) {
2931             $node->{unavailable_storages} = [ sort keys %$unavail ];
2932         }
2933     }
2934
2935     return $nodehash
2936 }
2937
2938 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2939 sub check_running {
2940     my ($vmid, $nocheck, $node) = @_;
2941
2942     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2943     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2944 }
2945
2946 sub vzlist {
2947
2948     my $vzlist = config_list();
2949
2950     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2951
2952     while (defined(my $de = $fd->read)) {
2953         next if $de !~ m/^(\d+)\.pid$/;
2954         my $vmid = $1;
2955         next if !defined($vzlist->{$vmid});
2956         if (my $pid = check_running($vmid)) {
2957             $vzlist->{$vmid}->{pid} = $pid;
2958         }
2959     }
2960
2961     return $vzlist;
2962 }
2963
2964 sub disksize {
2965     my ($storecfg, $conf) = @_;
2966
2967     my $bootdisk = $conf->{bootdisk};
2968     return undef if !$bootdisk;
2969     return undef if !is_valid_drivename($bootdisk);
2970
2971     return undef if !$conf->{$bootdisk};
2972
2973     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2974     return undef if !defined($drive);
2975
2976     return undef if drive_is_cdrom($drive);
2977
2978     my $volid = $drive->{file};
2979     return undef if !$volid;
2980
2981     return $drive->{size};
2982 }
2983
2984 our $vmstatus_return_properties = {
2985     vmid => get_standard_option('pve-vmid'),
2986     status => {
2987         description => "Qemu process status.",
2988         type => 'string',
2989         enum => ['stopped', 'running'],
2990     },
2991     maxmem => {
2992         description => "Maximum memory in bytes.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'bytes',
2996     },
2997     maxdisk => {
2998         description => "Root disk size in bytes.",
2999         type => 'integer',
3000         optional => 1,
3001         renderer => 'bytes',
3002     },
3003     name => {
3004         description => "VM name.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     qmpstatus => {
3009         description => "Qemu QMP agent status.",
3010         type => 'string',
3011         optional => 1,
3012     },
3013     pid => {
3014         description => "PID of running qemu process.",
3015         type => 'integer',
3016         optional => 1,
3017     },
3018     uptime => {
3019         description => "Uptime.",
3020         type => 'integer',
3021         optional => 1,
3022         renderer => 'duration',
3023     },
3024     cpus => {
3025         description => "Maximum usable CPUs.",
3026         type => 'number',
3027         optional => 1,
3028     },
3029     lock => {
3030         description => "The current config lock, if any.",
3031         type => 'string',
3032         optional => 1,
3033     },
3034     tags => {
3035         description  => "The current configured tags, if any",
3036         type => 'string',
3037         optional => 1,
3038     },
3039 };
3040
3041 my $last_proc_pid_stat;
3042
3043 # get VM status information
3044 # This must be fast and should not block ($full == false)
3045 # We only query KVM using QMP if $full == true (this can be slow)
3046 sub vmstatus {
3047     my ($opt_vmid, $full) = @_;
3048
3049     my $res = {};
3050
3051     my $storecfg = PVE::Storage::config();
3052
3053     my $list = vzlist();
3054     my $defaults = load_defaults();
3055
3056     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3057
3058     my $cpucount = $cpuinfo->{cpus} || 1;
3059
3060     foreach my $vmid (keys %$list) {
3061         next if $opt_vmid && ($vmid ne $opt_vmid);
3062
3063         my $conf = PVE::QemuConfig->load_config($vmid);
3064
3065         my $d = { vmid => $vmid };
3066         $d->{pid} = $list->{$vmid}->{pid};
3067
3068         # fixme: better status?
3069         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3070
3071         my $size = disksize($storecfg, $conf);
3072         if (defined($size)) {
3073             $d->{disk} = 0; # no info available
3074             $d->{maxdisk} = $size;
3075         } else {
3076             $d->{disk} = 0;
3077             $d->{maxdisk} = 0;
3078         }
3079
3080         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3081             * ($conf->{cores} || $defaults->{cores});
3082         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3083         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3084
3085         $d->{name} = $conf->{name} || "VM $vmid";
3086         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3087             : $defaults->{memory}*(1024*1024);
3088
3089         if ($conf->{balloon}) {
3090             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3091             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3092                 : $defaults->{shares};
3093         }
3094
3095         $d->{uptime} = 0;
3096         $d->{cpu} = 0;
3097         $d->{mem} = 0;
3098
3099         $d->{netout} = 0;
3100         $d->{netin} = 0;
3101
3102         $d->{diskread} = 0;
3103         $d->{diskwrite} = 0;
3104
3105         $d->{template} = PVE::QemuConfig->is_template($conf);
3106
3107         $d->{serial} = 1 if conf_has_serial($conf);
3108         $d->{lock} = $conf->{lock} if $conf->{lock};
3109         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3110
3111         $res->{$vmid} = $d;
3112     }
3113
3114     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3115     foreach my $dev (keys %$netdev) {
3116         next if $dev !~ m/^tap([1-9]\d*)i/;
3117         my $vmid = $1;
3118         my $d = $res->{$vmid};
3119         next if !$d;
3120
3121         $d->{netout} += $netdev->{$dev}->{receive};
3122         $d->{netin} += $netdev->{$dev}->{transmit};
3123
3124         if ($full) {
3125             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3126             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3127         }
3128
3129     }
3130
3131     my $ctime = gettimeofday;
3132
3133     foreach my $vmid (keys %$list) {
3134
3135         my $d = $res->{$vmid};
3136         my $pid = $d->{pid};
3137         next if !$pid;
3138
3139         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3140         next if !$pstat; # not running
3141
3142         my $used = $pstat->{utime} + $pstat->{stime};
3143
3144         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3145
3146         if ($pstat->{vsize}) {
3147             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3148         }
3149
3150         my $old = $last_proc_pid_stat->{$pid};
3151         if (!$old) {
3152             $last_proc_pid_stat->{$pid} = {
3153                 time => $ctime,
3154                 used => $used,
3155                 cpu => 0,
3156             };
3157             next;
3158         }
3159
3160         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3161
3162         if ($dtime > 1000) {
3163             my $dutime = $used -  $old->{used};
3164
3165             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3166             $last_proc_pid_stat->{$pid} = {
3167                 time => $ctime,
3168                 used => $used,
3169                 cpu => $d->{cpu},
3170             };
3171         } else {
3172             $d->{cpu} = $old->{cpu};
3173         }
3174     }
3175
3176     return $res if !$full;
3177
3178     my $qmpclient = PVE::QMPClient->new();
3179
3180     my $ballooncb = sub {
3181         my ($vmid, $resp) = @_;
3182
3183         my $info = $resp->{'return'};
3184         return if !$info->{max_mem};
3185
3186         my $d = $res->{$vmid};
3187
3188         # use memory assigned to VM
3189         $d->{maxmem} = $info->{max_mem};
3190         $d->{balloon} = $info->{actual};
3191
3192         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3193             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3194             $d->{freemem} = $info->{free_mem};
3195         }
3196
3197         $d->{ballooninfo} = $info;
3198     };
3199
3200     my $blockstatscb = sub {
3201         my ($vmid, $resp) = @_;
3202         my $data = $resp->{'return'} || [];
3203         my $totalrdbytes = 0;
3204         my $totalwrbytes = 0;
3205
3206         for my $blockstat (@$data) {
3207             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3208             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3209
3210             $blockstat->{device} =~ s/drive-//;
3211             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3212         }
3213         $res->{$vmid}->{diskread} = $totalrdbytes;
3214         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3215     };
3216
3217     my $statuscb = sub {
3218         my ($vmid, $resp) = @_;
3219
3220         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3221         # this fails if ballon driver is not loaded, so this must be
3222         # the last commnand (following command are aborted if this fails).
3223         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3224
3225         my $status = 'unknown';
3226         if (!defined($status = $resp->{'return'}->{status})) {
3227             warn "unable to get VM status\n";
3228             return;
3229         }
3230
3231         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3232     };
3233
3234     foreach my $vmid (keys %$list) {
3235         next if $opt_vmid && ($vmid ne $opt_vmid);
3236         next if !$res->{$vmid}->{pid}; # not running
3237         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3238     }
3239
3240     $qmpclient->queue_execute(undef, 2);
3241
3242     foreach my $vmid (keys %$list) {
3243         next if $opt_vmid && ($vmid ne $opt_vmid);
3244         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3245     }
3246
3247     return $res;
3248 }
3249
3250 sub foreach_drive {
3251     my ($conf, $func, @param) = @_;
3252
3253     foreach my $ds (valid_drive_names()) {
3254         next if !defined($conf->{$ds});
3255
3256         my $drive = parse_drive($ds, $conf->{$ds});
3257         next if !$drive;
3258
3259         &$func($ds, $drive, @param);
3260     }
3261 }
3262
3263 sub foreach_volid {
3264     my ($conf, $func, @param) = @_;
3265
3266     my $volhash = {};
3267
3268     my $test_volid = sub {
3269         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3270
3271         return if !$volid;
3272
3273         $volhash->{$volid}->{cdrom} //= 1;
3274         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3275
3276         $volhash->{$volid}->{replicate} //= 0;
3277         $volhash->{$volid}->{replicate} = 1 if $replicate;
3278
3279         $volhash->{$volid}->{shared} //= 0;
3280         $volhash->{$volid}->{shared} = 1 if $shared;
3281
3282         $volhash->{$volid}->{referenced_in_config} //= 0;
3283         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3284
3285         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3286             if defined($snapname);
3287         $volhash->{$volid}->{size} = $size if $size;
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub conf_has_audio {
3322     my ($conf, $id) = @_;
3323
3324     $id //= 0;
3325     my $audio = $conf->{"audio$id"};
3326     return undef if !defined($audio);
3327
3328     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3329     my $audiodriver = $audioproperties->{driver} // 'spice';
3330
3331     return {
3332         dev => $audioproperties->{device},
3333         dev_id => "audiodev$id",
3334         backend => $audiodriver,
3335         backend_id => "$audiodriver-backend${id}",
3336     };
3337 }
3338
3339 sub vga_conf_has_spice {
3340     my ($vga) = @_;
3341
3342     my $vgaconf = parse_vga($vga);
3343     my $vgatype = $vgaconf->{type};
3344     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3345
3346     return $1 || 1;
3347 }
3348
3349 sub is_native($) {
3350     my ($arch) = @_;
3351     return get_host_arch() eq $arch;
3352 }
3353
3354 sub get_vm_arch {
3355     my ($conf) = @_;
3356     return $conf->{arch} // get_host_arch();
3357 }
3358
3359 my $default_machines = {
3360     x86_64 => 'pc',
3361     aarch64 => 'virt',
3362 };
3363
3364 sub get_vm_machine {
3365     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3366
3367     my $machine = $forcemachine || $conf->{machine};
3368
3369     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370         $arch //= 'x86_64';
3371         $machine ||= $default_machines->{$arch};
3372         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3373     }
3374
3375     return $machine;
3376 }
3377
3378 sub get_ovmf_files($) {
3379     my ($arch) = @_;
3380
3381     my $ovmf = $OVMF->{$arch}
3382         or die "no OVMF images known for architecture '$arch'\n";
3383
3384     return @$ovmf;
3385 }
3386
3387 my $Arch2Qemu = {
3388     aarch64 => '/usr/bin/qemu-system-aarch64',
3389     x86_64 => '/usr/bin/qemu-system-x86_64',
3390 };
3391 sub get_command_for_arch($) {
3392     my ($arch) = @_;
3393     return '/usr/bin/kvm' if is_native($arch);
3394
3395     my $cmd = $Arch2Qemu->{$arch}
3396         or die "don't know how to emulate architecture '$arch'\n";
3397     return $cmd;
3398 }
3399
3400 sub get_cpu_options {
3401     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3402
3403     my $cpuFlags = [];
3404     my $ostype = $conf->{ostype};
3405
3406     my $cpu = $kvm ? "kvm64" : "qemu64";
3407     if ($arch eq 'aarch64') {
3408         $cpu = 'cortex-a57';
3409     }
3410     my $hv_vendor_id;
3411     if (my $cputype = $conf->{cpu}) {
3412         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3413             or die "Cannot parse cpu description: $cputype\n";
3414         $cpu = $cpuconf->{cputype};
3415         $kvm_off = 1 if $cpuconf->{hidden};
3416         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3417
3418         if (defined(my $flags = $cpuconf->{flags})) {
3419             push @$cpuFlags, split(";", $flags);
3420         }
3421     }
3422
3423     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3424
3425     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3426
3427     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3428
3429     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3430
3431     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3432
3433         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3434         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3435     }
3436
3437     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3438
3439     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3440
3441     push @$cpuFlags, 'kvm=off' if $kvm_off;
3442
3443     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3444         push @$cpuFlags, "vendor=${cpu_vendor}"
3445             if $cpu_vendor ne 'default';
3446     } elsif ($arch ne 'aarch64') {
3447         die "internal error"; # should not happen
3448     }
3449
3450     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3451
3452     return ('-cpu', $cpu);
3453 }
3454
3455 sub config_to_command {
3456     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3457
3458     my $cmd = [];
3459     my $globalFlags = [];
3460     my $machineFlags = [];
3461     my $rtcFlags = [];
3462     my $devices = [];
3463     my $pciaddr = '';
3464     my $bridges = {};
3465     my $vernum = 0; # unknown
3466     my $ostype = $conf->{ostype};
3467     my $winversion = windows_version($ostype);
3468     my $kvm = $conf->{kvm};
3469     my $nodename = nodename();
3470
3471     my $arch = get_vm_arch($conf);
3472     my $kvm_binary = get_command_for_arch($arch);
3473     my $kvmver = kvm_user_version($kvm_binary);
3474
3475     my $add_pve_version = min_version($kvmver, 4, 1);
3476
3477     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3478     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3479     $kvm //= 1 if is_native($arch);
3480
3481     $machine_version =~ m/(\d+)\.(\d+)/;
3482     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3483         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3484
3485     if ($kvm) {
3486         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3487             if !defined kvm_version();
3488     }
3489
3490     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3491         $vernum = $1*1000000+$2*1000;
3492     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3493         $vernum = $1*1000000+$2*1000+$3;
3494     }
3495
3496     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3497
3498     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3499     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3500     my $use_old_bios_files = undef;
3501     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3502
3503     my $cpuunits = defined($conf->{cpuunits}) ?
3504             $conf->{cpuunits} : $defaults->{cpuunits};
3505
3506     push @$cmd, $kvm_binary;
3507
3508     push @$cmd, '-id', $vmid;
3509
3510     my $vmname = $conf->{name} || "vm$vmid";
3511
3512     push @$cmd, '-name', $vmname;
3513
3514     my $use_virtio = 0;
3515
3516     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3517     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3518     push @$cmd, '-mon', "chardev=qmp,mode=control";
3519
3520     if (min_version($machine_version, 2, 12)) {
3521         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3522         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3523     }
3524
3525     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3526
3527     push @$cmd, '-daemonize';
3528
3529     if ($conf->{smbios1}) {
3530         my $smbios_conf = parse_smbios1($conf->{smbios1});
3531         if ($smbios_conf->{base64}) {
3532             # Do not pass base64 flag to qemu
3533             delete $smbios_conf->{base64};
3534             my $smbios_string = "";
3535             foreach my $key (keys %$smbios_conf) {
3536                 my $value;
3537                 if ($key eq "uuid") {
3538                     $value = $smbios_conf->{uuid}
3539                 } else {
3540                     $value = decode_base64($smbios_conf->{$key});
3541                 }
3542                 # qemu accepts any binary data, only commas need escaping by double comma
3543                 $value =~ s/,/,,/g;
3544                 $smbios_string .= "," . $key . "=" . $value if $value;
3545             }
3546             push @$cmd, '-smbios', "type=1" . $smbios_string;
3547         } else {
3548             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3549         }
3550     }
3551
3552     if ($conf->{vmgenid}) {
3553         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3554     }
3555
3556     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3557     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3558         die "uefi base image not found\n" if ! -f $ovmf_code;
3559
3560         my $path;
3561         my $format;
3562         if (my $efidisk = $conf->{efidisk0}) {
3563             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3564             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3565             $format = $d->{format};
3566             if ($storeid) {
3567                 $path = PVE::Storage::path($storecfg, $d->{file});
3568                 if (!defined($format)) {
3569                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3570                     $format = qemu_img_format($scfg, $volname);
3571                 }
3572             } else {
3573                 $path = $d->{file};
3574                 die "efidisk format must be specified\n"
3575                     if !defined($format);
3576             }
3577         } else {
3578             warn "no efidisk configured! Using temporary efivars disk.\n";
3579             $path = "/tmp/$vmid-ovmf.fd";
3580             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3581             $format = 'raw';
3582         }
3583
3584         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3585         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3586     }
3587
3588     # load q35 config
3589     if ($q35) {
3590         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3591         if (min_version($machine_version, 4, 0)) {
3592             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3593         } else {
3594             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3595         }
3596     }
3597
3598     # add usb controllers
3599     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3600     push @$devices, @usbcontrollers if @usbcontrollers;
3601     my $vga = parse_vga($conf->{vga});
3602
3603     my $qxlnum = vga_conf_has_spice($conf->{vga});
3604     $vga->{type} = 'qxl' if $qxlnum;
3605
3606     if (!$vga->{type}) {
3607         if ($arch eq 'aarch64') {
3608             $vga->{type} = 'virtio';
3609         } elsif (min_version($machine_version, 2, 9)) {
3610             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3611         } else {
3612             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3613         }
3614     }
3615
3616     # enable absolute mouse coordinates (needed by vnc)
3617     my $tablet;
3618     if (defined($conf->{tablet})) {
3619         $tablet = $conf->{tablet};
3620     } else {
3621         $tablet = $defaults->{tablet};
3622         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3623         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3624     }
3625
3626     if ($tablet) {
3627         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3628         my $kbd = print_keyboarddevice_full($conf, $arch);
3629         push @$devices, '-device', $kbd if defined($kbd);
3630     }
3631
3632     my $kvm_off = 0;
3633     my $gpu_passthrough;
3634
3635     # host pci devices
3636     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3637         my $id = "hostpci$i";
3638         my $d = parse_hostpci($conf->{$id});
3639         next if !$d;
3640
3641         if (my $pcie = $d->{pcie}) {
3642             die "q35 machine model is not enabled" if !$q35;
3643             # win7 wants to have the pcie devices directly on the pcie bus
3644             # instead of in the root port
3645             if ($winversion == 7) {
3646                 $pciaddr = print_pcie_addr("${id}bus0");
3647             } else {
3648                 # add more root ports if needed, 4 are present by default
3649                 # by pve-q35 cfgs, rest added here on demand.
3650                 if ($i > 3) {
3651                     push @$devices, '-device', print_pcie_root_port($i);
3652                 }
3653                 $pciaddr = print_pcie_addr($id);
3654             }
3655         } else {
3656             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3657         }
3658
3659         my $xvga = '';
3660         if ($d->{'x-vga'}) {
3661             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3662             $kvm_off = 1;
3663             $vga->{type} = 'none' if !defined($conf->{vga});
3664             $gpu_passthrough = 1;
3665         }
3666
3667         my $pcidevices = $d->{pciid};
3668         my $multifunction = 1 if @$pcidevices > 1;
3669
3670         my $sysfspath;
3671         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3672             my $pci_id = $pcidevices->[0]->{id};
3673             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3674             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3675         } elsif ($d->{mdev}) {
3676             warn "ignoring mediated device '$id' with multifunction device\n";
3677         }
3678
3679         my $j=0;
3680         foreach my $pcidevice (@$pcidevices) {
3681             my $devicestr = "vfio-pci";
3682
3683             if ($sysfspath) {
3684                 $devicestr .= ",sysfsdev=$sysfspath";
3685             } else {
3686                 $devicestr .= ",host=$pcidevice->{id}";
3687             }
3688
3689             my $mf_addr = $multifunction ? ".$j" : '';
3690             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3691
3692             if ($j == 0) {
3693                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3694                 $devicestr .= "$xvga";
3695                 $devicestr .= ",multifunction=on" if $multifunction;
3696                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3697             }
3698
3699             push @$devices, '-device', $devicestr;
3700             $j++;
3701         }
3702     }
3703
3704     # usb devices
3705     my $usb_dev_features = {};
3706     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3707
3708     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3709     push @$devices, @usbdevices if @usbdevices;
3710     # serial devices
3711     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3712         if (my $path = $conf->{"serial$i"}) {
3713             if ($path eq 'socket') {
3714                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3715                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3716                 # On aarch64, serial0 is the UART device. Qemu only allows
3717                 # connecting UART devices via the '-serial' command line, as
3718                 # the device has a fixed slot on the hardware...
3719                 if ($arch eq 'aarch64' && $i == 0) {
3720                     push @$devices, '-serial', "chardev:serial$i";
3721                 } else {
3722                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3723                 }
3724             } else {
3725                 die "no such serial device\n" if ! -c $path;
3726                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3727                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3728             }
3729         }
3730     }
3731
3732     # parallel devices
3733     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3734         if (my $path = $conf->{"parallel$i"}) {
3735             die "no such parallel device\n" if ! -c $path;
3736             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3737             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3738             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3739         }
3740     }
3741
3742     if (my $audio = conf_has_audio($conf)) {
3743
3744         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3745
3746         my $id = $audio->{dev_id};
3747         if ($audio->{dev} eq 'AC97') {
3748             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3749         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3750             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3751             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3752             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3753         } else {
3754             die "unkown audio device '$audio->{dev}', implement me!";
3755         }
3756
3757         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3758     }
3759
3760     my $sockets = 1;
3761     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3762     $sockets = $conf->{sockets} if  $conf->{sockets};
3763
3764     my $cores = $conf->{cores} || 1;
3765
3766     my $maxcpus = $sockets * $cores;
3767
3768     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3769
3770     my $allowed_vcpus = $cpuinfo->{cpus};
3771
3772     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3773         if ($allowed_vcpus < $maxcpus);
3774
3775     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3776
3777         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3778         for (my $i = 2; $i <= $vcpus; $i++)  {
3779             my $cpustr = print_cpu_device($conf,$i);
3780             push @$cmd, '-device', $cpustr;
3781         }
3782
3783     } else {
3784
3785         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3786     }
3787     push @$cmd, '-nodefaults';
3788
3789     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3790
3791     my $bootindex_hash = {};
3792     my $i = 1;
3793     foreach my $o (split(//, $bootorder)) {
3794         $bootindex_hash->{$o} = $i*100;
3795         $i++;
3796     }
3797
3798     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3799
3800     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3801
3802     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3803
3804     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3805         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3806         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3807         push @$cmd,  '-vnc', "unix:$socket,password";
3808     } else {
3809         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3810         push @$cmd, '-nographic';
3811     }
3812
3813     # time drift fix
3814     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3815
3816     my $useLocaltime = $conf->{localtime};
3817
3818     if ($winversion >= 5) { # windows
3819         $useLocaltime = 1 if !defined($conf->{localtime});
3820
3821         # use time drift fix when acpi is enabled
3822         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3823             $tdf = 1 if !defined($conf->{tdf});
3824         }
3825     }
3826
3827     if ($winversion >= 6) {
3828         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3829         push @$cmd, '-no-hpet';
3830     }
3831
3832     push @$rtcFlags, 'driftfix=slew' if $tdf;
3833
3834     if (!$kvm) {
3835         push @$machineFlags, 'accel=tcg';
3836     }
3837
3838     if ($machine_type) {
3839         push @$machineFlags, "type=${machine_type}";
3840     }
3841
3842     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3843         push @$rtcFlags, "base=$conf->{startdate}";
3844     } elsif ($useLocaltime) {
3845         push @$rtcFlags, 'base=localtime';
3846     }
3847
3848     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3849
3850     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3851
3852     push @$cmd, '-S' if $conf->{freeze};
3853
3854     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3855
3856     my $guest_agent = parse_guest_agent($conf);
3857
3858     if ($guest_agent->{enabled}) {
3859         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3860         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3861
3862         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3863             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3864             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3865             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3866         } elsif ($guest_agent->{type} eq 'isa') {
3867             push @$devices, '-device', "isa-serial,chardev=qga0";
3868         }
3869     }
3870
3871     my $spice_port;
3872
3873     if ($qxlnum) {
3874         if ($qxlnum > 1) {
3875             if ($winversion){
3876                 for(my $i = 1; $i < $qxlnum; $i++){
3877                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3878                 }
3879             } else {
3880                 # assume other OS works like Linux
3881                 my ($ram, $vram) = ("134217728", "67108864");
3882                 if ($vga->{memory}) {
3883                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3884                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3885                 }
3886                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3887                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3888             }
3889         }
3890
3891         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3892
3893         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3894         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3895         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3896
3897         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3898         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3899         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3900
3901         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3902         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3903
3904         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3905         if ($spice_enhancement->{foldersharing}) {
3906             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3907             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3908         }
3909
3910         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3911         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3912         push @$devices, '-spice', "$spice_opts";
3913     }
3914
3915     # enable balloon by default, unless explicitly disabled
3916     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3917         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3918         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3919     }
3920
3921     if ($conf->{watchdog}) {
3922         my $wdopts = parse_watchdog($conf->{watchdog});
3923         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3924         my $watchdog = $wdopts->{model} || 'i6300esb';
3925         push @$devices, '-device', "$watchdog$pciaddr";
3926         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3927     }
3928
3929     my $vollist = [];
3930     my $scsicontroller = {};
3931     my $ahcicontroller = {};
3932     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3933
3934     # Add iscsi initiator name if available
3935     if (my $initiator = get_initiator_name()) {
3936         push @$devices, '-iscsi', "initiator-name=$initiator";
3937     }
3938
3939     foreach_drive($conf, sub {
3940         my ($ds, $drive) = @_;
3941
3942         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3943             push @$vollist, $drive->{file};
3944         }
3945
3946         # ignore efidisk here, already added in bios/fw handling code above
3947         return if $drive->{interface} eq 'efidisk';
3948
3949         $use_virtio = 1 if $ds =~ m/^virtio/;
3950
3951         if (drive_is_cdrom ($drive)) {
3952             if ($bootindex_hash->{d}) {
3953                 $drive->{bootindex} = $bootindex_hash->{d};
3954                 $bootindex_hash->{d} += 1;
3955             }
3956         } else {
3957             if ($bootindex_hash->{c}) {
3958                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3959                 $bootindex_hash->{c} += 1;
3960             }
3961         }
3962
3963         if($drive->{interface} eq 'virtio'){
3964            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3965         }
3966
3967         if ($drive->{interface} eq 'scsi') {
3968
3969             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3970
3971             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3972             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3973
3974             my $iothread = '';
3975             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3976                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3977                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3978             } elsif ($drive->{iothread}) {
3979                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3980             }
3981
3982             my $queues = '';
3983             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3984                 $queues = ",num_queues=$drive->{queues}";
3985             }
3986
3987             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3988             $scsicontroller->{$controller}=1;
3989         }
3990
3991         if ($drive->{interface} eq 'sata') {
3992            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3993            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3994            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3995            $ahcicontroller->{$controller}=1;
3996         }
3997
3998         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3999         push @$devices, '-drive',$drive_cmd;
4000         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4001     });
4002
4003     for (my $i = 0; $i < $MAX_NETS; $i++) {
4004          next if !$conf->{"net$i"};
4005          my $d = parse_net($conf->{"net$i"});
4006          next if !$d;
4007
4008          $use_virtio = 1 if $d->{model} eq 'virtio';
4009
4010          if ($bootindex_hash->{n}) {
4011             $d->{bootindex} = $bootindex_hash->{n};
4012             $bootindex_hash->{n} += 1;
4013          }
4014
4015          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4016          push @$devices, '-netdev', $netdevfull;
4017
4018          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4019          push @$devices, '-device', $netdevicefull;
4020     }
4021
4022     if ($conf->{ivshmem}) {
4023         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4024
4025         my $bus;
4026         if ($q35) {
4027             $bus = print_pcie_addr("ivshmem");
4028         } else {
4029             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4030         }
4031
4032         my $ivshmem_name = $ivshmem->{name} // $vmid;
4033         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4034
4035         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4036         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4037     }
4038
4039     if (!$q35) {
4040         # add pci bridges
4041         if (min_version($machine_version, 2, 3)) {
4042            $bridges->{1} = 1;
4043            $bridges->{2} = 1;
4044         }
4045
4046         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4047
4048         for my $k (sort {$b cmp $a} keys %$bridges) {
4049             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4050             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4051         }
4052     }
4053
4054     push @$cmd, @$devices;
4055     push @$cmd, '-rtc', join(',', @$rtcFlags)
4056         if scalar(@$rtcFlags);
4057     push @$cmd, '-machine', join(',', @$machineFlags)
4058         if scalar(@$machineFlags);
4059     push @$cmd, '-global', join(',', @$globalFlags)
4060         if scalar(@$globalFlags);
4061
4062     if (my $vmstate = $conf->{vmstate}) {
4063         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4064         push @$vollist, $vmstate;
4065         push @$cmd, '-loadstate', $statepath;
4066         print "activating and using '$vmstate' as vmstate\n";
4067     }
4068
4069     # add custom args
4070     if ($conf->{args}) {
4071         my $aa = PVE::Tools::split_args($conf->{args});
4072         push @$cmd, @$aa;
4073     }
4074
4075     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4076 }
4077
4078 sub spice_port {
4079     my ($vmid) = @_;
4080
4081     my $res = mon_cmd($vmid, 'query-spice');
4082
4083     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4084 }
4085
4086 sub vm_devices_list {
4087     my ($vmid) = @_;
4088
4089     my $res = mon_cmd($vmid, 'query-pci');
4090     my $devices_to_check = [];
4091     my $devices = {};
4092     foreach my $pcibus (@$res) {
4093         push @$devices_to_check, @{$pcibus->{devices}},
4094     }
4095
4096     while (@$devices_to_check) {
4097         my $to_check = [];
4098         for my $d (@$devices_to_check) {
4099             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4100             next if !$d->{'pci_bridge'};
4101
4102             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4103             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4104         }
4105         $devices_to_check = $to_check;
4106     }
4107
4108     my $resblock = mon_cmd($vmid, 'query-block');
4109     foreach my $block (@$resblock) {
4110         if($block->{device} =~ m/^drive-(\S+)/){
4111                 $devices->{$1} = 1;
4112         }
4113     }
4114
4115     my $resmice = mon_cmd($vmid, 'query-mice');
4116     foreach my $mice (@$resmice) {
4117         if ($mice->{name} eq 'QEMU HID Tablet') {
4118             $devices->{tablet} = 1;
4119             last;
4120         }
4121     }
4122
4123     # for usb devices there is no query-usb
4124     # but we can iterate over the entries in
4125     # qom-list path=/machine/peripheral
4126     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4127     foreach my $per (@$resperipheral) {
4128         if ($per->{name} =~ m/^usb\d+$/) {
4129             $devices->{$per->{name}} = 1;
4130         }
4131     }
4132
4133     return $devices;
4134 }
4135
4136 sub vm_deviceplug {
4137     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4138
4139     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4140
4141     my $devices_list = vm_devices_list($vmid);
4142     return 1 if defined($devices_list->{$deviceid});
4143
4144     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4145
4146     if ($deviceid eq 'tablet') {
4147
4148         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4149
4150     } elsif ($deviceid eq 'keyboard') {
4151
4152         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4153
4154     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4155
4156         die "usb hotplug currently not reliable\n";
4157         # since we can't reliably hot unplug all added usb devices
4158         # and usb passthrough disables live migration
4159         # we disable usb hotplugging for now
4160         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4161
4162     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4163
4164         qemu_iothread_add($vmid, $deviceid, $device);
4165
4166         qemu_driveadd($storecfg, $vmid, $device);
4167         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4168
4169         qemu_deviceadd($vmid, $devicefull);
4170         eval { qemu_deviceaddverify($vmid, $deviceid); };
4171         if (my $err = $@) {
4172             eval { qemu_drivedel($vmid, $deviceid); };
4173             warn $@ if $@;
4174             die $err;
4175         }
4176
4177     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4178
4179
4180         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4181         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4182         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4183
4184         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4185
4186         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4187             qemu_iothread_add($vmid, $deviceid, $device);
4188             $devicefull .= ",iothread=iothread-$deviceid";
4189         }
4190
4191         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4192             $devicefull .= ",num_queues=$device->{queues}";
4193         }
4194
4195         qemu_deviceadd($vmid, $devicefull);
4196         qemu_deviceaddverify($vmid, $deviceid);
4197
4198     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4199
4200         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4201         qemu_driveadd($storecfg, $vmid, $device);
4202
4203         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4204         eval { qemu_deviceadd($vmid, $devicefull); };
4205         if (my $err = $@) {
4206             eval { qemu_drivedel($vmid, $deviceid); };
4207             warn $@ if $@;
4208             die $err;
4209         }
4210
4211     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4212
4213         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4214
4215         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4216         my $use_old_bios_files = undef;
4217         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4218
4219         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4220         qemu_deviceadd($vmid, $netdevicefull);
4221         eval {
4222             qemu_deviceaddverify($vmid, $deviceid);
4223             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4224         };
4225         if (my $err = $@) {
4226             eval { qemu_netdevdel($vmid, $deviceid); };
4227             warn $@ if $@;
4228             die $err;
4229         }
4230
4231     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4232
4233         my $bridgeid = $2;
4234         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4235         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4236
4237         qemu_deviceadd($vmid, $devicefull);
4238         qemu_deviceaddverify($vmid, $deviceid);
4239
4240     } else {
4241         die "can't hotplug device '$deviceid'\n";
4242     }
4243
4244     return 1;
4245 }
4246
4247 # fixme: this should raise exceptions on error!
4248 sub vm_deviceunplug {
4249     my ($vmid, $conf, $deviceid) = @_;
4250
4251     my $devices_list = vm_devices_list($vmid);
4252     return 1 if !defined($devices_list->{$deviceid});
4253
4254     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4255
4256     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4257
4258         qemu_devicedel($vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^usb\d+$/) {
4261
4262         die "usb hotplug currently not reliable\n";
4263         # when unplugging usb devices this way,
4264         # there may be remaining usb controllers/hubs
4265         # so we disable it for now
4266         qemu_devicedel($vmid, $deviceid);
4267         qemu_devicedelverify($vmid, $deviceid);
4268
4269     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4270
4271         qemu_devicedel($vmid, $deviceid);
4272         qemu_devicedelverify($vmid, $deviceid);
4273         qemu_drivedel($vmid, $deviceid);
4274         qemu_iothread_del($conf, $vmid, $deviceid);
4275
4276     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4277
4278         qemu_devicedel($vmid, $deviceid);
4279         qemu_devicedelverify($vmid, $deviceid);
4280         qemu_iothread_del($conf, $vmid, $deviceid);
4281
4282     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4283
4284         qemu_devicedel($vmid, $deviceid);
4285         qemu_drivedel($vmid, $deviceid);
4286         qemu_deletescsihw($conf, $vmid, $deviceid);
4287
4288     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4289
4290         qemu_devicedel($vmid, $deviceid);
4291         qemu_devicedelverify($vmid, $deviceid);
4292         qemu_netdevdel($vmid, $deviceid);
4293
4294     } else {
4295         die "can't unplug device '$deviceid'\n";
4296     }
4297
4298     return 1;
4299 }
4300
4301 sub qemu_deviceadd {
4302     my ($vmid, $devicefull) = @_;
4303
4304     $devicefull = "driver=".$devicefull;
4305     my %options =  split(/[=,]/, $devicefull);
4306
4307     mon_cmd($vmid, "device_add" , %options);
4308 }
4309
4310 sub qemu_devicedel {
4311     my ($vmid, $deviceid) = @_;
4312
4313     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4314 }
4315
4316 sub qemu_iothread_add {
4317     my($vmid, $deviceid, $device) = @_;
4318
4319     if ($device->{iothread}) {
4320         my $iothreads = vm_iothreads_list($vmid);
4321         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4322     }
4323 }
4324
4325 sub qemu_iothread_del {
4326     my($conf, $vmid, $deviceid) = @_;
4327
4328     my $confid = $deviceid;
4329     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4330         $confid = 'scsi' . $1;
4331     }
4332     my $device = parse_drive($confid, $conf->{$confid});
4333     if ($device->{iothread}) {
4334         my $iothreads = vm_iothreads_list($vmid);
4335         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4336     }
4337 }
4338
4339 sub qemu_objectadd {
4340     my($vmid, $objectid, $qomtype) = @_;
4341
4342     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4343
4344     return 1;
4345 }
4346
4347 sub qemu_objectdel {
4348     my($vmid, $objectid) = @_;
4349
4350     mon_cmd($vmid, "object-del", id => $objectid);
4351
4352     return 1;
4353 }
4354
4355 sub qemu_driveadd {
4356     my ($storecfg, $vmid, $device) = @_;
4357
4358     my $drive = print_drive_full($storecfg, $vmid, $device);
4359     $drive =~ s/\\/\\\\/g;
4360     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4361
4362     # If the command succeeds qemu prints: "OK"
4363     return 1 if $ret =~ m/OK/s;
4364
4365     die "adding drive failed: $ret\n";
4366 }
4367
4368 sub qemu_drivedel {
4369     my($vmid, $deviceid) = @_;
4370
4371     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4372     $ret =~ s/^\s+//;
4373
4374     return 1 if $ret eq "";
4375
4376     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4377     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4378
4379     die "deleting drive $deviceid failed : $ret\n";
4380 }
4381
4382 sub qemu_deviceaddverify {
4383     my ($vmid, $deviceid) = @_;
4384
4385     for (my $i = 0; $i <= 5; $i++) {
4386          my $devices_list = vm_devices_list($vmid);
4387          return 1 if defined($devices_list->{$deviceid});
4388          sleep 1;
4389     }
4390
4391     die "error on hotplug device '$deviceid'\n";
4392 }
4393
4394
4395 sub qemu_devicedelverify {
4396     my ($vmid, $deviceid) = @_;
4397
4398     # need to verify that the device is correctly removed as device_del
4399     # is async and empty return is not reliable
4400
4401     for (my $i = 0; $i <= 5; $i++) {
4402          my $devices_list = vm_devices_list($vmid);
4403          return 1 if !defined($devices_list->{$deviceid});
4404          sleep 1;
4405     }
4406
4407     die "error on hot-unplugging device '$deviceid'\n";
4408 }
4409
4410 sub qemu_findorcreatescsihw {
4411     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4412
4413     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4414
4415     my $scsihwid="$controller_prefix$controller";
4416     my $devices_list = vm_devices_list($vmid);
4417
4418     if(!defined($devices_list->{$scsihwid})) {
4419         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4420     }
4421
4422     return 1;
4423 }
4424
4425 sub qemu_deletescsihw {
4426     my ($conf, $vmid, $opt) = @_;
4427
4428     my $device = parse_drive($opt, $conf->{$opt});
4429
4430     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4431         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4432         return 1;
4433     }
4434
4435     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4436
4437     my $devices_list = vm_devices_list($vmid);
4438     foreach my $opt (keys %{$devices_list}) {
4439         if (PVE::QemuServer::is_valid_drivename($opt)) {
4440             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4441             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4442                 return 1;
4443             }
4444         }
4445     }
4446
4447     my $scsihwid="scsihw$controller";
4448
4449     vm_deviceunplug($vmid, $conf, $scsihwid);
4450
4451     return 1;
4452 }
4453
4454 sub qemu_add_pci_bridge {
4455     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4456
4457     my $bridges = {};
4458
4459     my $bridgeid;
4460
4461     print_pci_addr($device, $bridges, $arch, $machine_type);
4462
4463     while (my ($k, $v) = each %$bridges) {
4464         $bridgeid = $k;
4465     }
4466     return 1 if !defined($bridgeid) || $bridgeid < 1;
4467
4468     my $bridge = "pci.$bridgeid";
4469     my $devices_list = vm_devices_list($vmid);
4470
4471     if (!defined($devices_list->{$bridge})) {
4472         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4473     }
4474
4475     return 1;
4476 }
4477
4478 sub qemu_set_link_status {
4479     my ($vmid, $device, $up) = @_;
4480
4481     mon_cmd($vmid, "set_link", name => $device,
4482                up => $up ? JSON::true : JSON::false);
4483 }
4484
4485 sub qemu_netdevadd {
4486     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4487
4488     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4489     my %options =  split(/[=,]/, $netdev);
4490
4491     mon_cmd($vmid, "netdev_add",  %options);
4492     return 1;
4493 }
4494
4495 sub qemu_netdevdel {
4496     my ($vmid, $deviceid) = @_;
4497
4498     mon_cmd($vmid, "netdev_del", id => $deviceid);
4499 }
4500
4501 sub qemu_usb_hotplug {
4502     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4503
4504     return if !$device;
4505
4506     # remove the old one first
4507     vm_deviceunplug($vmid, $conf, $deviceid);
4508
4509     # check if xhci controller is necessary and available
4510     if ($device->{usb3}) {
4511
4512         my $devicelist = vm_devices_list($vmid);
4513
4514         if (!$devicelist->{xhci}) {
4515             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4516             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4517         }
4518     }
4519     my $d = parse_usb_device($device->{host});
4520     $d->{usb3} = $device->{usb3};
4521
4522     # add the new one
4523     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4524 }
4525
4526 sub qemu_cpu_hotplug {
4527     my ($vmid, $conf, $vcpus) = @_;
4528
4529     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4530
4531     my $sockets = 1;
4532     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4533     $sockets = $conf->{sockets} if  $conf->{sockets};
4534     my $cores = $conf->{cores} || 1;
4535     my $maxcpus = $sockets * $cores;
4536
4537     $vcpus = $maxcpus if !$vcpus;
4538
4539     die "you can't add more vcpus than maxcpus\n"
4540         if $vcpus > $maxcpus;
4541
4542     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4543
4544     if ($vcpus < $currentvcpus) {
4545
4546         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4547
4548             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4549                 qemu_devicedel($vmid, "cpu$i");
4550                 my $retry = 0;
4551                 my $currentrunningvcpus = undef;
4552                 while (1) {
4553                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4554                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4555                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4556                     $retry++;
4557                     sleep 1;
4558                 }
4559                 #update conf after each succesfull cpu unplug
4560                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4561                 PVE::QemuConfig->write_config($vmid, $conf);
4562             }
4563         } else {
4564             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4565         }
4566
4567         return;
4568     }
4569
4570     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4571     die "vcpus in running vm does not match its configuration\n"
4572         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4573
4574     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4575
4576         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4577             my $cpustr = print_cpu_device($conf, $i);
4578             qemu_deviceadd($vmid, $cpustr);
4579
4580             my $retry = 0;
4581             my $currentrunningvcpus = undef;
4582             while (1) {
4583                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4584                 last if scalar(@{$currentrunningvcpus}) == $i;
4585                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4586                 sleep 1;
4587                 $retry++;
4588             }
4589             #update conf after each succesfull cpu hotplug
4590             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4591             PVE::QemuConfig->write_config($vmid, $conf);
4592         }
4593     } else {
4594
4595         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4596             mon_cmd($vmid, "cpu-add", id => int($i));
4597         }
4598     }
4599 }
4600
4601 sub qemu_block_set_io_throttle {
4602     my ($vmid, $deviceid,
4603         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4604         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4605         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4606         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4607
4608     return if !check_running($vmid) ;
4609
4610     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4611         bps => int($bps),
4612         bps_rd => int($bps_rd),
4613         bps_wr => int($bps_wr),
4614         iops => int($iops),
4615         iops_rd => int($iops_rd),
4616         iops_wr => int($iops_wr),
4617         bps_max => int($bps_max),
4618         bps_rd_max => int($bps_rd_max),
4619         bps_wr_max => int($bps_wr_max),
4620         iops_max => int($iops_max),
4621         iops_rd_max => int($iops_rd_max),
4622         iops_wr_max => int($iops_wr_max),
4623         bps_max_length => int($bps_max_length),
4624         bps_rd_max_length => int($bps_rd_max_length),
4625         bps_wr_max_length => int($bps_wr_max_length),
4626         iops_max_length => int($iops_max_length),
4627         iops_rd_max_length => int($iops_rd_max_length),
4628         iops_wr_max_length => int($iops_wr_max_length),
4629     );
4630
4631 }
4632
4633 # old code, only used to shutdown old VM after update
4634 sub __read_avail {
4635     my ($fh, $timeout) = @_;
4636
4637     my $sel = new IO::Select;
4638     $sel->add($fh);
4639
4640     my $res = '';
4641     my $buf;
4642
4643     my @ready;
4644     while (scalar (@ready = $sel->can_read($timeout))) {
4645         my $count;
4646         if ($count = $fh->sysread($buf, 8192)) {
4647             if ($buf =~ /^(.*)\(qemu\) $/s) {
4648                 $res .= $1;
4649                 last;
4650             } else {
4651                 $res .= $buf;
4652             }
4653         } else {
4654             if (!defined($count)) {
4655                 die "$!\n";
4656             }
4657             last;
4658         }
4659     }
4660
4661     die "monitor read timeout\n" if !scalar(@ready);
4662
4663     return $res;
4664 }
4665
4666 sub qemu_block_resize {
4667     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4668
4669     my $running = check_running($vmid);
4670
4671     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4672
4673     return if !$running;
4674
4675     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4676
4677 }
4678
4679 sub qemu_volume_snapshot {
4680     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4681
4682     my $running = check_running($vmid);
4683
4684     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4685         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4686     } else {
4687         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4688     }
4689 }
4690
4691 sub qemu_volume_snapshot_delete {
4692     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4693
4694     my $running = check_running($vmid);
4695
4696     if($running) {
4697
4698         $running = undef;
4699         my $conf = PVE::QemuConfig->load_config($vmid);
4700         foreach_drive($conf, sub {
4701             my ($ds, $drive) = @_;
4702             $running = 1 if $drive->{file} eq $volid;
4703         });
4704     }
4705
4706     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4707         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4708     } else {
4709         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4710     }
4711 }
4712
4713 sub set_migration_caps {
4714     my ($vmid) = @_;
4715
4716     my $cap_ref = [];
4717
4718     my $enabled_cap = {
4719         "auto-converge" => 1,
4720         "xbzrle" => 1,
4721         "x-rdma-pin-all" => 0,
4722         "zero-blocks" => 0,
4723         "compress" => 0
4724     };
4725
4726     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4727
4728     for my $supported_capability (@$supported_capabilities) {
4729         push @$cap_ref, {
4730             capability => $supported_capability->{capability},
4731             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4732         };
4733     }
4734
4735     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4736 }
4737
4738 my $fast_plug_option = {
4739     'lock' => 1,
4740     'name' => 1,
4741     'onboot' => 1,
4742     'shares' => 1,
4743     'startup' => 1,
4744     'description' => 1,
4745     'protection' => 1,
4746     'vmstatestorage' => 1,
4747     'hookscript' => 1,
4748     'tags' => 1,
4749 };
4750
4751 # hotplug changes in [PENDING]
4752 # $selection hash can be used to only apply specified options, for
4753 # example: { cores => 1 } (only apply changed 'cores')
4754 # $errors ref is used to return error messages
4755 sub vmconfig_hotplug_pending {
4756     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4757
4758     my $defaults = load_defaults();
4759     my $arch = get_vm_arch($conf);
4760     my $machine_type = get_vm_machine($conf, undef, $arch);
4761
4762     # commit values which do not have any impact on running VM first
4763     # Note: those option cannot raise errors, we we do not care about
4764     # $selection and always apply them.
4765
4766     my $add_error = sub {
4767         my ($opt, $msg) = @_;
4768         $errors->{$opt} = "hotplug problem - $msg";
4769     };
4770
4771     my $changes = 0;
4772     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4773         if ($fast_plug_option->{$opt}) {
4774             $conf->{$opt} = $conf->{pending}->{$opt};
4775             delete $conf->{pending}->{$opt};
4776             $changes = 1;
4777         }
4778     }
4779
4780     if ($changes) {
4781         PVE::QemuConfig->write_config($vmid, $conf);
4782         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4783     }
4784
4785     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4786
4787     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4788     foreach my $opt (sort keys %$pending_delete_hash) {
4789         next if $selection && !$selection->{$opt};
4790         my $force = $pending_delete_hash->{$opt}->{force};
4791         eval {
4792             if ($opt eq 'hotplug') {
4793                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4794             } elsif ($opt eq 'tablet') {
4795                 die "skip\n" if !$hotplug_features->{usb};
4796                 if ($defaults->{tablet}) {
4797                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4798                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4799                         if $arch eq 'aarch64';
4800                 } else {
4801                     vm_deviceunplug($vmid, $conf, 'tablet');
4802                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4803                 }
4804             } elsif ($opt =~ m/^usb\d+/) {
4805                 die "skip\n";
4806                 # since we cannot reliably hot unplug usb devices
4807                 # we are disabling it
4808                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4809                 vm_deviceunplug($vmid, $conf, $opt);
4810             } elsif ($opt eq 'vcpus') {
4811                 die "skip\n" if !$hotplug_features->{cpu};
4812                 qemu_cpu_hotplug($vmid, $conf, undef);
4813             } elsif ($opt eq 'balloon') {
4814                 # enable balloon device is not hotpluggable
4815                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4816                 # here we reset the ballooning value to memory
4817                 my $balloon = $conf->{memory} || $defaults->{memory};
4818                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4819             } elsif ($fast_plug_option->{$opt}) {
4820                 # do nothing
4821             } elsif ($opt =~ m/^net(\d+)$/) {
4822                 die "skip\n" if !$hotplug_features->{network};
4823                 vm_deviceunplug($vmid, $conf, $opt);
4824             } elsif (is_valid_drivename($opt)) {
4825                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4826                 vm_deviceunplug($vmid, $conf, $opt);
4827                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4828             } elsif ($opt =~ m/^memory$/) {
4829                 die "skip\n" if !$hotplug_features->{memory};
4830                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4831             } elsif ($opt eq 'cpuunits') {
4832                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4833             } elsif ($opt eq 'cpulimit') {
4834                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4835             } else {
4836                 die "skip\n";
4837             }
4838         };
4839         if (my $err = $@) {
4840             &$add_error($opt, $err) if $err ne "skip\n";
4841         } else {
4842             # save new config if hotplug was successful
4843             delete $conf->{$opt};
4844             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4845             PVE::QemuConfig->write_config($vmid, $conf);
4846             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4847         }
4848     }
4849
4850     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4851     $apply_pending_cloudinit = sub {
4852         return if $apply_pending_cloudinit_done; # once is enough
4853         $apply_pending_cloudinit_done = 1; # once is enough
4854
4855         my ($key, $value) = @_;
4856
4857         my @cloudinit_opts = keys %$confdesc_cloudinit;
4858         foreach my $opt (keys %{$conf->{pending}}) {
4859             next if !grep { $_ eq $opt } @cloudinit_opts;
4860             $conf->{$opt} = delete $conf->{pending}->{$opt};
4861         }
4862
4863         my $new_conf = { %$conf };
4864         $new_conf->{$key} = $value;
4865         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4866     };
4867
4868     foreach my $opt (keys %{$conf->{pending}}) {
4869         next if $selection && !$selection->{$opt};
4870         my $value = $conf->{pending}->{$opt};
4871         eval {
4872             if ($opt eq 'hotplug') {
4873                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4874             } elsif ($opt eq 'tablet') {
4875                 die "skip\n" if !$hotplug_features->{usb};
4876                 if ($value == 1) {
4877                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4878                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4879                         if $arch eq 'aarch64';
4880                 } elsif ($value == 0) {
4881                     vm_deviceunplug($vmid, $conf, 'tablet');
4882                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4883                 }
4884             } elsif ($opt =~ m/^usb\d+$/) {
4885                 die "skip\n";
4886                 # since we cannot reliably hot unplug usb devices
4887                 # we are disabling it
4888                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4889                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4890                 die "skip\n" if !$d;
4891                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4892             } elsif ($opt eq 'vcpus') {
4893                 die "skip\n" if !$hotplug_features->{cpu};
4894                 qemu_cpu_hotplug($vmid, $conf, $value);
4895             } elsif ($opt eq 'balloon') {
4896                 # enable/disable balloning device is not hotpluggable
4897                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4898                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4899                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4900
4901                 # allow manual ballooning if shares is set to zero
4902                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4903                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4904                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4905                 }
4906             } elsif ($opt =~ m/^net(\d+)$/) {
4907                 # some changes can be done without hotplug
4908                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4909                                     $vmid, $opt, $value, $arch, $machine_type);
4910             } elsif (is_valid_drivename($opt)) {
4911                 die "skip\n" if $opt eq 'efidisk0';
4912                 # some changes can be done without hotplug
4913                 my $drive = parse_drive($opt, $value);
4914                 if (drive_is_cloudinit($drive)) {
4915                     &$apply_pending_cloudinit($opt, $value);
4916                 }
4917                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4918                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4919             } elsif ($opt =~ m/^memory$/) { #dimms
4920                 die "skip\n" if !$hotplug_features->{memory};
4921                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4922             } elsif ($opt eq 'cpuunits') {
4923                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4924             } elsif ($opt eq 'cpulimit') {
4925                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4926                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4927             } else {
4928                 die "skip\n";  # skip non-hot-pluggable options
4929             }
4930         };
4931         if (my $err = $@) {
4932             &$add_error($opt, $err) if $err ne "skip\n";
4933         } else {
4934             # save new config if hotplug was successful
4935             $conf->{$opt} = $value;
4936             delete $conf->{pending}->{$opt};
4937             PVE::QemuConfig->write_config($vmid, $conf);
4938             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4939         }
4940     }
4941 }
4942
4943 sub try_deallocate_drive {
4944     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4945
4946     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4947         my $volid = $drive->{file};
4948         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4949             my $sid = PVE::Storage::parse_volume_id($volid);
4950             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4951
4952             # check if the disk is really unused
4953             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4954                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4955             PVE::Storage::vdisk_free($storecfg, $volid);
4956             return 1;
4957         } else {
4958             # If vm is not owner of this disk remove from config
4959             return 1;
4960         }
4961     }
4962
4963     return undef;
4964 }
4965
4966 sub vmconfig_delete_or_detach_drive {
4967     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4968
4969     my $drive = parse_drive($opt, $conf->{$opt});
4970
4971     my $rpcenv = PVE::RPCEnvironment::get();
4972     my $authuser = $rpcenv->get_user();
4973
4974     if ($force) {
4975         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4976         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4977     } else {
4978         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4979     }
4980 }
4981
4982
4983
4984 sub vmconfig_apply_pending {
4985     my ($vmid, $conf, $storecfg) = @_;
4986
4987     # cold plug
4988
4989     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4990     foreach my $opt (sort keys %$pending_delete_hash) {
4991         die "internal error" if $opt =~ m/^unused/;
4992         my $force = $pending_delete_hash->{$opt}->{force};
4993         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4994         if (!defined($conf->{$opt})) {
4995             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4996             PVE::QemuConfig->write_config($vmid, $conf);
4997         } elsif (is_valid_drivename($opt)) {
4998             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4999             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5000             delete $conf->{$opt};
5001             PVE::QemuConfig->write_config($vmid, $conf);
5002         } else {
5003             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5004             delete $conf->{$opt};
5005             PVE::QemuConfig->write_config($vmid, $conf);
5006         }
5007     }
5008
5009     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5010
5011     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5012         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5013
5014         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5015             # skip if nothing changed
5016         } elsif (is_valid_drivename($opt)) {
5017             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5018                 if defined($conf->{$opt});
5019             $conf->{$opt} = $conf->{pending}->{$opt};
5020         } else {
5021             $conf->{$opt} = $conf->{pending}->{$opt};
5022         }
5023
5024         delete $conf->{pending}->{$opt};
5025         PVE::QemuConfig->write_config($vmid, $conf);
5026     }
5027 }
5028
5029 my $safe_num_ne = sub {
5030     my ($a, $b) = @_;
5031
5032     return 0 if !defined($a) && !defined($b);
5033     return 1 if !defined($a);
5034     return 1 if !defined($b);
5035
5036     return $a != $b;
5037 };
5038
5039 my $safe_string_ne = sub {
5040     my ($a, $b) = @_;
5041
5042     return 0 if !defined($a) && !defined($b);
5043     return 1 if !defined($a);
5044     return 1 if !defined($b);
5045
5046     return $a ne $b;
5047 };
5048
5049 sub vmconfig_update_net {
5050     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5051
5052     my $newnet = parse_net($value);
5053
5054     if ($conf->{$opt}) {
5055         my $oldnet = parse_net($conf->{$opt});
5056
5057         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5058             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5059             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5060             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5061
5062             # for non online change, we try to hot-unplug
5063             die "skip\n" if !$hotplug;
5064             vm_deviceunplug($vmid, $conf, $opt);
5065         } else {
5066
5067             die "internal error" if $opt !~ m/net(\d+)/;
5068             my $iface = "tap${vmid}i$1";
5069
5070             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5071                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5072                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5073                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5074                 PVE::Network::tap_unplug($iface);
5075                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5076             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5077                 # Rate can be applied on its own but any change above needs to
5078                 # include the rate in tap_plug since OVS resets everything.
5079                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5080             }
5081
5082             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5083                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5084             }
5085
5086             return 1;
5087         }
5088     }
5089
5090     if ($hotplug) {
5091         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5092     } else {
5093         die "skip\n";
5094     }
5095 }
5096
5097 sub vmconfig_update_disk {
5098     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5099
5100     # fixme: do we need force?
5101
5102     my $drive = parse_drive($opt, $value);
5103
5104     if ($conf->{$opt}) {
5105
5106         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5107
5108             my $media = $drive->{media} || 'disk';
5109             my $oldmedia = $old_drive->{media} || 'disk';
5110             die "unable to change media type\n" if $media ne $oldmedia;
5111
5112             if (!drive_is_cdrom($old_drive)) {
5113
5114                 if ($drive->{file} ne $old_drive->{file}) {
5115
5116                     die "skip\n" if !$hotplug;
5117
5118                     # unplug and register as unused
5119                     vm_deviceunplug($vmid, $conf, $opt);
5120                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5121
5122                 } else {
5123                     # update existing disk
5124
5125                     # skip non hotpluggable value
5126                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5127                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5128                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5129                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5130                         die "skip\n";
5131                     }
5132
5133                     # apply throttle
5134                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5135                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5136                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5137                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5138                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5139                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5140                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5141                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5142                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5143                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5144                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5145                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5146                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5147                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5148                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5149                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5150                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5151                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5152
5153                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5154                                                    ($drive->{mbps} || 0)*1024*1024,
5155                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5156                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5157                                                    $drive->{iops} || 0,
5158                                                    $drive->{iops_rd} || 0,
5159                                                    $drive->{iops_wr} || 0,
5160                                                    ($drive->{mbps_max} || 0)*1024*1024,
5161                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5162                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5163                                                    $drive->{iops_max} || 0,
5164                                                    $drive->{iops_rd_max} || 0,
5165                                                    $drive->{iops_wr_max} || 0,
5166                                                    $drive->{bps_max_length} || 1,
5167                                                    $drive->{bps_rd_max_length} || 1,
5168                                                    $drive->{bps_wr_max_length} || 1,
5169                                                    $drive->{iops_max_length} || 1,
5170                                                    $drive->{iops_rd_max_length} || 1,
5171                                                    $drive->{iops_wr_max_length} || 1);
5172
5173                     }
5174
5175                     return 1;
5176                 }
5177
5178             } else { # cdrom
5179
5180                 if ($drive->{file} eq 'none') {
5181                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5182                     if (drive_is_cloudinit($old_drive)) {
5183                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5184                     }
5185                 } else {
5186                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5187                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5188                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5189                 }
5190
5191                 return 1;
5192             }
5193         }
5194     }
5195
5196     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5197     # hotplug new disks
5198     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5199     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5200 }
5201
5202 sub vm_start {
5203     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5204         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5205
5206     PVE::QemuConfig->lock_config($vmid, sub {
5207         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5208
5209         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5210
5211         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5212
5213         PVE::QemuConfig->check_lock($conf)
5214             if !($skiplock || $is_suspended);
5215
5216         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5217
5218         # clean up leftover reboot request files
5219         eval { clear_reboot_request($vmid); };
5220         warn $@ if $@;
5221
5222         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5223             vmconfig_apply_pending($vmid, $conf, $storecfg);
5224             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5225         }
5226
5227         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5228
5229         my $defaults = load_defaults();
5230
5231         # set environment variable useful inside network script
5232         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5233
5234         my $local_volumes = {};
5235
5236         if ($targetstorage) {
5237             foreach_drive($conf, sub {
5238                 my ($ds, $drive) = @_;
5239
5240                 return if drive_is_cdrom($drive);
5241
5242                 my $volid = $drive->{file};
5243
5244                 return if !$volid;
5245
5246                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5247
5248                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5249                 return if $scfg->{shared};
5250                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5251             });
5252
5253             my $format = undef;
5254
5255             foreach my $opt (sort keys %$local_volumes) {
5256
5257                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5258                 my $drive = parse_drive($opt, $conf->{$opt});
5259
5260                 #if remote storage is specified, use default format
5261                 if ($targetstorage && $targetstorage ne "1") {
5262                     $storeid = $targetstorage;
5263                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5264                     $format = $defFormat;
5265                 } else {
5266                     #else we use same format than original
5267                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5268                     $format = qemu_img_format($scfg, $volid);
5269                 }
5270
5271                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5272                 my $newdrive = $drive;
5273                 $newdrive->{format} = $format;
5274                 $newdrive->{file} = $newvolid;
5275                 my $drivestr = print_drive($newdrive);
5276                 $local_volumes->{$opt} = $drivestr;
5277                 #pass drive to conf for command line
5278                 $conf->{$opt} = $drivestr;
5279             }
5280         }
5281
5282         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5283
5284         if ($is_suspended) {
5285             # enforce machine type on suspended vm to ensure HW compatibility
5286             $forcemachine = $conf->{runningmachine};
5287             print "Resuming suspended VM\n";
5288         }
5289
5290         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5291
5292         my $migration_ip;
5293         my $get_migration_ip = sub {
5294             my ($cidr, $nodename) = @_;
5295
5296             return $migration_ip if defined($migration_ip);
5297
5298             if (!defined($cidr)) {
5299                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5300                 $cidr = $dc_conf->{migration}->{network};
5301             }
5302
5303             if (defined($cidr)) {
5304                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5305
5306                 die "could not get IP: no address configured on local " .
5307                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5308
5309                 die "could not get IP: multiple addresses configured on local " .
5310                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5311
5312                 $migration_ip = @$ips[0];
5313             }
5314
5315             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5316                 if !defined($migration_ip);
5317
5318             return $migration_ip;
5319         };
5320
5321         my $migrate_uri;
5322         if ($statefile) {
5323             if ($statefile eq 'tcp') {
5324                 my $localip = "localhost";
5325                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5326                 my $nodename = nodename();
5327
5328                 if (!defined($migration_type)) {
5329                     if (defined($datacenterconf->{migration}->{type})) {
5330                         $migration_type = $datacenterconf->{migration}->{type};
5331                     } else {
5332                         $migration_type = 'secure';
5333                     }
5334                 }
5335
5336                 if ($migration_type eq 'insecure') {
5337                     $localip = $get_migration_ip->($migration_network, $nodename);
5338                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5339                 }
5340
5341                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5342                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5343                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5344                 push @$cmd, '-incoming', $migrate_uri;
5345                 push @$cmd, '-S';
5346
5347             } elsif ($statefile eq 'unix') {
5348                 # should be default for secure migrations as a ssh TCP forward
5349                 # tunnel is not deterministic reliable ready and fails regurarly
5350                 # to set up in time, so use UNIX socket forwards
5351                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5352                 unlink $socket_addr;
5353
5354                 $migrate_uri = "unix:$socket_addr";
5355
5356                 push @$cmd, '-incoming', $migrate_uri;
5357                 push @$cmd, '-S';
5358
5359             } elsif (-e $statefile) {
5360                 push @$cmd, '-loadstate', $statefile;
5361             } else {
5362                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5363                 push @$vollist, $statefile;
5364                 push @$cmd, '-loadstate', $statepath;
5365             }
5366         } elsif ($paused) {
5367             push @$cmd, '-S';
5368         }
5369
5370         # host pci devices
5371         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5372           my $d = parse_hostpci($conf->{"hostpci$i"});
5373           next if !$d;
5374           my $pcidevices = $d->{pciid};
5375           foreach my $pcidevice (@$pcidevices) {
5376                 my $pciid = $pcidevice->{id};
5377
5378                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5379                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5380                 die "no pci device info for device '$pciid'\n" if !$info;
5381
5382                 if ($d->{mdev}) {
5383                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5384                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5385                 } else {
5386                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5387                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5388                     die "can't reset pci device '$pciid'\n"
5389                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5390                 }
5391           }
5392         }
5393
5394         PVE::Storage::activate_volumes($storecfg, $vollist);
5395
5396         eval {
5397             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5398                 outfunc => sub {}, errfunc => sub {});
5399         };
5400         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5401         # timeout should be more than enough here...
5402         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5403
5404         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5405                                                   : $defaults->{cpuunits};
5406
5407         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5408         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5409
5410         my %properties = (
5411             Slice => 'qemu.slice',
5412             KillMode => 'none',
5413             CPUShares => $cpuunits
5414         );
5415
5416         if (my $cpulimit = $conf->{cpulimit}) {
5417             $properties{CPUQuota} = int($cpulimit * 100);
5418         }
5419         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5420
5421         my $run_qemu = sub {
5422             PVE::Tools::run_fork sub {
5423                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5424                 run_command($cmd, %run_params);
5425             };
5426         };
5427
5428         if ($conf->{hugepages}) {
5429
5430             my $code = sub {
5431                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5432                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5433
5434                 PVE::QemuServer::Memory::hugepages_mount();
5435                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5436
5437                 eval { $run_qemu->() };
5438                 if (my $err = $@) {
5439                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5440                     die $err;
5441                 }
5442
5443                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5444             };
5445             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5446
5447         } else {
5448             eval { $run_qemu->() };
5449         }
5450
5451         if (my $err = $@) {
5452             # deactivate volumes if start fails
5453             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5454             die "start failed: $err";
5455         }
5456
5457         print "migration listens on $migrate_uri\n" if $migrate_uri;
5458
5459         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5460             eval { mon_cmd($vmid, "cont"); };
5461             warn $@ if $@;
5462         }
5463
5464         #start nbd server for storage migration
5465         if ($targetstorage) {
5466             my $nodename = nodename();
5467             my $localip = $get_migration_ip->($migration_network, $nodename);
5468             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5469             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5470
5471             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5472
5473             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5474
5475             foreach my $opt (sort keys %$local_volumes) {
5476                 my $volid = $local_volumes->{$opt};
5477                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5478                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5479                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5480             }
5481         }
5482
5483         if ($migratedfrom) {
5484             eval {
5485                 set_migration_caps($vmid);
5486             };
5487             warn $@ if $@;
5488
5489             if ($spice_port) {
5490                 print "spice listens on port $spice_port\n";
5491                 if ($spice_ticket) {
5492                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5493                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5494                 }
5495             }
5496
5497         } else {
5498             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5499                 if !$statefile && $conf->{balloon};
5500
5501             foreach my $opt (keys %$conf) {
5502                 next if $opt !~  m/^net\d+$/;
5503                 my $nicconf = parse_net($conf->{$opt});
5504                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5505             }
5506         }
5507
5508         mon_cmd($vmid, 'qom-set',
5509                     path => "machine/peripheral/balloon0",
5510                     property => "guest-stats-polling-interval",
5511                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5512
5513         if ($is_suspended) {
5514             print "Resumed VM, removing state\n";
5515             if (my $vmstate = $conf->{vmstate}) {
5516                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5517                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5518             }
5519             delete $conf->@{qw(lock vmstate runningmachine)};
5520             PVE::QemuConfig->write_config($vmid, $conf);
5521         }
5522
5523         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5524     });
5525 }
5526
5527 sub vm_commandline {
5528     my ($storecfg, $vmid, $snapname) = @_;
5529
5530     my $conf = PVE::QemuConfig->load_config($vmid);
5531     my $forcemachine;
5532
5533     if ($snapname) {
5534         my $snapshot = $conf->{snapshots}->{$snapname};
5535         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5536
5537         # check for a 'runningmachine' in snapshot
5538         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5539
5540         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5541
5542         $conf = $snapshot;
5543     }
5544
5545     my $defaults = load_defaults();
5546
5547     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5548
5549     return PVE::Tools::cmd2string($cmd);
5550 }
5551
5552 sub vm_reset {
5553     my ($vmid, $skiplock) = @_;
5554
5555     PVE::QemuConfig->lock_config($vmid, sub {
5556
5557         my $conf = PVE::QemuConfig->load_config($vmid);
5558
5559         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5560
5561         mon_cmd($vmid, "system_reset");
5562     });
5563 }
5564
5565 sub get_vm_volumes {
5566     my ($conf) = @_;
5567
5568     my $vollist = [];
5569     foreach_volid($conf, sub {
5570         my ($volid, $attr) = @_;
5571
5572         return if $volid =~ m|^/|;
5573
5574         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5575         return if !$sid;
5576
5577         push @$vollist, $volid;
5578     });
5579
5580     return $vollist;
5581 }
5582
5583 sub vm_stop_cleanup {
5584     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5585
5586     eval {
5587
5588         if (!$keepActive) {
5589             my $vollist = get_vm_volumes($conf);
5590             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5591         }
5592
5593         foreach my $ext (qw(mon qmp pid vnc qga)) {
5594             unlink "/var/run/qemu-server/${vmid}.$ext";
5595         }
5596
5597         if ($conf->{ivshmem}) {
5598             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5599             # just delete it for now, VMs which have this already open do not
5600             # are affected, but new VMs will get a separated one. If this
5601             # becomes an issue we either add some sort of ref-counting or just
5602             # add a "don't delete on stop" flag to the ivshmem format.
5603             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5604         }
5605
5606         foreach my $key (keys %$conf) {
5607             next if $key !~ m/^hostpci(\d+)$/;
5608             my $hostpciindex = $1;
5609             my $d = parse_hostpci($conf->{$key});
5610             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5611
5612             foreach my $pci (@{$d->{pciid}}) {
5613                 my $pciid = $pci->{id};
5614                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5615             }
5616         }
5617
5618         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5619     };
5620     warn $@ if $@; # avoid errors - just warn
5621 }
5622
5623 # call only in locked context
5624 sub _do_vm_stop {
5625     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5626
5627     my $pid = check_running($vmid, $nocheck);
5628     return if !$pid;
5629
5630     my $conf;
5631     if (!$nocheck) {
5632         $conf = PVE::QemuConfig->load_config($vmid);
5633         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5634         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5635             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5636             $timeout = $opts->{down} if $opts->{down};
5637         }
5638         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5639     }
5640
5641     eval {
5642         if ($shutdown) {
5643             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5644                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5645             } else {
5646                 mon_cmd($vmid, "system_powerdown");
5647             }
5648         } else {
5649             mon_cmd($vmid, "quit");
5650         }
5651     };
5652     my $err = $@;
5653
5654     if (!$err) {
5655         $timeout = 60 if !defined($timeout);
5656
5657         my $count = 0;
5658         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5659             $count++;
5660             sleep 1;
5661         }
5662
5663         if ($count >= $timeout) {
5664             if ($force) {
5665                 warn "VM still running - terminating now with SIGTERM\n";
5666                 kill 15, $pid;
5667             } else {
5668                 die "VM quit/powerdown failed - got timeout\n";
5669             }
5670         } else {
5671             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5672             return;
5673         }
5674     } else {
5675         if ($force) {
5676             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5677             kill 15, $pid;
5678         } else {
5679             die "VM quit/powerdown failed\n";
5680         }
5681     }
5682
5683     # wait again
5684     $timeout = 10;
5685
5686     my $count = 0;
5687     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5688         $count++;
5689         sleep 1;
5690     }
5691
5692     if ($count >= $timeout) {
5693         warn "VM still running - terminating now with SIGKILL\n";
5694         kill 9, $pid;
5695         sleep 1;
5696     }
5697
5698     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5699 }
5700
5701 # Note: use $nocheck to skip tests if VM configuration file exists.
5702 # We need that when migration VMs to other nodes (files already moved)
5703 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5704 sub vm_stop {
5705     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5706
5707     $force = 1 if !defined($force) && !$shutdown;
5708
5709     if ($migratedfrom){
5710         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5711         kill 15, $pid if $pid;
5712         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5713         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5714         return;
5715     }
5716
5717     PVE::QemuConfig->lock_config($vmid, sub {
5718         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5719    });
5720 }
5721
5722 sub vm_reboot {
5723     my ($vmid, $timeout) = @_;
5724
5725     PVE::QemuConfig->lock_config($vmid, sub {
5726         eval {
5727
5728             # only reboot if running, as qmeventd starts it again on a stop event
5729             return if !check_running($vmid);
5730
5731             create_reboot_request($vmid);
5732
5733             my $storecfg = PVE::Storage::config();
5734             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5735
5736         };
5737         if (my $err = $@) {
5738             # avoid that the next normal shutdown will be confused for a reboot
5739             clear_reboot_request($vmid);
5740             die $err;
5741         }
5742    });
5743 }
5744
5745 sub vm_suspend {
5746     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5747
5748     my $conf;
5749     my $path;
5750     my $storecfg;
5751     my $vmstate;
5752
5753     PVE::QemuConfig->lock_config($vmid, sub {
5754
5755         $conf = PVE::QemuConfig->load_config($vmid);
5756
5757         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5758         PVE::QemuConfig->check_lock($conf)
5759             if !($skiplock || $is_backing_up);
5760
5761         die "cannot suspend to disk during backup\n"
5762             if $is_backing_up && $includestate;
5763
5764         if ($includestate) {
5765             $conf->{lock} = 'suspending';
5766             my $date = strftime("%Y-%m-%d", localtime(time()));
5767             $storecfg = PVE::Storage::config();
5768             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5769             $path = PVE::Storage::path($storecfg, $vmstate);
5770             PVE::QemuConfig->write_config($vmid, $conf);
5771         } else {
5772             mon_cmd($vmid, "stop");
5773         }
5774     });
5775
5776     if ($includestate) {
5777         # save vm state
5778         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5779
5780         eval {
5781             mon_cmd($vmid, "savevm-start", statefile => $path);
5782             for(;;) {
5783                 my $state = mon_cmd($vmid, "query-savevm");
5784                 if (!$state->{status}) {
5785                     die "savevm not active\n";
5786                 } elsif ($state->{status} eq 'active') {
5787                     sleep(1);
5788                     next;
5789                 } elsif ($state->{status} eq 'completed') {
5790                     print "State saved, quitting\n";
5791                     last;
5792                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5793                     die "query-savevm failed with error '$state->{error}'\n"
5794                 } else {
5795                     die "query-savevm returned status '$state->{status}'\n";
5796                 }
5797             }
5798         };
5799         my $err = $@;
5800
5801         PVE::QemuConfig->lock_config($vmid, sub {
5802             $conf = PVE::QemuConfig->load_config($vmid);
5803             if ($err) {
5804                 # cleanup, but leave suspending lock, to indicate something went wrong
5805                 eval {
5806                     mon_cmd($vmid, "savevm-end");
5807                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5808                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5809                     delete $conf->@{qw(vmstate runningmachine)};
5810                     PVE::QemuConfig->write_config($vmid, $conf);
5811                 };
5812                 warn $@ if $@;
5813                 die $err;
5814             }
5815
5816             die "lock changed unexpectedly\n"
5817                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5818
5819             mon_cmd($vmid, "quit");
5820             $conf->{lock} = 'suspended';
5821             PVE::QemuConfig->write_config($vmid, $conf);
5822         });
5823     }
5824 }
5825
5826 sub vm_resume {
5827     my ($vmid, $skiplock, $nocheck) = @_;
5828
5829     PVE::QemuConfig->lock_config($vmid, sub {
5830         my $res = mon_cmd($vmid, 'query-status');
5831         my $resume_cmd = 'cont';
5832
5833         if ($res->{status} && $res->{status} eq 'suspended') {
5834             $resume_cmd = 'system_wakeup';
5835         }
5836
5837         if (!$nocheck) {
5838
5839             my $conf = PVE::QemuConfig->load_config($vmid);
5840
5841             PVE::QemuConfig->check_lock($conf)
5842                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5843         }
5844
5845         mon_cmd($vmid, $resume_cmd);
5846     });
5847 }
5848
5849 sub vm_sendkey {
5850     my ($vmid, $skiplock, $key) = @_;
5851
5852     PVE::QemuConfig->lock_config($vmid, sub {
5853
5854         my $conf = PVE::QemuConfig->load_config($vmid);
5855
5856         # there is no qmp command, so we use the human monitor command
5857         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5858         die $res if $res ne '';
5859     });
5860 }
5861
5862 # vzdump restore implementaion
5863
5864 sub tar_archive_read_firstfile {
5865     my $archive = shift;
5866
5867     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5868
5869     # try to detect archive type first
5870     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5871         die "unable to open file '$archive'\n";
5872     my $firstfile = <$fh>;
5873     kill 15, $pid;
5874     close $fh;
5875
5876     die "ERROR: archive contaions no data\n" if !$firstfile;
5877     chomp $firstfile;
5878
5879     return $firstfile;
5880 }
5881
5882 sub tar_restore_cleanup {
5883     my ($storecfg, $statfile) = @_;
5884
5885     print STDERR "starting cleanup\n";
5886
5887     if (my $fd = IO::File->new($statfile, "r")) {
5888         while (defined(my $line = <$fd>)) {
5889             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5890                 my $volid = $2;
5891                 eval {
5892                     if ($volid =~ m|^/|) {
5893                         unlink $volid || die 'unlink failed\n';
5894                     } else {
5895                         PVE::Storage::vdisk_free($storecfg, $volid);
5896                     }
5897                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5898                 };
5899                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5900             } else {
5901                 print STDERR "unable to parse line in statfile - $line";
5902             }
5903         }
5904         $fd->close();
5905     }
5906 }
5907
5908 sub restore_archive {
5909     my ($archive, $vmid, $user, $opts) = @_;
5910
5911     my $format = $opts->{format};
5912     my $comp;
5913
5914     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5915         $format = 'tar' if !$format;
5916         $comp = 'gzip';
5917     } elsif ($archive =~ m/\.tar$/) {
5918         $format = 'tar' if !$format;
5919     } elsif ($archive =~ m/.tar.lzo$/) {
5920         $format = 'tar' if !$format;
5921         $comp = 'lzop';
5922     } elsif ($archive =~ m/\.vma$/) {
5923         $format = 'vma' if !$format;
5924     } elsif ($archive =~ m/\.vma\.gz$/) {
5925         $format = 'vma' if !$format;
5926         $comp = 'gzip';
5927     } elsif ($archive =~ m/\.vma\.lzo$/) {
5928         $format = 'vma' if !$format;
5929         $comp = 'lzop';
5930     } else {
5931         $format = 'vma' if !$format; # default
5932     }
5933
5934     # try to detect archive format
5935     if ($format eq 'tar') {
5936         return restore_tar_archive($archive, $vmid, $user, $opts);
5937     } else {
5938         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5939     }
5940 }
5941
5942 sub restore_update_config_line {
5943     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5944
5945     return if $line =~ m/^\#qmdump\#/;
5946     return if $line =~ m/^\#vzdump\#/;
5947     return if $line =~ m/^lock:/;
5948     return if $line =~ m/^unused\d+:/;
5949     return if $line =~ m/^parent:/;
5950
5951     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5952     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5953         # try to convert old 1.X settings
5954         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5955         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5956             my ($model, $macaddr) = split(/\=/, $devconfig);
5957             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5958             my $net = {
5959                 model => $model,
5960                 bridge => "vmbr$ind",
5961                 macaddr => $macaddr,
5962             };
5963             my $netstr = print_net($net);
5964
5965             print $outfd "net$cookie->{netcount}: $netstr\n";
5966             $cookie->{netcount}++;
5967         }
5968     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5969         my ($id, $netstr) = ($1, $2);
5970         my $net = parse_net($netstr);
5971         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5972         $netstr = print_net($net);
5973         print $outfd "$id: $netstr\n";
5974     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5975         my $virtdev = $1;
5976         my $value = $3;
5977         my $di = parse_drive($virtdev, $value);
5978         if (defined($di->{backup}) && !$di->{backup}) {
5979             print $outfd "#$line";
5980         } elsif ($map->{$virtdev}) {
5981             delete $di->{format}; # format can change on restore
5982             $di->{file} = $map->{$virtdev};
5983             $value = print_drive($di);
5984             print $outfd "$virtdev: $value\n";
5985         } else {
5986             print $outfd $line;
5987         }
5988     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5989         my $vmgenid = $1;
5990         if ($vmgenid ne '0') {
5991             # always generate a new vmgenid if there was a valid one setup
5992             $vmgenid = generate_uuid();
5993         }
5994         print $outfd "vmgenid: $vmgenid\n";
5995     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5996         my ($uuid, $uuid_str);
5997         UUID::generate($uuid);
5998         UUID::unparse($uuid, $uuid_str);
5999         my $smbios1 = parse_smbios1($2);
6000         $smbios1->{uuid} = $uuid_str;
6001         print $outfd $1.print_smbios1($smbios1)."\n";
6002     } else {
6003         print $outfd $line;
6004     }
6005 }
6006
6007 sub scan_volids {
6008     my ($cfg, $vmid) = @_;
6009
6010     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6011
6012     my $volid_hash = {};
6013     foreach my $storeid (keys %$info) {
6014         foreach my $item (@{$info->{$storeid}}) {
6015             next if !($item->{volid} && $item->{size});
6016             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6017             $volid_hash->{$item->{volid}} = $item;
6018         }
6019     }
6020
6021     return $volid_hash;
6022 }
6023
6024 sub is_volume_in_use {
6025     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6026
6027     my $path = PVE::Storage::path($storecfg, $volid);
6028
6029     my $scan_config = sub {
6030         my ($cref, $snapname) = @_;
6031
6032         foreach my $key (keys %$cref) {
6033             my $value = $cref->{$key};
6034             if (is_valid_drivename($key)) {
6035                 next if $skip_drive && $key eq $skip_drive;
6036                 my $drive = parse_drive($key, $value);
6037                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6038                 return 1 if $volid eq $drive->{file};
6039                 if ($drive->{file} =~ m!^/!) {
6040                     return 1 if $drive->{file} eq $path;
6041                 } else {
6042                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6043                     next if !$storeid;
6044                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6045                     next if !$scfg;
6046                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6047                 }
6048             }
6049         }
6050
6051         return 0;
6052     };
6053
6054     return 1 if &$scan_config($conf);
6055
6056     undef $skip_drive;
6057
6058     foreach my $snapname (keys %{$conf->{snapshots}}) {
6059         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6060     }
6061
6062     return 0;
6063 }
6064
6065 sub update_disksize {
6066     my ($vmid, $conf, $volid_hash) = @_;
6067
6068     my $changes;
6069     my $prefix = "VM $vmid:";
6070
6071     # used and unused disks
6072     my $referenced = {};
6073
6074     # Note: it is allowed to define multiple storages with same path (alias), so
6075     # we need to check both 'volid' and real 'path' (two different volid can point
6076     # to the same path).
6077
6078     my $referencedpath = {};
6079
6080     # update size info
6081     foreach my $opt (keys %$conf) {
6082         if (is_valid_drivename($opt)) {
6083             my $drive = parse_drive($opt, $conf->{$opt});
6084             my $volid = $drive->{file};
6085             next if !$volid;
6086
6087             $referenced->{$volid} = 1;
6088             if ($volid_hash->{$volid} &&
6089                 (my $path = $volid_hash->{$volid}->{path})) {
6090                 $referencedpath->{$path} = 1;
6091             }
6092
6093             next if drive_is_cdrom($drive);
6094             next if !$volid_hash->{$volid};
6095
6096             $drive->{size} = $volid_hash->{$volid}->{size};
6097             my $new = print_drive($drive);
6098             if ($new ne $conf->{$opt}) {
6099                 $changes = 1;
6100                 $conf->{$opt} = $new;
6101                 print "$prefix update disk '$opt' information.\n";
6102             }
6103         }
6104     }
6105
6106     # remove 'unusedX' entry if volume is used
6107     foreach my $opt (keys %$conf) {
6108         next if $opt !~ m/^unused\d+$/;
6109         my $volid = $conf->{$opt};
6110         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6111         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6112             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6113             $changes = 1;
6114             delete $conf->{$opt};
6115         }
6116
6117         $referenced->{$volid} = 1;
6118         $referencedpath->{$path} = 1 if $path;
6119     }
6120
6121     foreach my $volid (sort keys %$volid_hash) {
6122         next if $volid =~ m/vm-$vmid-state-/;
6123         next if $referenced->{$volid};
6124         my $path = $volid_hash->{$volid}->{path};
6125         next if !$path; # just to be sure
6126         next if $referencedpath->{$path};
6127         $changes = 1;
6128         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6129         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6130         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6131     }
6132
6133     return $changes;
6134 }
6135
6136 sub rescan {
6137     my ($vmid, $nolock, $dryrun) = @_;
6138
6139     my $cfg = PVE::Storage::config();
6140
6141     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6142     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6143     foreach my $stor (keys %{$cfg->{ids}}) {
6144         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6145     }
6146
6147     print "rescan volumes...\n";
6148     my $volid_hash = scan_volids($cfg, $vmid);
6149
6150     my $updatefn =  sub {
6151         my ($vmid) = @_;
6152
6153         my $conf = PVE::QemuConfig->load_config($vmid);
6154
6155         PVE::QemuConfig->check_lock($conf);
6156
6157         my $vm_volids = {};
6158         foreach my $volid (keys %$volid_hash) {
6159             my $info = $volid_hash->{$volid};
6160             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6161         }
6162
6163         my $changes = update_disksize($vmid, $conf, $vm_volids);
6164
6165         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6166     };
6167
6168     if (defined($vmid)) {
6169         if ($nolock) {
6170             &$updatefn($vmid);
6171         } else {
6172             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6173         }
6174     } else {
6175         my $vmlist = config_list();
6176         foreach my $vmid (keys %$vmlist) {
6177             if ($nolock) {
6178                 &$updatefn($vmid);
6179             } else {
6180                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6181             }
6182         }
6183     }
6184 }
6185
6186 sub restore_vma_archive {
6187     my ($archive, $vmid, $user, $opts, $comp) = @_;
6188
6189     my $readfrom = $archive;
6190
6191     my $cfg = PVE::Storage::config();
6192     my $commands = [];
6193     my $bwlimit = $opts->{bwlimit};
6194
6195     my $dbg_cmdstring = '';
6196     my $add_pipe = sub {
6197         my ($cmd) = @_;
6198         push @$commands, $cmd;
6199         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6200         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6201         $readfrom = '-';
6202     };
6203
6204     my $input = undef;
6205     if ($archive eq '-') {
6206         $input = '<&STDIN';
6207     } else {
6208         # If we use a backup from a PVE defined storage we also consider that
6209         # storage's rate limit:
6210         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6211         if (defined($volid)) {
6212             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6213             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6214             if ($readlimit) {
6215                 print STDERR "applying read rate limit: $readlimit\n";
6216                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6217                 $add_pipe->($cstream);
6218             }
6219         }
6220     }
6221
6222     if ($comp) {
6223         my $cmd;
6224         if ($comp eq 'gzip') {
6225             $cmd = ['zcat', $readfrom];
6226         } elsif ($comp eq 'lzop') {
6227             $cmd = ['lzop', '-d', '-c', $readfrom];
6228         } else {
6229             die "unknown compression method '$comp'\n";
6230         }
6231         $add_pipe->($cmd);
6232     }
6233
6234     my $tmpdir = "/var/tmp/vzdumptmp$$";
6235     rmtree $tmpdir;
6236
6237     # disable interrupts (always do cleanups)
6238     local $SIG{INT} =
6239         local $SIG{TERM} =
6240         local $SIG{QUIT} =
6241         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6242
6243     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6244     POSIX::mkfifo($mapfifo, 0600);
6245     my $fifofh;
6246
6247     my $openfifo = sub {
6248         open($fifofh, '>', $mapfifo) || die $!;
6249     };
6250
6251     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6252
6253     my $oldtimeout;
6254     my $timeout = 5;
6255
6256     my $devinfo = {};
6257
6258     my $rpcenv = PVE::RPCEnvironment::get();
6259
6260     my $conffile = PVE::QemuConfig->config_file($vmid);
6261     my $tmpfn = "$conffile.$$.tmp";
6262
6263     # Note: $oldconf is undef if VM does not exists
6264     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6265     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6266
6267     my %storage_limits;
6268
6269     my $print_devmap = sub {
6270         my $virtdev_hash = {};
6271
6272         my $cfgfn = "$tmpdir/qemu-server.conf";
6273
6274         # we can read the config - that is already extracted
6275         my $fh = IO::File->new($cfgfn, "r") ||
6276             "unable to read qemu-server.conf - $!\n";
6277
6278         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6279         if (-f $fwcfgfn) {
6280             my $pve_firewall_dir = '/etc/pve/firewall';
6281             mkdir $pve_firewall_dir; # make sure the dir exists
6282             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6283         }
6284
6285         while (defined(my $line = <$fh>)) {
6286             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6287                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6288                 die "archive does not contain data for drive '$virtdev'\n"
6289                     if !$devinfo->{$devname};
6290                 if (defined($opts->{storage})) {
6291                     $storeid = $opts->{storage} || 'local';
6292                 } elsif (!$storeid) {
6293                     $storeid = 'local';
6294                 }
6295                 $format = 'raw' if !$format;
6296                 $devinfo->{$devname}->{devname} = $devname;
6297                 $devinfo->{$devname}->{virtdev} = $virtdev;
6298                 $devinfo->{$devname}->{format} = $format;
6299                 $devinfo->{$devname}->{storeid} = $storeid;
6300
6301                 # check permission on storage
6302                 my $pool = $opts->{pool}; # todo: do we need that?
6303                 if ($user ne 'root@pam') {
6304                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6305                 }
6306
6307                 $storage_limits{$storeid} = $bwlimit;
6308
6309                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6310             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6311                 my $virtdev = $1;
6312                 my $drive = parse_drive($virtdev, $2);
6313                 if (drive_is_cloudinit($drive)) {
6314                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6315                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6316                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6317
6318                     my $d = {
6319                         format => $format,
6320                         storeid => $opts->{storage} // $storeid,
6321                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6322                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6323                         name => "vm-$vmid-cloudinit",
6324                         is_cloudinit => 1,
6325                     };
6326                     $virtdev_hash->{$virtdev} = $d;
6327                 }
6328             }
6329         }
6330
6331         foreach my $key (keys %storage_limits) {
6332             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6333             next if !$limit;
6334             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6335             $storage_limits{$key} = $limit * 1024;
6336         }
6337
6338         foreach my $devname (keys %$devinfo) {
6339             die "found no device mapping information for device '$devname'\n"
6340                 if !$devinfo->{$devname}->{virtdev};
6341         }
6342
6343         # create empty/temp config
6344         if ($oldconf) {
6345             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6346             foreach_drive($oldconf, sub {
6347                 my ($ds, $drive) = @_;
6348
6349                 return if drive_is_cdrom($drive, 1);
6350
6351                 my $volid = $drive->{file};
6352                 return if !$volid || $volid =~ m|^/|;
6353
6354                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6355                 return if !$path || !$owner || ($owner != $vmid);
6356
6357                 # Note: only delete disk we want to restore
6358                 # other volumes will become unused
6359                 if ($virtdev_hash->{$ds}) {
6360                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6361                     if (my $err = $@) {
6362                         warn $err;
6363                     }
6364                 }
6365             });
6366
6367             # delete vmstate files, after the restore we have no snapshots anymore
6368             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6369                 my $snap = $oldconf->{snapshots}->{$snapname};
6370                 if ($snap->{vmstate}) {
6371                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6372                     if (my $err = $@) {
6373                         warn $err;
6374                     }
6375                 }
6376             }
6377         }
6378
6379         my $map = {};
6380         foreach my $virtdev (sort keys %$virtdev_hash) {
6381             my $d = $virtdev_hash->{$virtdev};
6382             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6383             my $storeid = $d->{storeid};
6384             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6385
6386             my $map_opts = '';
6387             if (my $limit = $storage_limits{$storeid}) {
6388                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6389             }
6390
6391             # test if requested format is supported
6392             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6393             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6394             $d->{format} = $defFormat if !$supported;
6395
6396             my $name;
6397             if ($d->{is_cloudinit}) {
6398                 $name = $d->{name};
6399                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6400             }
6401
6402             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6403             print STDERR "new volume ID is '$volid'\n";
6404             $d->{volid} = $volid;
6405
6406             PVE::Storage::activate_volumes($cfg, [$volid]);
6407
6408             my $write_zeros = 1;
6409             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6410                 $write_zeros = 0;
6411             }
6412
6413             if (!$d->{is_cloudinit}) {
6414                 my $path = PVE::Storage::path($cfg, $volid);
6415
6416                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6417
6418                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6419             }
6420             $map->{$virtdev} = $volid;
6421         }
6422
6423         $fh->seek(0, 0) || die "seek failed - $!\n";
6424
6425         my $outfd = new IO::File ($tmpfn, "w") ||
6426             die "unable to write config for VM $vmid\n";
6427
6428         my $cookie = { netcount => 0 };
6429         while (defined(my $line = <$fh>)) {
6430             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6431         }
6432
6433         $fh->close();
6434         $outfd->close();
6435     };
6436
6437     eval {
6438         # enable interrupts
6439         local $SIG{INT} =
6440             local $SIG{TERM} =
6441             local $SIG{QUIT} =
6442             local $SIG{HUP} =
6443             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6444         local $SIG{ALRM} = sub { die "got timeout\n"; };
6445
6446         $oldtimeout = alarm($timeout);
6447
6448         my $parser = sub {
6449             my $line = shift;
6450
6451             print "$line\n";
6452
6453             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6454                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6455                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6456             } elsif ($line =~ m/^CTIME: /) {
6457                 # we correctly received the vma config, so we can disable
6458                 # the timeout now for disk allocation (set to 10 minutes, so
6459                 # that we always timeout if something goes wrong)
6460                 alarm(600);
6461                 &$print_devmap();
6462                 print $fifofh "done\n";
6463                 my $tmp = $oldtimeout || 0;
6464                 $oldtimeout = undef;
6465                 alarm($tmp);
6466                 close($fifofh);
6467             }
6468         };
6469
6470         print "restore vma archive: $dbg_cmdstring\n";
6471         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6472     };
6473     my $err = $@;
6474
6475     alarm($oldtimeout) if $oldtimeout;
6476
6477     my $vollist = [];
6478     foreach my $devname (keys %$devinfo) {
6479         my $volid = $devinfo->{$devname}->{volid};
6480         push @$vollist, $volid if $volid;
6481     }
6482
6483     PVE::Storage::deactivate_volumes($cfg, $vollist);
6484
6485     unlink $mapfifo;
6486
6487     if ($err) {
6488         rmtree $tmpdir;
6489         unlink $tmpfn;
6490
6491         foreach my $devname (keys %$devinfo) {
6492             my $volid = $devinfo->{$devname}->{volid};
6493             next if !$volid;
6494             eval {
6495                 if ($volid =~ m|^/|) {
6496                     unlink $volid || die 'unlink failed\n';
6497                 } else {
6498                     PVE::Storage::vdisk_free($cfg, $volid);
6499                 }
6500                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6501             };
6502             print STDERR "unable to cleanup '$volid' - $@" if $@;
6503         }
6504         die $err;
6505     }
6506
6507     rmtree $tmpdir;
6508
6509     rename($tmpfn, $conffile) ||
6510         die "unable to commit configuration file '$conffile'\n";
6511
6512     PVE::Cluster::cfs_update(); # make sure we read new file
6513
6514     eval { rescan($vmid, 1); };
6515     warn $@ if $@;
6516 }
6517
6518 sub restore_tar_archive {
6519     my ($archive, $vmid, $user, $opts) = @_;
6520
6521     if ($archive ne '-') {
6522         my $firstfile = tar_archive_read_firstfile($archive);
6523         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6524             if $firstfile ne 'qemu-server.conf';
6525     }
6526
6527     my $storecfg = PVE::Storage::config();
6528
6529     # avoid zombie disks when restoring over an existing VM -> cleanup first
6530     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6531     # skiplock=1 because qmrestore has set the 'create' lock itself already
6532     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6533     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6534
6535     my $tocmd = "/usr/lib/qemu-server/qmextract";
6536
6537     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6538     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6539     $tocmd .= ' --prealloc' if $opts->{prealloc};
6540     $tocmd .= ' --info' if $opts->{info};
6541
6542     # tar option "xf" does not autodetect compression when read from STDIN,
6543     # so we pipe to zcat
6544     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6545         PVE::Tools::shellquote("--to-command=$tocmd");
6546
6547     my $tmpdir = "/var/tmp/vzdumptmp$$";
6548     mkpath $tmpdir;
6549
6550     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6551     local $ENV{VZDUMP_VMID} = $vmid;
6552     local $ENV{VZDUMP_USER} = $user;
6553
6554     my $conffile = PVE::QemuConfig->config_file($vmid);
6555     my $tmpfn = "$conffile.$$.tmp";
6556
6557     # disable interrupts (always do cleanups)
6558     local $SIG{INT} =
6559         local $SIG{TERM} =
6560         local $SIG{QUIT} =
6561         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6562
6563     eval {
6564         # enable interrupts
6565         local $SIG{INT} =
6566             local $SIG{TERM} =
6567             local $SIG{QUIT} =
6568             local $SIG{HUP} =
6569             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6570
6571         if ($archive eq '-') {
6572             print "extracting archive from STDIN\n";
6573             run_command($cmd, input => "<&STDIN");
6574         } else {
6575             print "extracting archive '$archive'\n";
6576             run_command($cmd);
6577         }
6578
6579         return if $opts->{info};
6580
6581         # read new mapping
6582         my $map = {};
6583         my $statfile = "$tmpdir/qmrestore.stat";
6584         if (my $fd = IO::File->new($statfile, "r")) {
6585             while (defined (my $line = <$fd>)) {
6586                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6587                     $map->{$1} = $2 if $1;
6588                 } else {
6589                     print STDERR "unable to parse line in statfile - $line\n";
6590                 }
6591             }
6592             $fd->close();
6593         }
6594
6595         my $confsrc = "$tmpdir/qemu-server.conf";
6596
6597         my $srcfd = new IO::File($confsrc, "r") ||
6598             die "unable to open file '$confsrc'\n";
6599
6600         my $outfd = new IO::File ($tmpfn, "w") ||
6601             die "unable to write config for VM $vmid\n";
6602
6603         my $cookie = { netcount => 0 };
6604         while (defined (my $line = <$srcfd>)) {
6605             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6606         }
6607
6608         $srcfd->close();
6609         $outfd->close();
6610     };
6611     if (my $err = $@) {
6612         unlink $tmpfn;
6613         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6614         die $err;
6615     }
6616
6617     rmtree $tmpdir;
6618
6619     rename $tmpfn, $conffile ||
6620         die "unable to commit configuration file '$conffile'\n";
6621
6622     PVE::Cluster::cfs_update(); # make sure we read new file
6623
6624     eval { rescan($vmid, 1); };
6625     warn $@ if $@;
6626 };
6627
6628 sub foreach_storage_used_by_vm {
6629     my ($conf, $func) = @_;
6630
6631     my $sidhash = {};
6632
6633     foreach_drive($conf, sub {
6634         my ($ds, $drive) = @_;
6635         return if drive_is_cdrom($drive);
6636
6637         my $volid = $drive->{file};
6638
6639         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6640         $sidhash->{$sid} = $sid if $sid;
6641     });
6642
6643     foreach my $sid (sort keys %$sidhash) {
6644         &$func($sid);
6645     }
6646 }
6647
6648 my $qemu_snap_storage = {
6649     rbd => 1,
6650 };
6651 sub do_snapshots_with_qemu {
6652     my ($storecfg, $volid) = @_;
6653
6654     my $storage_name = PVE::Storage::parse_volume_id($volid);
6655     my $scfg = $storecfg->{ids}->{$storage_name};
6656
6657     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6658         return 1;
6659     }
6660
6661     if ($volid =~ m/\.(qcow2|qed)$/){
6662         return 1;
6663     }
6664
6665     return undef;
6666 }
6667
6668 sub qga_check_running {
6669     my ($vmid, $nowarn) = @_;
6670
6671     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6672     if ($@) {
6673         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6674         return 0;
6675     }
6676     return 1;
6677 }
6678
6679 sub template_create {
6680     my ($vmid, $conf, $disk) = @_;
6681
6682     my $storecfg = PVE::Storage::config();
6683
6684     foreach_drive($conf, sub {
6685         my ($ds, $drive) = @_;
6686
6687         return if drive_is_cdrom($drive);
6688         return if $disk && $ds ne $disk;
6689
6690         my $volid = $drive->{file};
6691         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6692
6693         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6694         $drive->{file} = $voliddst;
6695         $conf->{$ds} = print_drive($drive);
6696         PVE::QemuConfig->write_config($vmid, $conf);
6697     });
6698 }
6699
6700 sub convert_iscsi_path {
6701     my ($path) = @_;
6702
6703     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6704         my $portal = $1;
6705         my $target = $2;
6706         my $lun = $3;
6707
6708         my $initiator_name = get_initiator_name();
6709
6710         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6711                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6712     }
6713
6714     die "cannot convert iscsi path '$path', unkown format\n";
6715 }
6716
6717 sub qemu_img_convert {
6718     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6719
6720     my $storecfg = PVE::Storage::config();
6721     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6722     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6723
6724     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6725
6726     my $cachemode;
6727     my $src_path;
6728     my $src_is_iscsi = 0;
6729     my $src_format;
6730
6731     if ($src_storeid) {
6732         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6733         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6734         $src_format = qemu_img_format($src_scfg, $src_volname);
6735         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6736         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6737         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6738     } elsif (-f $src_volid) {
6739         $src_path = $src_volid;
6740         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6741             $src_format = $1;
6742         }
6743     }
6744
6745     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6746
6747     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6748     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6749     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6750     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6751
6752     my $cmd = [];
6753     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6754     push @$cmd, '-l', "snapshot.name=$snapname"
6755         if $snapname && $src_format && $src_format eq "qcow2";
6756     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6757     push @$cmd, '-T', $cachemode if defined($cachemode);
6758
6759     if ($src_is_iscsi) {
6760         push @$cmd, '--image-opts';
6761         $src_path = convert_iscsi_path($src_path);
6762     } elsif ($src_format) {
6763         push @$cmd, '-f', $src_format;
6764     }
6765
6766     if ($dst_is_iscsi) {
6767         push @$cmd, '--target-image-opts';
6768         $dst_path = convert_iscsi_path($dst_path);
6769     } else {
6770         push @$cmd, '-O', $dst_format;
6771     }
6772
6773     push @$cmd, $src_path;
6774
6775     if (!$dst_is_iscsi && $is_zero_initialized) {
6776         push @$cmd, "zeroinit:$dst_path";
6777     } else {
6778         push @$cmd, $dst_path;
6779     }
6780
6781     my $parser = sub {
6782         my $line = shift;
6783         if($line =~ m/\((\S+)\/100\%\)/){
6784             my $percent = $1;
6785             my $transferred = int($size * $percent / 100);
6786             my $remaining = $size - $transferred;
6787
6788             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6789         }
6790
6791     };
6792
6793     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6794     my $err = $@;
6795     die "copy failed: $err" if $err;
6796 }
6797
6798 sub qemu_img_format {
6799     my ($scfg, $volname) = @_;
6800
6801     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6802         return $1;
6803     } else {
6804         return "raw";
6805     }
6806 }
6807
6808 sub qemu_drive_mirror {
6809     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6810
6811     $jobs = {} if !$jobs;
6812
6813     my $qemu_target;
6814     my $format;
6815     $jobs->{"drive-$drive"} = {};
6816
6817     if ($dst_volid =~ /^nbd:/) {
6818         $qemu_target = $dst_volid;
6819         $format = "nbd";
6820     } else {
6821         my $storecfg = PVE::Storage::config();
6822         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6823
6824         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6825
6826         $format = qemu_img_format($dst_scfg, $dst_volname);
6827
6828         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6829
6830         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6831     }
6832
6833     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6834     $opts->{format} = $format if $format;
6835
6836     if (defined($bwlimit)) {
6837         $opts->{speed} = $bwlimit * 1024;
6838         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6839     } else {
6840         print "drive mirror is starting for drive-$drive\n";
6841     }
6842
6843     # if a job already runs for this device we get an error, catch it for cleanup
6844     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6845     if (my $err = $@) {
6846         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6847         warn "$@\n" if $@;
6848         die "mirroring error: $err\n";
6849     }
6850
6851     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6852 }
6853
6854 sub qemu_drive_mirror_monitor {
6855     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6856
6857     eval {
6858         my $err_complete = 0;
6859
6860         while (1) {
6861             die "storage migration timed out\n" if $err_complete > 300;
6862
6863             my $stats = mon_cmd($vmid, "query-block-jobs");
6864
6865             my $running_mirror_jobs = {};
6866             foreach my $stat (@$stats) {
6867                 next if $stat->{type} ne 'mirror';
6868                 $running_mirror_jobs->{$stat->{device}} = $stat;
6869             }
6870
6871             my $readycounter = 0;
6872
6873             foreach my $job (keys %$jobs) {
6874
6875                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6876                     print "$job : finished\n";
6877                     delete $jobs->{$job};
6878                     next;
6879                 }
6880
6881                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6882
6883                 my $busy = $running_mirror_jobs->{$job}->{busy};
6884                 my $ready = $running_mirror_jobs->{$job}->{ready};
6885                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6886                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6887                     my $remaining = $total - $transferred;
6888                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6889
6890                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6891                 }
6892
6893                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6894             }
6895
6896             last if scalar(keys %$jobs) == 0;
6897
6898             if ($readycounter == scalar(keys %$jobs)) {
6899                 print "all mirroring jobs are ready \n";
6900                 last if $skipcomplete; #do the complete later
6901
6902                 if ($vmiddst && $vmiddst != $vmid) {
6903                     my $agent_running = $qga && qga_check_running($vmid);
6904                     if ($agent_running) {
6905                         print "freeze filesystem\n";
6906                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6907                     } else {
6908                         print "suspend vm\n";
6909                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6910                     }
6911
6912                     # if we clone a disk for a new target vm, we don't switch the disk
6913                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6914
6915                     if ($agent_running) {
6916                         print "unfreeze filesystem\n";
6917                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6918                     } else {
6919                         print "resume vm\n";
6920                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6921                     }
6922
6923                     last;
6924                 } else {
6925
6926                     foreach my $job (keys %$jobs) {
6927                         # try to switch the disk if source and destination are on the same guest
6928                         print "$job: Completing block job...\n";
6929
6930                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6931                         if ($@ =~ m/cannot be completed/) {
6932                             print "$job: Block job cannot be completed, try again.\n";
6933                             $err_complete++;
6934                         }else {
6935                             print "$job: Completed successfully.\n";
6936                             $jobs->{$job}->{complete} = 1;
6937                         }
6938                     }
6939                 }
6940             }
6941             sleep 1;
6942         }
6943     };
6944     my $err = $@;
6945
6946     if ($err) {
6947         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6948         die "mirroring error: $err";
6949     }
6950
6951 }
6952
6953 sub qemu_blockjobs_cancel {
6954     my ($vmid, $jobs) = @_;
6955
6956     foreach my $job (keys %$jobs) {
6957         print "$job: Cancelling block job\n";
6958         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6959         $jobs->{$job}->{cancel} = 1;
6960     }
6961
6962     while (1) {
6963         my $stats = mon_cmd($vmid, "query-block-jobs");
6964
6965         my $running_jobs = {};
6966         foreach my $stat (@$stats) {
6967             $running_jobs->{$stat->{device}} = $stat;
6968         }
6969
6970         foreach my $job (keys %$jobs) {
6971
6972             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6973                 print "$job: Done.\n";
6974                 delete $jobs->{$job};
6975             }
6976         }
6977
6978         last if scalar(keys %$jobs) == 0;
6979
6980         sleep 1;
6981     }
6982 }
6983
6984 sub clone_disk {
6985     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6986         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6987
6988     my $newvolid;
6989
6990     if (!$full) {
6991         print "create linked clone of drive $drivename ($drive->{file})\n";
6992         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6993         push @$newvollist, $newvolid;
6994     } else {
6995
6996         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6997         $storeid = $storage if $storage;
6998
6999         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7000         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7001
7002         print "create full clone of drive $drivename ($drive->{file})\n";
7003         my $name = undef;
7004         if (drive_is_cloudinit($drive)) {
7005             $name = "vm-$newvmid-cloudinit";
7006             $name .= ".$dst_format" if $dst_format ne 'raw';
7007             $snapname = undef;
7008             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7009         }
7010         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7011         push @$newvollist, $newvolid;
7012
7013         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7014
7015         if (drive_is_cloudinit($drive)) {
7016             goto no_data_clone;
7017         }
7018
7019         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7020         if (!$running || $snapname) {
7021             # TODO: handle bwlimits
7022             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7023         } else {
7024
7025             my $kvmver = get_running_qemu_version ($vmid);
7026             if (!min_version($kvmver, 2, 7)) {
7027                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7028                     if $drive->{iothread};
7029             }
7030
7031             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7032         }
7033     }
7034
7035 no_data_clone:
7036     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7037
7038     my $disk = $drive;
7039     $disk->{format} = undef;
7040     $disk->{file} = $newvolid;
7041     $disk->{size} = $size;
7042
7043     return $disk;
7044 }
7045
7046 sub get_running_qemu_version {
7047     my ($vmid) = @_;
7048     my $res = mon_cmd($vmid, "query-version");
7049     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7050 }
7051
7052 sub qemu_use_old_bios_files {
7053     my ($machine_type) = @_;
7054
7055     return if !$machine_type;
7056
7057     my $use_old_bios_files = undef;
7058
7059     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7060         $machine_type = $1;
7061         $use_old_bios_files = 1;
7062     } else {
7063         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7064         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7065         # load new efi bios files on migration. So this hack is required to allow
7066         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7067         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7068         $use_old_bios_files = !min_version($version, 2, 4);
7069     }
7070
7071     return ($use_old_bios_files, $machine_type);
7072 }
7073
7074 sub create_efidisk($$$$$) {
7075     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7076
7077     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7078     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7079
7080     my $vars_size_b = -s $ovmf_vars;
7081     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7082     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7083     PVE::Storage::activate_volumes($storecfg, [$volid]);
7084
7085     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7086     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7087
7088     return ($volid, $size/1024);
7089 }
7090
7091 sub vm_iothreads_list {
7092     my ($vmid) = @_;
7093
7094     my $res = mon_cmd($vmid, 'query-iothreads');
7095
7096     my $iothreads = {};
7097     foreach my $iothread (@$res) {
7098         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7099     }
7100
7101     return $iothreads;
7102 }
7103
7104 sub scsihw_infos {
7105     my ($conf, $drive) = @_;
7106
7107     my $maxdev = 0;
7108
7109     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7110         $maxdev = 7;
7111     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7112         $maxdev = 1;
7113     } else {
7114         $maxdev = 256;
7115     }
7116
7117     my $controller = int($drive->{index} / $maxdev);
7118     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7119
7120     return ($maxdev, $controller, $controller_prefix);
7121 }
7122
7123 sub add_hyperv_enlightenments {
7124     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7125
7126     return if $winversion < 6;
7127     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7128
7129     if ($gpu_passthrough || defined($hv_vendor_id)) {
7130         $hv_vendor_id //= 'proxmox';
7131         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7132     }
7133
7134     if (min_version($machine_version, 2, 3)) {
7135         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7136         push @$cpuFlags , 'hv_vapic';
7137         push @$cpuFlags , 'hv_time';
7138     } else {
7139         push @$cpuFlags , 'hv_spinlocks=0xffff';
7140     }
7141
7142     if (min_version($machine_version, 2, 6)) {
7143         push @$cpuFlags , 'hv_reset';
7144         push @$cpuFlags , 'hv_vpindex';
7145         push @$cpuFlags , 'hv_runtime';
7146     }
7147
7148     if ($winversion >= 7) {
7149         push @$cpuFlags , 'hv_relaxed';
7150
7151         if (min_version($machine_version, 2, 12)) {
7152             push @$cpuFlags , 'hv_synic';
7153             push @$cpuFlags , 'hv_stimer';
7154         }
7155
7156         if (min_version($machine_version, 3, 1)) {
7157             push @$cpuFlags , 'hv_ipi';
7158         }
7159     }
7160 }
7161
7162 sub windows_version {
7163     my ($ostype) = @_;
7164
7165     return 0 if !$ostype;
7166
7167     my $winversion = 0;
7168
7169     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7170         $winversion = 5;
7171     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7172         $winversion = 6;
7173     } elsif ($ostype =~ m/^win(\d+)$/) {
7174         $winversion = $1;
7175     }
7176
7177     return $winversion;
7178 }
7179
7180 sub resolve_dst_disk_format {
7181         my ($storecfg, $storeid, $src_volname, $format) = @_;
7182         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7183
7184         if (!$format) {
7185             # if no target format is specified, use the source disk format as hint
7186             if ($src_volname) {
7187                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7188                 $format = qemu_img_format($scfg, $src_volname);
7189             } else {
7190                 return $defFormat;
7191             }
7192         }
7193
7194         # test if requested format is supported - else use default
7195         my $supported = grep { $_ eq $format } @$validFormats;
7196         $format = $defFormat if !$supported;
7197         return $format;
7198 }
7199
7200 sub resolve_first_disk {
7201     my $conf = shift;
7202     my @disks = PVE::QemuServer::valid_drive_names();
7203     my $firstdisk;
7204     foreach my $ds (reverse @disks) {
7205         next if !$conf->{$ds};
7206         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7207         next if PVE::QemuServer::drive_is_cdrom($disk);
7208         $firstdisk = $ds;
7209     }
7210     return $firstdisk;
7211 }
7212
7213 sub generate_uuid {
7214     my ($uuid, $uuid_str);
7215     UUID::generate($uuid);
7216     UUID::unparse($uuid, $uuid_str);
7217     return $uuid_str;
7218 }
7219
7220 sub generate_smbios1_uuid {
7221     return "uuid=".generate_uuid();
7222 }
7223
7224 sub nbd_stop {
7225     my ($vmid) = @_;
7226
7227     mon_cmd($vmid, 'nbd-server-stop');
7228 }
7229
7230 sub create_reboot_request {
7231     my ($vmid) = @_;
7232     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7233         or die "failed to create reboot trigger file: $!\n";
7234     close($fh);
7235 }
7236
7237 sub clear_reboot_request {
7238     my ($vmid) = @_;
7239     my $path = "/run/qemu-server/$vmid.reboot";
7240     my $res = 0;
7241
7242     $res = unlink($path);
7243     die "could not remove reboot request for $vmid: $!"
7244         if !$res && $! != POSIX::ENOENT;
7245
7246     return $res;
7247 }
7248
7249 # bash completion helper
7250
7251 sub complete_backup_archives {
7252     my ($cmdname, $pname, $cvalue) = @_;
7253
7254     my $cfg = PVE::Storage::config();
7255
7256     my $storeid;
7257
7258     if ($cvalue =~ m/^([^:]+):/) {
7259         $storeid = $1;
7260     }
7261
7262     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7263
7264     my $res = [];
7265     foreach my $id (keys %$data) {
7266         foreach my $item (@{$data->{$id}}) {
7267             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7268             push @$res, $item->{volid} if defined($item->{volid});
7269         }
7270     }
7271
7272     return $res;
7273 }
7274
7275 my $complete_vmid_full = sub {
7276     my ($running) = @_;
7277
7278     my $idlist = vmstatus();
7279
7280     my $res = [];
7281
7282     foreach my $id (keys %$idlist) {
7283         my $d = $idlist->{$id};
7284         if (defined($running)) {
7285             next if $d->{template};
7286             next if $running && $d->{status} ne 'running';
7287             next if !$running && $d->{status} eq 'running';
7288         }
7289         push @$res, $id;
7290
7291     }
7292     return $res;
7293 };
7294
7295 sub complete_vmid {
7296     return &$complete_vmid_full();
7297 }
7298
7299 sub complete_vmid_stopped {
7300     return &$complete_vmid_full(0);
7301 }
7302
7303 sub complete_vmid_running {
7304     return &$complete_vmid_full(1);
7305 }
7306
7307 sub complete_storage {
7308
7309     my $cfg = PVE::Storage::config();
7310     my $ids = $cfg->{ids};
7311
7312     my $res = [];
7313     foreach my $sid (keys %$ids) {
7314         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7315         next if !$ids->{$sid}->{content}->{images};
7316         push @$res, $sid;
7317     }
7318
7319     return $res;
7320 }
7321
7322 sub complete_migration_storage {
7323     my ($cmd, $param, $current_value, $all_args) = @_;
7324
7325     my $targetnode = @$all_args[1];
7326
7327     my $cfg = PVE::Storage::config();
7328     my $ids = $cfg->{ids};
7329
7330     my $res = [];
7331     foreach my $sid (keys %$ids) {
7332         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7333         next if !$ids->{$sid}->{content}->{images};
7334         push @$res, $sid;
7335     }
7336
7337     return $res;
7338 }
7339
7340 1;