PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::DataCenterConfig;
  31 use PVE::Exception qw(raise raise_param_exc);
  32 use PVE::GuestHelpers;
  33 use PVE::INotify;
  34 use PVE::JSONSchema qw(get_standard_option);
  35 use PVE::ProcFSTools;
  36 use PVE::RPCEnvironment;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Helpers qw(min_version);
  45 use PVE::QemuServer::Cloudinit;
  46 use PVE::QemuServer::Machine;
  47 use PVE::QemuServer::Memory;
  48 use PVE::QemuServer::Monitor qw(mon_cmd);
  49 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  50 use PVE::QemuServer::USB qw(parse_usb_device);
  51
  52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  53 my $OVMF = {
  54     x86_64 => [
  55         "$EDK2_FW_BASE/OVMF_CODE.fd",
  56         "$EDK2_FW_BASE/OVMF_VARS.fd"
  57     ],
  58     aarch64 => [
  59         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  60         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  61     ],
  62 };
  63
  64 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  65
  66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  67
  68 # Note about locking: we use flock on the config file protect
  69 # against concurent actions.
  70 # Aditionaly, we have a 'lock' setting in the config file. This
  71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  72 # allowed when such lock is set. But you can ignore this kind of
  73 # lock with the --skiplock flag.
  74
  75 cfs_register_file('/qemu-server/',
  76                   \&parse_vm_config,
  77                   \&write_vm_config);
  78
  79 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  80     description => "Some command save/restore state from this location.",
  81     type => 'string',
  82     maxLength => 128,
  83     optional => 1,
  84 });
  85
  86 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  87     type => 'string',
  88     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  89     description => "The drive's backing file's data format.",
  90     optional => 1,
  91 });
  92
  93 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  94         description => "Specifies the Qemu machine type.",
  95         type => 'string',
  96         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  97         maxLength => 40,
  98         optional => 1,
  99 });
 100
 101 #no warnings 'redefine';
 102
 103 sub cgroups_write {
 104    my ($controller, $vmid, $option, $value) = @_;
 105
 106    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 107    PVE::ProcFSTools::write_proc_entry($path, $value);
 108
 109 }
 110
 111 my $nodename_cache;
 112 sub nodename {
 113     $nodename_cache //= PVE::INotify::nodename();
 114     return $nodename_cache;
 115 }
 116
 117 my $cpu_vendor_list = {
 118     # Intel CPUs
 119     486 => 'GenuineIntel',
 120     pentium => 'GenuineIntel',
 121     pentium2  => 'GenuineIntel',
 122     pentium3  => 'GenuineIntel',
 123     coreduo => 'GenuineIntel',
 124     core2duo => 'GenuineIntel',
 125     Conroe  => 'GenuineIntel',
 126     Penryn  => 'GenuineIntel',
 127     Nehalem  => 'GenuineIntel',
 128     'Nehalem-IBRS'  => 'GenuineIntel',
 129     Westmere => 'GenuineIntel',
 130     'Westmere-IBRS' => 'GenuineIntel',
 131     SandyBridge => 'GenuineIntel',
 132     'SandyBridge-IBRS' => 'GenuineIntel',
 133     IvyBridge => 'GenuineIntel',
 134     'IvyBridge-IBRS' => 'GenuineIntel',
 135     Haswell => 'GenuineIntel',
 136     'Haswell-IBRS' => 'GenuineIntel',
 137     'Haswell-noTSX' => 'GenuineIntel',
 138     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 139     Broadwell => 'GenuineIntel',
 140     'Broadwell-IBRS' => 'GenuineIntel',
 141     'Broadwell-noTSX' => 'GenuineIntel',
 142     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 143     'Skylake-Client' => 'GenuineIntel',
 144     'Skylake-Client-IBRS' => 'GenuineIntel',
 145     'Skylake-Server' => 'GenuineIntel',
 146     'Skylake-Server-IBRS' => 'GenuineIntel',
 147     'Cascadelake-Server' => 'GenuineIntel',
 148     KnightsMill => 'GenuineIntel',
 149
 150
 151     # AMD CPUs
 152     athlon => 'AuthenticAMD',
 153     phenom  => 'AuthenticAMD',
 154     Opteron_G1  => 'AuthenticAMD',
 155     Opteron_G2  => 'AuthenticAMD',
 156     Opteron_G3  => 'AuthenticAMD',
 157     Opteron_G4  => 'AuthenticAMD',
 158     Opteron_G5  => 'AuthenticAMD',
 159     EPYC => 'AuthenticAMD',
 160     'EPYC-IBPB' => 'AuthenticAMD',
 161
 162     # generic types, use vendor from host node
 163     host => 'default',
 164     kvm32 => 'default',
 165     kvm64 => 'default',
 166     qemu32 => 'default',
 167     qemu64 => 'default',
 168     max => 'default',
 169 };
 170
 171 my @supported_cpu_flags = (
 172     'pcid',
 173     'spec-ctrl',
 174     'ibpb',
 175     'ssbd',
 176     'virt-ssbd',
 177     'amd-ssbd',
 178     'amd-no-ssb',
 179     'pdpe1gb',
 180     'md-clear',
 181     'hv-tlbflush',
 182     'hv-evmcs',
 183     'aes'
 184 );
 185 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 186
 187 my $cpu_fmt = {
 188     cputype => {
 189         description => "Emulated CPU type.",
 190         type => 'string',
 191         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 192         default => 'kvm64',
 193         default_key => 1,
 194     },
 195     hidden => {
 196         description => "Do not identify as a KVM virtual machine.",
 197         type => 'boolean',
 198         optional => 1,
 199         default => 0
 200     },
 201     'hv-vendor-id' => {
 202         type => 'string',
 203         pattern => qr/[a-zA-Z0-9]{1,12}/,
 204         format_description => 'vendor-id',
 205         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 206         optional => 1,
 207     },
 208     flags => {
 209         description => "List of additional CPU flags separated by ';'."
 210                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 211                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 212         format_description => '+FLAG[;-FLAG...]',
 213         type => 'string',
 214         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 215         optional => 1,
 216     },
 217 };
 218
 219 my $watchdog_fmt = {
 220     model => {
 221         default_key => 1,
 222         type => 'string',
 223         enum => [qw(i6300esb ib700)],
 224         description => "Watchdog type to emulate.",
 225         default => 'i6300esb',
 226         optional => 1,
 227     },
 228     action => {
 229         type => 'string',
 230         enum => [qw(reset shutdown poweroff pause debug none)],
 231         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 232         optional => 1,
 233     },
 234 };
 235 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 236
 237 my $agent_fmt = {
 238     enabled => {
 239         description => "Enable/disable Qemu GuestAgent.",
 240         type => 'boolean',
 241         default => 0,
 242         default_key => 1,
 243     },
 244     fstrim_cloned_disks => {
 245         description => "Run fstrim after cloning/moving a disk.",
 246         type => 'boolean',
 247         optional => 1,
 248         default => 0
 249     },
 250     type => {
 251         description => "Select the agent type",
 252         type => 'string',
 253         default => 'virtio',
 254         optional => 1,
 255         enum => [qw(virtio isa)],
 256     },
 257 };
 258
 259 my $vga_fmt = {
 260     type => {
 261         description => "Select the VGA type.",
 262         type => 'string',
 263         default => 'std',
 264         optional => 1,
 265         default_key => 1,
 266         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 267     },
 268     memory => {
 269         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 270         type => 'integer',
 271         optional => 1,
 272         minimum => 4,
 273         maximum => 512,
 274     },
 275 };
 276
 277 my $ivshmem_fmt = {
 278     size => {
 279         type => 'integer',
 280         minimum => 1,
 281         description => "The size of the file in MB.",
 282     },
 283     name => {
 284         type => 'string',
 285         pattern => '[a-zA-Z0-9\-]+',
 286         optional => 1,
 287         format_description => 'string',
 288         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 289     },
 290 };
 291
 292 my $audio_fmt = {
 293     device => {
 294         type => 'string',
 295         enum => [qw(ich9-intel-hda intel-hda AC97)],
 296         description =>  "Configure an audio device."
 297     },
 298     driver =>  {
 299         type => 'string',
 300         enum => ['spice'],
 301         default => 'spice',
 302         optional => 1,
 303         description => "Driver backend for the audio device."
 304     },
 305 };
 306
 307 my $spice_enhancements_fmt = {
 308     foldersharing => {
 309         type => 'boolean',
 310         optional => 1,
 311         default => '0',
 312         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 313     },
 314     videostreaming =>  {
 315         type => 'string',
 316         enum => ['off', 'all', 'filter'],
 317         default => 'off',
 318         optional => 1,
 319         description => "Enable video streaming. Uses compression for detected video streams."
 320     },
 321 };
 322
 323 my $confdesc = {
 324     onboot => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Specifies whether a VM will be started during system bootup.",
 328         default => 0,
 329     },
 330     autostart => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Automatic restart after crash (currently ignored).",
 334         default => 0,
 335     },
 336     hotplug => {
 337         optional => 1,
 338         type => 'string', format => 'pve-hotplug-features',
 339         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 340         default => 'network,disk,usb',
 341     },
 342     reboot => {
 343         optional => 1,
 344         type => 'boolean',
 345         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 346         default => 1,
 347     },
 348     lock => {
 349         optional => 1,
 350         type => 'string',
 351         description => "Lock/unlock the VM.",
 352         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 353     },
 354     cpulimit => {
 355         optional => 1,
 356         type => 'number',
 357         description => "Limit of CPU usage.",
 358         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 359         minimum => 0,
 360         maximum => 128,
 361         default => 0,
 362     },
 363     cpuunits => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "CPU weight for a VM.",
 367         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 368         minimum => 2,
 369         maximum => 262144,
 370         default => 1024,
 371     },
 372     memory => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 376         minimum => 16,
 377         default => 512,
 378     },
 379     balloon => {
 380         optional => 1,
 381         type => 'integer',
 382         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 383         minimum => 0,
 384     },
 385     shares => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 389         minimum => 0,
 390         maximum => 50000,
 391         default => 1000,
 392     },
 393     keyboard => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 397                        "It should not be necessary to set it.",
 398         enum => PVE::Tools::kvmkeymaplist(),
 399         default => undef,
 400     },
 401     name => {
 402         optional => 1,
 403         type => 'string', format => 'dns-name',
 404         description => "Set a name for the VM. Only used on the configuration web interface.",
 405     },
 406     scsihw => {
 407         optional => 1,
 408         type => 'string',
 409         description => "SCSI controller model",
 410         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 411         default => 'lsi',
 412     },
 413     description => {
 414         optional => 1,
 415         type => 'string',
 416         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 417     },
 418     ostype => {
 419         optional => 1,
 420         type => 'string',
 421         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 422         description => "Specify guest operating system.",
 423         verbose_description => <<EODESC,
 424 Specify guest operating system. This is used to enable special
 425 optimization/features for specific operating systems:
 426
 427 [horizontal]
 428 other;; unspecified OS
 429 wxp;; Microsoft Windows XP
 430 w2k;; Microsoft Windows 2000
 431 w2k3;; Microsoft Windows 2003
 432 w2k8;; Microsoft Windows 2008
 433 wvista;; Microsoft Windows Vista
 434 win7;; Microsoft Windows 7
 435 win8;; Microsoft Windows 8/2012/2012r2
 436 win10;; Microsoft Windows 10/2016
 437 l24;; Linux 2.4 Kernel
 438 l26;; Linux 2.6 - 5.X Kernel
 439 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 440 EODESC
 441     },
 442     boot => {
 443         optional => 1,
 444         type => 'string',
 445         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 446         pattern => '[acdn]{1,4}',
 447         default => 'cdn',
 448     },
 449     bootdisk => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-bootdisk',
 452         description => "Enable booting from specified disk.",
 453         pattern => '(ide|sata|scsi|virtio)\d+',
 454     },
 455     smp => {
 456         optional => 1,
 457         type => 'integer',
 458         description => "The number of CPUs. Please use option -sockets instead.",
 459         minimum => 1,
 460         default => 1,
 461     },
 462     sockets => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "The number of CPU sockets.",
 466         minimum => 1,
 467         default => 1,
 468     },
 469     cores => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "The number of cores per socket.",
 473         minimum => 1,
 474         default => 1,
 475     },
 476     numa => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable NUMA.",
 480         default => 0,
 481     },
 482     hugepages => {
 483         optional => 1,
 484         type => 'string',
 485         description => "Enable/disable hugepages memory.",
 486         enum => [qw(any 2 1024)],
 487     },
 488     vcpus => {
 489         optional => 1,
 490         type => 'integer',
 491         description => "Number of hotplugged vcpus.",
 492         minimum => 1,
 493         default => 0,
 494     },
 495     acpi => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable ACPI.",
 499         default => 1,
 500     },
 501     agent => {
 502         optional => 1,
 503         description => "Enable/disable Qemu GuestAgent and its properties.",
 504         type => 'string',
 505         format => $agent_fmt,
 506     },
 507     kvm => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable KVM hardware virtualization.",
 511         default => 1,
 512     },
 513     tdf => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Enable/disable time drift fix.",
 517         default => 0,
 518     },
 519     localtime => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 523     },
 524     freeze => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 528     },
 529     vga => {
 530         optional => 1,
 531         type => 'string', format => $vga_fmt,
 532         description => "Configure the VGA hardware.",
 533         verbose_description => "Configure the VGA Hardware. If you want to use ".
 534             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 535             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 536             "is 'std' for all OS types besides some Windows versions (XP and " .
 537             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 538             "display server. For win* OS you can select how many independent " .
 539             "displays you want, Linux guests can add displays them self.\n".
 540             "You can also run without any graphic card, using a serial device as terminal.",
 541     },
 542     watchdog => {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-watchdog',
 545         description => "Create a virtual hardware watchdog device.",
 546         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 547             " (by a guest action), the watchdog must be periodically polled " .
 548             "by an agent inside the guest or else the watchdog will reset " .
 549             "the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 556         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 557         default => 'now',
 558     },
 559     startup =>  get_standard_option('pve-startup-order'),
 560     template => {
 561         optional => 1,
 562         type => 'boolean',
 563         description => "Enable/disable Template.",
 564         default => 0,
 565     },
 566     args => {
 567         optional => 1,
 568         type => 'string',
 569         description => "Arbitrary arguments passed to kvm.",
 570         verbose_description => <<EODESCR,
 571 Arbitrary arguments passed to kvm, for example:
 572
 573 args: -no-reboot -no-hpet
 574
 575 NOTE: this option is for experts only.
 576 EODESCR
 577     },
 578     tablet => {
 579         optional => 1,
 580         type => 'boolean',
 581         default => 1,
 582         description => "Enable/disable the USB tablet device.",
 583         verbose_description => "Enable/disable the USB tablet device. This device is " .
 584             "usually needed to allow absolute mouse positioning with VNC. " .
 585             "Else the mouse runs out of sync with normal VNC clients. " .
 586             "If you're running lots of console-only guests on one host, " .
 587             "you may consider disabling this to save some context switches. " .
 588             "This is turned off by default if you use spice (-vga=qxl).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => $cpu_fmt,
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 630     },
 631     vmstatestorage => get_standard_option('pve-storage-id', {
 632         description => "Default storage for VM state volumes/files.",
 633         optional => 1,
 634     }),
 635     runningmachine => get_standard_option('pve-qemu-machine', {
 636         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 637     }),
 638     machine => get_standard_option('pve-qemu-machine'),
 639     arch => {
 640         description => "Virtual processor architecture. Defaults to the host.",
 641         optional => 1,
 642         type => 'string',
 643         enum => [qw(x86_64 aarch64)],
 644     },
 645     smbios1 => {
 646         description => "Specify SMBIOS type 1 fields.",
 647         type => 'string', format => 'pve-qm-smbios1',
 648         maxLength => 512,
 649         optional => 1,
 650     },
 651     protection => {
 652         optional => 1,
 653         type => 'boolean',
 654         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 669         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 670             " 128-bit integer value identifier to the guest OS. This allows to".
 671             " notify the guest operating system when the virtual machine is".
 672             " executed with a different configuration (e.g. snapshot execution".
 673             " or creation from a template). The guest operating system notices".
 674             " the change, and is then able to react as appropriate by marking".
 675             " its copies of distributed databases as dirty, re-initializing its".
 676             " random number generator, etc.\n".
 677             "Note that auto-creation only works when done throug API/CLI create".
 678             " or update methods, but not when manually editing the config file.",
 679         default => "1 (autogenerated)",
 680         optional => 1,
 681     },
 682     hookscript => {
 683         type => 'string',
 684         format => 'pve-volume-id',
 685         optional => 1,
 686         description => "Script that will be executed during various steps in the vms lifetime.",
 687     },
 688     ivshmem => {
 689         type => 'string',
 690         format => $ivshmem_fmt,
 691         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711 };
 712
 713 my $cicustom_fmt = {
 714     meta => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728     user => {
 729         type => 'string',
 730         optional => 1,
 731         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735 };
 736 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 737
 738 my $confdesc_cloudinit = {
 739     citype => {
 740         optional => 1,
 741         type => 'string',
 742         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 743         enum => ['configdrive2', 'nocloud'],
 744     },
 745     ciuser => {
 746         optional => 1,
 747         type => 'string',
 748         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 749     },
 750     cipassword => {
 751         optional => 1,
 752         type => 'string',
 753         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 754     },
 755     cicustom => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 759         format => 'pve-qm-cicustom',
 760     },
 761     searchdomain => {
 762         optional => 1,
 763         type => 'string',
 764         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 765     },
 766     nameserver => {
 767         optional => 1,
 768         type => 'string', format => 'address-list',
 769         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 770     },
 771     sshkeys => {
 772         optional => 1,
 773         type => 'string',
 774         format => 'urlencoded',
 775         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 776     },
 777 };
 778
 779 # what about other qemu settings ?
 780 #cpu => 'string',
 781 #machine => 'string',
 782 #fda => 'file',
 783 #fdb => 'file',
 784 #mtdblock => 'file',
 785 #sd => 'file',
 786 #pflash => 'file',
 787 #snapshot => 'bool',
 788 #bootp => 'file',
 789 ##tftp => 'dir',
 790 ##smb => 'dir',
 791 #kernel => 'file',
 792 #append => 'string',
 793 #initrd => 'file',
 794 ##soundhw => 'string',
 795
 796 while (my ($k, $v) = each %$confdesc) {
 797     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 798 }
 799
 800 my $MAX_IDE_DISKS = 4;
 801 my $MAX_SCSI_DISKS = 14;
 802 my $MAX_VIRTIO_DISKS = 16;
 803 my $MAX_SATA_DISKS = 6;
 804 my $MAX_USB_DEVICES = 5;
 805 my $MAX_NETS = 32;
 806 my $MAX_UNUSED_DISKS = 256;
 807 my $MAX_HOSTPCI_DEVICES = 16;
 808 my $MAX_SERIAL_PORTS = 4;
 809 my $MAX_PARALLEL_PORTS = 3;
 810 my $MAX_NUMA = 8;
 811
 812 my $numa_fmt = {
 813     cpus => {
 814         type => "string",
 815         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 816         description => "CPUs accessing this NUMA node.",
 817         format_description => "id[-id];...",
 818     },
 819     memory => {
 820         type => "number",
 821         description => "Amount of memory this NUMA node provides.",
 822         optional => 1,
 823     },
 824     hostnodes => {
 825         type => "string",
 826         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 827         description => "Host NUMA nodes to use.",
 828         format_description => "id[-id];...",
 829         optional => 1,
 830     },
 831     policy => {
 832         type => 'string',
 833         enum => [qw(preferred bind interleave)],
 834         description => "NUMA allocation policy.",
 835         optional => 1,
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 839 my $numadesc = {
 840     optional => 1,
 841     type => 'string', format => $numa_fmt,
 842     description => "NUMA topology.",
 843 };
 844 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 845
 846 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 847     $confdesc->{"numa$i"} = $numadesc;
 848 }
 849
 850 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 851                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 852                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 853 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 854
 855 my $net_fmt_bridge_descr = <<__EOD__;
 856 Bridge to attach the network device to. The Proxmox VE standard bridge
 857 is called 'vmbr0'.
 858
 859 If you do not specify a bridge, we create a kvm user (NATed) network
 860 device, which provides DHCP and DNS services. The following addresses
 861 are used:
 862
 863  10.0.2.2   Gateway
 864  10.0.2.3   DNS Server
 865  10.0.2.4   SMB Server
 866
 867 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 868 __EOD__
 869
 870 my $net_fmt = {
 871     macaddr  => get_standard_option('mac-addr', {
 872         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 873     }),
 874     model => {
 875         type => 'string',
 876         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 877         enum => $nic_model_list,
 878         default_key => 1,
 879     },
 880     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 881     bridge => {
 882         type => 'string',
 883         description => $net_fmt_bridge_descr,
 884         format_description => 'bridge',
 885         optional => 1,
 886     },
 887     queues => {
 888         type => 'integer',
 889         minimum => 0, maximum => 16,
 890         description => 'Number of packet queues to be used on the device.',
 891         optional => 1,
 892     },
 893     rate => {
 894         type => 'number',
 895         minimum => 0,
 896         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 897         optional => 1,
 898     },
 899     tag => {
 900         type => 'integer',
 901         minimum => 1, maximum => 4094,
 902         description => 'VLAN tag to apply to packets on this interface.',
 903         optional => 1,
 904     },
 905     trunks => {
 906         type => 'string',
 907         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 908         description => 'VLAN trunks to pass through this interface.',
 909         format_description => 'vlanid[;vlanid...]',
 910         optional => 1,
 911     },
 912     firewall => {
 913         type => 'boolean',
 914         description => 'Whether this interface should be protected by the firewall.',
 915         optional => 1,
 916     },
 917     link_down => {
 918         type => 'boolean',
 919         description => 'Whether this interface should be disconnected (like pulling the plug).',
 920         optional => 1,
 921     },
 922 };
 923
 924 my $netdesc = {
 925     optional => 1,
 926     type => 'string', format => $net_fmt,
 927     description => "Specify network devices.",
 928 };
 929
 930 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 931
 932 my $ipconfig_fmt = {
 933     ip => {
 934         type => 'string',
 935         format => 'pve-ipv4-config',
 936         format_description => 'IPv4Format/CIDR',
 937         description => 'IPv4 address in CIDR format.',
 938         optional => 1,
 939         default => 'dhcp',
 940     },
 941     gw => {
 942         type => 'string',
 943         format => 'ipv4',
 944         format_description => 'GatewayIPv4',
 945         description => 'Default gateway for IPv4 traffic.',
 946         optional => 1,
 947         requires => 'ip',
 948     },
 949     ip6 => {
 950         type => 'string',
 951         format => 'pve-ipv6-config',
 952         format_description => 'IPv6Format/CIDR',
 953         description => 'IPv6 address in CIDR format.',
 954         optional => 1,
 955         default => 'dhcp',
 956     },
 957     gw6 => {
 958         type => 'string',
 959         format => 'ipv6',
 960         format_description => 'GatewayIPv6',
 961         description => 'Default gateway for IPv6 traffic.',
 962         optional => 1,
 963         requires => 'ip6',
 964     },
 965 };
 966 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 967 my $ipconfigdesc = {
 968     optional => 1,
 969     type => 'string', format => 'pve-qm-ipconfig',
 970     description => <<'EODESCR',
 971 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 972
 973 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 974
 975 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 976 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 977
 978 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 979 EODESCR
 980 };
 981 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 982
 983 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 984     $confdesc->{"net$i"} = $netdesc;
 985     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 986 }
 987
 988 foreach my $key (keys %$confdesc_cloudinit) {
 989     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 990 }
 991
 992 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 993 sub verify_volume_id_or_qm_path {
 994     my ($volid, $noerr) = @_;
 995
 996     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 997         return $volid;
 998     }
 999
1000     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1001     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1002     if ($@) {
1003         return undef if $noerr;
1004         die $@;
1005     }
1006     return $volid;
1007 }
1008
1009 my $drivename_hash;
1010
1011 my %drivedesc_base = (
1012     volume => { alias => 'file' },
1013     file => {
1014         type => 'string',
1015         format => 'pve-volume-id-or-qm-path',
1016         default_key => 1,
1017         format_description => 'volume',
1018         description => "The drive's backing volume.",
1019     },
1020     media => {
1021         type => 'string',
1022         enum => [qw(cdrom disk)],
1023         description => "The drive's media type.",
1024         default => 'disk',
1025         optional => 1
1026     },
1027     cyls => {
1028         type => 'integer',
1029         description => "Force the drive's physical geometry to have a specific cylinder count.",
1030         optional => 1
1031     },
1032     heads => {
1033         type => 'integer',
1034         description => "Force the drive's physical geometry to have a specific head count.",
1035         optional => 1
1036     },
1037     secs => {
1038         type => 'integer',
1039         description => "Force the drive's physical geometry to have a specific sector count.",
1040         optional => 1
1041     },
1042     trans => {
1043         type => 'string',
1044         enum => [qw(none lba auto)],
1045         description => "Force disk geometry bios translation mode.",
1046         optional => 1,
1047     },
1048     snapshot => {
1049         type => 'boolean',
1050         description => "Controls qemu's snapshot mode feature."
1051             . " If activated, changes made to the disk are temporary and will"
1052             . " be discarded when the VM is shutdown.",
1053         optional => 1,
1054     },
1055     cache => {
1056         type => 'string',
1057         enum => [qw(none writethrough writeback unsafe directsync)],
1058         description => "The drive's cache mode",
1059         optional => 1,
1060     },
1061     format => get_standard_option('pve-qm-image-format'),
1062     size => {
1063         type => 'string',
1064         format => 'disk-size',
1065         format_description => 'DiskSize',
1066         description => "Disk size. This is purely informational and has no effect.",
1067         optional => 1,
1068     },
1069     backup => {
1070         type => 'boolean',
1071         description => "Whether the drive should be included when making backups.",
1072         optional => 1,
1073     },
1074     replicate => {
1075         type => 'boolean',
1076         description => 'Whether the drive should considered for replication jobs.',
1077         optional => 1,
1078         default => 1,
1079     },
1080     rerror => {
1081         type => 'string',
1082         enum => [qw(ignore report stop)],
1083         description => 'Read error action.',
1084         optional => 1,
1085     },
1086     werror => {
1087         type => 'string',
1088         enum => [qw(enospc ignore report stop)],
1089         description => 'Write error action.',
1090         optional => 1,
1091     },
1092     aio => {
1093         type => 'string',
1094         enum => [qw(native threads)],
1095         description => 'AIO type to use.',
1096         optional => 1,
1097     },
1098     discard => {
1099         type => 'string',
1100         enum => [qw(ignore on)],
1101         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1102         optional => 1,
1103     },
1104     detect_zeroes => {
1105         type => 'boolean',
1106         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1107         optional => 1,
1108     },
1109     serial => {
1110         type => 'string',
1111         format => 'urlencoded',
1112         format_description => 'serial',
1113         maxLength => 20*3, # *3 since it's %xx url enoded
1114         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1115         optional => 1,
1116     },
1117     shared => {
1118         type => 'boolean',
1119         description => 'Mark this locally-managed volume as available on all nodes',
1120         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1121         optional => 1,
1122         default => 0,
1123     }
1124 );
1125
1126 my %iothread_fmt = ( iothread => {
1127         type => 'boolean',
1128         description => "Whether to use iothreads for this drive",
1129         optional => 1,
1130 });
1131
1132 my %model_fmt = (
1133     model => {
1134         type => 'string',
1135         format => 'urlencoded',
1136         format_description => 'model',
1137         maxLength => 40*3, # *3 since it's %xx url enoded
1138         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1139         optional => 1,
1140     },
1141 );
1142
1143 my %queues_fmt = (
1144     queues => {
1145         type => 'integer',
1146         description => "Number of queues.",
1147         minimum => 2,
1148         optional => 1
1149     }
1150 );
1151
1152 my %scsiblock_fmt = (
1153     scsiblock => {
1154         type => 'boolean',
1155         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1156         optional => 1,
1157         default => 0,
1158     },
1159 );
1160
1161 my %ssd_fmt = (
1162     ssd => {
1163         type => 'boolean',
1164         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1165         optional => 1,
1166     },
1167 );
1168
1169 my %wwn_fmt = (
1170     wwn => {
1171         type => 'string',
1172         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1173         format_description => 'wwn',
1174         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1175         optional => 1,
1176     },
1177 );
1178
1179 my $add_throttle_desc = sub {
1180     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1181     my $d = {
1182         type => $type,
1183         format_description => $unit,
1184         description => "Maximum $what in $longunit.",
1185         optional => 1,
1186     };
1187     $d->{minimum} = $minimum if defined($minimum);
1188     $drivedesc_base{$key} = $d;
1189 };
1190 # throughput: (leaky bucket)
1191 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1192 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1193 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1194 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1200
1201 # pools: (pool of IO before throttling starts taking effect)
1202 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1203 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1204 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1205 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1206 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1207 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1208
1209 # burst lengths
1210 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1211 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1212 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1213 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1214 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1215 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1216
1217 # legacy support
1218 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1219 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1220 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1221 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1222
1223 my $ide_fmt = {
1224     %drivedesc_base,
1225     %model_fmt,
1226     %ssd_fmt,
1227     %wwn_fmt,
1228 };
1229 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1230
1231 my $idedesc = {
1232     optional => 1,
1233     type => 'string', format => $ide_fmt,
1234     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1235 };
1236 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1237
1238 my $scsi_fmt = {
1239     %drivedesc_base,
1240     %iothread_fmt,
1241     %queues_fmt,
1242     %scsiblock_fmt,
1243     %ssd_fmt,
1244     %wwn_fmt,
1245 };
1246 my $scsidesc = {
1247     optional => 1,
1248     type => 'string', format => $scsi_fmt,
1249     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1250 };
1251 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1252
1253 my $sata_fmt = {
1254     %drivedesc_base,
1255     %ssd_fmt,
1256     %wwn_fmt,
1257 };
1258 my $satadesc = {
1259     optional => 1,
1260     type => 'string', format => $sata_fmt,
1261     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1262 };
1263 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1264
1265 my $virtio_fmt = {
1266     %drivedesc_base,
1267     %iothread_fmt,
1268 };
1269 my $virtiodesc = {
1270     optional => 1,
1271     type => 'string', format => $virtio_fmt,
1272     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1273 };
1274 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1275
1276 my $alldrive_fmt = {
1277     %drivedesc_base,
1278     %iothread_fmt,
1279     %model_fmt,
1280     %queues_fmt,
1281     %scsiblock_fmt,
1282     %ssd_fmt,
1283     %wwn_fmt,
1284 };
1285
1286 my $efidisk_fmt = {
1287     volume => { alias => 'file' },
1288     file => {
1289         type => 'string',
1290         format => 'pve-volume-id-or-qm-path',
1291         default_key => 1,
1292         format_description => 'volume',
1293         description => "The drive's backing volume.",
1294     },
1295     format => get_standard_option('pve-qm-image-format'),
1296     size => {
1297         type => 'string',
1298         format => 'disk-size',
1299         format_description => 'DiskSize',
1300         description => "Disk size. This is purely informational and has no effect.",
1301         optional => 1,
1302     },
1303 };
1304
1305 my $efidisk_desc = {
1306     optional => 1,
1307     type => 'string', format => $efidisk_fmt,
1308     description => "Configure a Disk for storing EFI vars",
1309 };
1310
1311 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1312
1313 my $usb_fmt = {
1314     host => {
1315         default_key => 1,
1316         type => 'string', format => 'pve-qm-usb-device',
1317         format_description => 'HOSTUSBDEVICE|spice',
1318         description => <<EODESCR,
1319 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1320
1321  'bus-port(.port)*' (decimal numbers) or
1322  'vendor_id:product_id' (hexadeciaml numbers) or
1323  'spice'
1324
1325 You can use the 'lsusb -t' command to list existing usb devices.
1326
1327 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1328
1329 The value 'spice' can be used to add a usb redirection devices for spice.
1330 EODESCR
1331     },
1332     usb3 => {
1333         optional => 1,
1334         type => 'boolean',
1335         description => "Specifies whether if given host option is a USB3 device or port.",
1336         default => 0,
1337     },
1338 };
1339
1340 my $usbdesc = {
1341     optional => 1,
1342     type => 'string', format => $usb_fmt,
1343     description => "Configure an USB device (n is 0 to 4).",
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1346
1347 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1348 my $hostpci_fmt = {
1349     host => {
1350         default_key => 1,
1351         type => 'string',
1352         pattern => qr/$PCIRE(;$PCIRE)*/,
1353         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1354         description => <<EODESCR,
1355 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1356 of PCI virtual functions of the host. HOSTPCIID syntax is:
1357
1358 'bus:dev.func' (hexadecimal numbers)
1359
1360 You can us the 'lspci' command to list existing PCI devices.
1361 EODESCR
1362     },
1363     rombar => {
1364         type => 'boolean',
1365         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1366         optional => 1,
1367         default => 1,
1368     },
1369     romfile => {
1370         type => 'string',
1371         pattern => '[^,;]+',
1372         format_description => 'string',
1373         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1374         optional => 1,
1375     },
1376     pcie => {
1377         type => 'boolean',
1378         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1379         optional => 1,
1380         default => 0,
1381     },
1382     'x-vga' => {
1383         type => 'boolean',
1384         description =>  "Enable vfio-vga device support.",
1385         optional => 1,
1386         default => 0,
1387     },
1388     'mdev' => {
1389         type => 'string',
1390         format_description => 'string',
1391         pattern => '[^/\.:]+',
1392         optional => 1,
1393         description => <<EODESCR
1394 The type of mediated device to use.
1395 An instance of this type will be created on startup of the VM and
1396 will be cleaned up when the VM stops.
1397 EODESCR
1398     }
1399 };
1400 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1401
1402 my $hostpcidesc = {
1403         optional => 1,
1404         type => 'string', format => 'pve-qm-hostpci',
1405         description => "Map host PCI devices into guest.",
1406         verbose_description =>  <<EODESCR,
1407 Map host PCI devices into guest.
1408
1409 NOTE: This option allows direct access to host hardware. So it is no longer
1410 possible to migrate such machines - use with special care.
1411
1412 CAUTION: Experimental! User reported problems with this option.
1413 EODESCR
1414 };
1415 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1416
1417 my $serialdesc = {
1418         optional => 1,
1419         type => 'string',
1420         pattern => '(/dev/.+|socket)',
1421         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1422         verbose_description =>  <<EODESCR,
1423 Create a serial device inside the VM (n is 0 to 3), and pass through a
1424 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1425 host side (use 'qm terminal' to open a terminal connection).
1426
1427 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1428
1429 CAUTION: Experimental! User reported problems with this option.
1430 EODESCR
1431 };
1432
1433 my $paralleldesc= {
1434         optional => 1,
1435         type => 'string',
1436         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1437         description =>  "Map host parallel devices (n is 0 to 2).",
1438         verbose_description =>  <<EODESCR,
1439 Map host parallel devices (n is 0 to 2).
1440
1441 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1442
1443 CAUTION: Experimental! User reported problems with this option.
1444 EODESCR
1445 };
1446
1447 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1448     $confdesc->{"parallel$i"} = $paralleldesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1452     $confdesc->{"serial$i"} = $serialdesc;
1453 }
1454
1455 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1456     $confdesc->{"hostpci$i"} = $hostpcidesc;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1460     $drivename_hash->{"ide$i"} = 1;
1461     $confdesc->{"ide$i"} = $idedesc;
1462 }
1463
1464 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1465     $drivename_hash->{"sata$i"} = 1;
1466     $confdesc->{"sata$i"} = $satadesc;
1467 }
1468
1469 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1470     $drivename_hash->{"scsi$i"} = 1;
1471     $confdesc->{"scsi$i"} = $scsidesc ;
1472 }
1473
1474 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1475     $drivename_hash->{"virtio$i"} = 1;
1476     $confdesc->{"virtio$i"} = $virtiodesc;
1477 }
1478
1479 $drivename_hash->{efidisk0} = 1;
1480 $confdesc->{efidisk0} = $efidisk_desc;
1481
1482 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1483     $confdesc->{"usb$i"} = $usbdesc;
1484 }
1485
1486 my $unuseddesc = {
1487     optional => 1,
1488     type => 'string', format => 'pve-volume-id',
1489     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1490 };
1491
1492 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1493     $confdesc->{"unused$i"} = $unuseddesc;
1494 }
1495
1496 my $kvm_api_version = 0;
1497
1498 sub kvm_version {
1499     return $kvm_api_version if $kvm_api_version;
1500
1501     open my $fh, '<', '/dev/kvm'
1502         or return undef;
1503
1504     # 0xae00 => KVM_GET_API_VERSION
1505     $kvm_api_version = ioctl($fh, 0xae00, 0);
1506
1507     return $kvm_api_version;
1508 }
1509
1510 my $kvm_user_version = {};
1511 my $kvm_mtime = {};
1512
1513 sub kvm_user_version {
1514     my ($binary) = @_;
1515
1516     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1517     my $st = stat($binary);
1518
1519     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1520     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1521         $cachedmtime == $st->mtime;
1522
1523     $kvm_user_version->{$binary} = 'unknown';
1524     $kvm_mtime->{$binary} = $st->mtime;
1525
1526     my $code = sub {
1527         my $line = shift;
1528         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1529             $kvm_user_version->{$binary} = $2;
1530         }
1531     };
1532
1533     eval { run_command([$binary, '--version'], outfunc => $code); };
1534     warn $@ if $@;
1535
1536     return $kvm_user_version->{$binary};
1537
1538 }
1539
1540 sub kernel_has_vhost_net {
1541     return -c '/dev/vhost-net';
1542 }
1543
1544 sub valid_drive_names {
1545     # order is important - used to autoselect boot disk
1546     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1547             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1548             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1549             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1550             'efidisk0');
1551 }
1552
1553 sub is_valid_drivename {
1554     my $dev = shift;
1555
1556     return defined($drivename_hash->{$dev});
1557 }
1558
1559 sub option_exists {
1560     my $key = shift;
1561     return defined($confdesc->{$key});
1562 }
1563
1564 my $cdrom_path;
1565 sub get_cdrom_path {
1566
1567     return  $cdrom_path if $cdrom_path;
1568
1569     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1570     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1571     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1572 }
1573
1574 sub get_iso_path {
1575     my ($storecfg, $vmid, $cdrom) = @_;
1576
1577     if ($cdrom eq 'cdrom') {
1578         return get_cdrom_path();
1579     } elsif ($cdrom eq 'none') {
1580         return '';
1581     } elsif ($cdrom =~ m|^/|) {
1582         return $cdrom;
1583     } else {
1584         return PVE::Storage::path($storecfg, $cdrom);
1585     }
1586 }
1587
1588 # try to convert old style file names to volume IDs
1589 sub filename_to_volume_id {
1590     my ($vmid, $file, $media) = @_;
1591
1592      if (!($file eq 'none' || $file eq 'cdrom' ||
1593           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1594
1595         return undef if $file =~ m|/|;
1596
1597         if ($media && $media eq 'cdrom') {
1598             $file = "local:iso/$file";
1599         } else {
1600             $file = "local:$vmid/$file";
1601         }
1602     }
1603
1604     return $file;
1605 }
1606
1607 sub verify_media_type {
1608     my ($opt, $vtype, $media) = @_;
1609
1610     return if !$media;
1611
1612     my $etype;
1613     if ($media eq 'disk') {
1614         $etype = 'images';
1615     } elsif ($media eq 'cdrom') {
1616         $etype = 'iso';
1617     } else {
1618         die "internal error";
1619     }
1620
1621     return if ($vtype eq $etype);
1622
1623     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1624 }
1625
1626 sub cleanup_drive_path {
1627     my ($opt, $storecfg, $drive) = @_;
1628
1629     # try to convert filesystem paths to volume IDs
1630
1631     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1632         ($drive->{file} !~ m|^/dev/.+|) &&
1633         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1634         ($drive->{file} !~ m/^\d+$/)) {
1635         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1636         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1637         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1638         verify_media_type($opt, $vtype, $drive->{media});
1639         $drive->{file} = $volid;
1640     }
1641
1642     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1643 }
1644
1645 sub parse_hotplug_features {
1646     my ($data) = @_;
1647
1648     my $res = {};
1649
1650     return $res if $data eq '0';
1651
1652     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1653
1654     foreach my $feature (PVE::Tools::split_list($data)) {
1655         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1656             $res->{$1} = 1;
1657         } else {
1658             die "invalid hotplug feature '$feature'\n";
1659         }
1660     }
1661     return $res;
1662 }
1663
1664 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1665 sub pve_verify_hotplug_features {
1666     my ($value, $noerr) = @_;
1667
1668     return $value if parse_hotplug_features($value);
1669
1670     return undef if $noerr;
1671
1672     die "unable to parse hotplug option\n";
1673 }
1674
1675 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1676 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1677 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1678 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1679 #        [,iothread=on][,serial=serial][,model=model]
1680
1681 sub parse_drive {
1682     my ($key, $data) = @_;
1683
1684     my ($interface, $index);
1685
1686     if ($key =~ m/^([^\d]+)(\d+)$/) {
1687         $interface = $1;
1688         $index = $2;
1689     } else {
1690         return undef;
1691     }
1692
1693     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1694                                      : $confdesc->{$key}->{format};
1695     if (!$desc) {
1696         warn "invalid drive key: $key\n";
1697         return undef;
1698     }
1699     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1700     return undef if !$res;
1701     $res->{interface} = $interface;
1702     $res->{index} = $index;
1703
1704     my $error = 0;
1705     foreach my $opt (qw(bps bps_rd bps_wr)) {
1706         if (my $bps = defined(delete $res->{$opt})) {
1707             if (defined($res->{"m$opt"})) {
1708                 warn "both $opt and m$opt specified\n";
1709                 ++$error;
1710                 next;
1711             }
1712             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1713         }
1714     }
1715
1716     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1717     for my $requirement (
1718         [mbps_max => 'mbps'],
1719         [mbps_rd_max => 'mbps_rd'],
1720         [mbps_wr_max => 'mbps_wr'],
1721         [miops_max => 'miops'],
1722         [miops_rd_max => 'miops_rd'],
1723         [miops_wr_max => 'miops_wr'],
1724         [bps_max_length => 'mbps_max'],
1725         [bps_rd_max_length => 'mbps_rd_max'],
1726         [bps_wr_max_length => 'mbps_wr_max'],
1727         [iops_max_length => 'iops_max'],
1728         [iops_rd_max_length => 'iops_rd_max'],
1729         [iops_wr_max_length => 'iops_wr_max']) {
1730         my ($option, $requires) = @$requirement;
1731         if ($res->{$option} && !$res->{$requires}) {
1732             warn "$option requires $requires\n";
1733             ++$error;
1734         }
1735     }
1736
1737     return undef if $error;
1738
1739     return undef if $res->{mbps_rd} && $res->{mbps};
1740     return undef if $res->{mbps_wr} && $res->{mbps};
1741     return undef if $res->{iops_rd} && $res->{iops};
1742     return undef if $res->{iops_wr} && $res->{iops};
1743
1744     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1745         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1746         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1747         return undef if $res->{interface} eq 'virtio';
1748     }
1749
1750     if (my $size = $res->{size}) {
1751         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1752     }
1753
1754     return $res;
1755 }
1756
1757 sub print_drive {
1758     my ($drive) = @_;
1759     my $data = { %$drive };
1760     delete $data->{$_} for qw(index interface);
1761     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1762 }
1763
1764 sub scsi_inquiry {
1765     my($fh, $noerr) = @_;
1766
1767     my $SG_IO = 0x2285;
1768     my $SG_GET_VERSION_NUM = 0x2282;
1769
1770     my $versionbuf = "\x00" x 8;
1771     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1772     if (!$ret) {
1773         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1774         return undef;
1775     }
1776     my $version = unpack("I", $versionbuf);
1777     if ($version < 30000) {
1778         die "scsi generic interface too old\n"  if !$noerr;
1779         return undef;
1780     }
1781
1782     my $buf = "\x00" x 36;
1783     my $sensebuf = "\x00" x 8;
1784     my $cmd = pack("C x3 C x1", 0x12, 36);
1785
1786     # see /usr/include/scsi/sg.h
1787     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1788
1789     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1790                       length($sensebuf), 0, length($buf), $buf,
1791                       $cmd, $sensebuf, 6000);
1792
1793     $ret = ioctl($fh, $SG_IO, $packet);
1794     if (!$ret) {
1795         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1796         return undef;
1797     }
1798
1799     my @res = unpack($sg_io_hdr_t, $packet);
1800     if ($res[17] || $res[18]) {
1801         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1802         return undef;
1803     }
1804
1805     my $res = {};
1806     (my $byte0, my $byte1, $res->{vendor},
1807      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1808
1809     $res->{removable} = $byte1 & 128 ? 1 : 0;
1810     $res->{type} = $byte0 & 31;
1811
1812     return $res;
1813 }
1814
1815 sub path_is_scsi {
1816     my ($path) = @_;
1817
1818     my $fh = IO::File->new("+<$path") || return undef;
1819     my $res = scsi_inquiry($fh, 1);
1820     close($fh);
1821
1822     return $res;
1823 }
1824
1825 sub print_tabletdevice_full {
1826     my ($conf, $arch) = @_;
1827
1828     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1829
1830     # we use uhci for old VMs because tablet driver was buggy in older qemu
1831     my $usbbus;
1832     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1833         $usbbus = 'ehci';
1834     } else {
1835         $usbbus = 'uhci';
1836     }
1837
1838     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1839 }
1840
1841 sub print_keyboarddevice_full {
1842     my ($conf, $arch, $machine) = @_;
1843
1844     return undef if $arch ne 'aarch64';
1845
1846     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1847 }
1848
1849 sub print_drivedevice_full {
1850     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1851
1852     my $device = '';
1853     my $maxdev = 0;
1854
1855     if ($drive->{interface} eq 'virtio') {
1856         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1857         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1858         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1859     } elsif ($drive->{interface} eq 'scsi') {
1860
1861         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1862         my $unit = $drive->{index} % $maxdev;
1863         my $devicetype = 'hd';
1864         my $path = '';
1865         if (drive_is_cdrom($drive)) {
1866             $devicetype = 'cd';
1867         } else {
1868             if ($drive->{file} =~ m|^/|) {
1869                 $path = $drive->{file};
1870                 if (my $info = path_is_scsi($path)) {
1871                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1872                         $devicetype = 'block';
1873                     } elsif ($info->{type} == 1) { # tape
1874                         $devicetype = 'generic';
1875                     }
1876                 }
1877             } else {
1878                  $path = PVE::Storage::path($storecfg, $drive->{file});
1879             }
1880
1881             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1882             my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1883             if ($path =~ m/^iscsi\:\/\// &&
1884                !min_version($version, 4, 1)) {
1885                 $devicetype = 'generic';
1886             }
1887         }
1888
1889         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1890             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1891         } else {
1892             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1893         }
1894
1895         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1896             $device .= ",rotation_rate=1";
1897         }
1898         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1899
1900     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1901         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1902         my $controller = int($drive->{index} / $maxdev);
1903         my $unit = $drive->{index} % $maxdev;
1904         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1905
1906         $device = "ide-$devicetype";
1907         if ($drive->{interface} eq 'ide') {
1908             $device .= ",bus=ide.$controller,unit=$unit";
1909         } else {
1910             $device .= ",bus=ahci$controller.$unit";
1911         }
1912         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1913
1914         if ($devicetype eq 'hd') {
1915             if (my $model = $drive->{model}) {
1916                 $model = URI::Escape::uri_unescape($model);
1917                 $device .= ",model=$model";
1918             }
1919             if ($drive->{ssd}) {
1920                 $device .= ",rotation_rate=1";
1921             }
1922         }
1923         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1924     } elsif ($drive->{interface} eq 'usb') {
1925         die "implement me";
1926         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1927     } else {
1928         die "unsupported interface type";
1929     }
1930
1931     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1932
1933     if (my $serial = $drive->{serial}) {
1934         $serial = URI::Escape::uri_unescape($serial);
1935         $device .= ",serial=$serial";
1936     }
1937
1938
1939     return $device;
1940 }
1941
1942 sub get_initiator_name {
1943     my $initiator;
1944
1945     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1946     while (defined(my $line = <$fh>)) {
1947         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1948         $initiator = $1;
1949         last;
1950     }
1951     $fh->close();
1952
1953     return $initiator;
1954 }
1955
1956 sub print_drive_full {
1957     my ($storecfg, $vmid, $drive) = @_;
1958
1959     my $path;
1960     my $volid = $drive->{file};
1961     my $format;
1962
1963     if (drive_is_cdrom($drive)) {
1964         $path = get_iso_path($storecfg, $vmid, $volid);
1965     } else {
1966         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1967         if ($storeid) {
1968             $path = PVE::Storage::path($storecfg, $volid);
1969             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1970             $format = qemu_img_format($scfg, $volname);
1971         } else {
1972             $path = $volid;
1973             $format = "raw";
1974         }
1975    }
1976
1977     my $opts = '';
1978     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1979     foreach my $o (@qemu_drive_options) {
1980         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1981     }
1982
1983     # snapshot only accepts on|off
1984     if (defined($drive->{snapshot})) {
1985         my $v = $drive->{snapshot} ? 'on' : 'off';
1986         $opts .= ",snapshot=$v";
1987     }
1988
1989     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1990         my ($dir, $qmpname) = @$type;
1991         if (my $v = $drive->{"mbps$dir"}) {
1992             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1993         }
1994         if (my $v = $drive->{"mbps${dir}_max"}) {
1995             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1996         }
1997         if (my $v = $drive->{"bps${dir}_max_length"}) {
1998             $opts .= ",throttling.bps$qmpname-max-length=$v";
1999         }
2000         if (my $v = $drive->{"iops${dir}"}) {
2001             $opts .= ",throttling.iops$qmpname=$v";
2002         }
2003         if (my $v = $drive->{"iops${dir}_max"}) {
2004             $opts .= ",throttling.iops$qmpname-max=$v";
2005         }
2006         if (my $v = $drive->{"iops${dir}_max_length"}) {
2007             $opts .= ",throttling.iops$qmpname-max-length=$v";
2008         }
2009     }
2010
2011     $opts .= ",format=$format" if $format && !$drive->{format};
2012
2013     my $cache_direct = 0;
2014
2015     if (my $cache = $drive->{cache}) {
2016         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2017     } elsif (!drive_is_cdrom($drive)) {
2018         $opts .= ",cache=none";
2019         $cache_direct = 1;
2020     }
2021
2022     # aio native works only with O_DIRECT
2023     if (!$drive->{aio}) {
2024         if($cache_direct) {
2025             $opts .= ",aio=native";
2026         } else {
2027             $opts .= ",aio=threads";
2028         }
2029     }
2030
2031     if (!drive_is_cdrom($drive)) {
2032         my $detectzeroes;
2033         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2034             $detectzeroes = 'off';
2035         } elsif ($drive->{discard}) {
2036             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2037         } else {
2038             # This used to be our default with discard not being specified:
2039             $detectzeroes = 'on';
2040         }
2041         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2042     }
2043
2044     my $pathinfo = $path ? "file=$path," : '';
2045
2046     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2047 }
2048
2049 sub print_netdevice_full {
2050     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2051
2052     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2053
2054     my $device = $net->{model};
2055     if ($net->{model} eq 'virtio') {
2056          $device = 'virtio-net-pci';
2057      };
2058
2059     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2060     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2061     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2062         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2063         my $vectors = $net->{queues} * 2 + 2;
2064         $tmpstr .= ",vectors=$vectors,mq=on";
2065     }
2066     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2067
2068     if ($use_old_bios_files) {
2069         my $romfile;
2070         if ($device eq 'virtio-net-pci') {
2071             $romfile = 'pxe-virtio.rom';
2072         } elsif ($device eq 'e1000') {
2073             $romfile = 'pxe-e1000.rom';
2074         } elsif ($device eq 'ne2k') {
2075             $romfile = 'pxe-ne2k_pci.rom';
2076         } elsif ($device eq 'pcnet') {
2077             $romfile = 'pxe-pcnet.rom';
2078         } elsif ($device eq 'rtl8139') {
2079             $romfile = 'pxe-rtl8139.rom';
2080         }
2081         $tmpstr .= ",romfile=$romfile" if $romfile;
2082     }
2083
2084     return $tmpstr;
2085 }
2086
2087 sub print_netdev_full {
2088     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2089
2090     my $i = '';
2091     if ($netid =~ m/^net(\d+)$/) {
2092         $i = int($1);
2093     }
2094
2095     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2096
2097     my $ifname = "tap${vmid}i$i";
2098
2099     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2100     die "interface name '$ifname' is too long (max 15 character)\n"
2101         if length($ifname) >= 16;
2102
2103     my $vhostparam = '';
2104     if (is_native($arch)) {
2105         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2106     }
2107
2108     my $vmname = $conf->{name} || "vm$vmid";
2109
2110     my $netdev = "";
2111     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2112
2113     if ($net->{bridge}) {
2114         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2115     } else {
2116         $netdev = "type=user,id=$netid,hostname=$vmname";
2117     }
2118
2119     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2120
2121     return $netdev;
2122 }
2123
2124
2125 sub print_cpu_device {
2126     my ($conf, $id) = @_;
2127
2128     my $kvm = $conf->{kvm} // 1;
2129     my $cpu = $kvm ? "kvm64" : "qemu64";
2130     if (my $cputype = $conf->{cpu}) {
2131         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2132             or die "Cannot parse cpu description: $cputype\n";
2133         $cpu = $cpuconf->{cputype};
2134     }
2135
2136     my $cores = $conf->{cores} || 1;
2137
2138     my $current_core = ($id - 1) % $cores;
2139     my $current_socket = int(($id - 1 - $current_core)/$cores);
2140
2141     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2142 }
2143
2144 my $vga_map = {
2145     'cirrus' => 'cirrus-vga',
2146     'std' => 'VGA',
2147     'vmware' => 'vmware-svga',
2148     'virtio' => 'virtio-vga',
2149 };
2150
2151 sub print_vga_device {
2152     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2153
2154     my $type = $vga_map->{$vga->{type}};
2155     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2156         $type = 'virtio-gpu';
2157     }
2158     my $vgamem_mb = $vga->{memory};
2159
2160     my $max_outputs = '';
2161     if ($qxlnum) {
2162         $type = $id ? 'qxl' : 'qxl-vga';
2163
2164         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
2165             # set max outputs so linux can have up to 4 qxl displays with one device
2166             if (min_version($machine_version, 4, 1)) {
2167                 $max_outputs = ",max_outputs=4";
2168             }
2169         }
2170     }
2171
2172     die "no devicetype for $vga->{type}\n" if !$type;
2173
2174     my $memory = "";
2175     if ($vgamem_mb) {
2176         if ($vga->{type} eq 'virtio') {
2177             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2178             $memory = ",max_hostmem=$bytes";
2179         } elsif ($qxlnum) {
2180             # from https://www.spice-space.org/multiple-monitors.html
2181             $memory = ",vgamem_mb=$vga->{memory}";
2182             my $ram = $vgamem_mb * 4;
2183             my $vram = $vgamem_mb * 2;
2184             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2185         } else {
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187         }
2188     } elsif ($qxlnum && $id) {
2189         $memory = ",ram_size=67108864,vram_size=33554432";
2190     }
2191
2192     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2193     my $vgaid = "vga" . ($id // '');
2194     my $pciaddr;
2195
2196     if ($q35 && $vgaid eq 'vga') {
2197         # the first display uses pcie.0 bus on q35 machines
2198         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2199     } else {
2200         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2201     }
2202
2203     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2204 }
2205
2206 sub drive_is_cloudinit {
2207     my ($drive) = @_;
2208     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2209 }
2210
2211 sub drive_is_cdrom {
2212     my ($drive, $exclude_cloudinit) = @_;
2213
2214     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2215
2216     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2217
2218 }
2219
2220 sub parse_number_sets {
2221     my ($set) = @_;
2222     my $res = [];
2223     foreach my $part (split(/;/, $set)) {
2224         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2225             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2226             push @$res, [ $1, $2 ];
2227         } else {
2228             die "invalid range: $part\n";
2229         }
2230     }
2231     return $res;
2232 }
2233
2234 sub parse_numa {
2235     my ($data) = @_;
2236
2237     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2238     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2239     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2240     return $res;
2241 }
2242
2243 sub parse_hostpci {
2244     my ($value) = @_;
2245
2246     return undef if !$value;
2247
2248     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2249
2250     my @idlist = split(/;/, $res->{host});
2251     delete $res->{host};
2252     foreach my $id (@idlist) {
2253         my $devs = PVE::SysFSTools::lspci($id);
2254         die "no PCI device found for '$id'\n" if !scalar(@$devs);
2255         push @{$res->{pciid}}, @$devs;
2256     }
2257     return $res;
2258 }
2259
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2261 sub parse_net {
2262     my ($data) = @_;
2263
2264     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2265     if ($@) {
2266         warn $@;
2267         return undef;
2268     }
2269     if (!defined($res->{macaddr})) {
2270         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2272     }
2273     return $res;
2274 }
2275
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig {
2278     my ($data) = @_;
2279
2280     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2281     if ($@) {
2282         warn $@;
2283         return undef;
2284     }
2285
2286     if ($res->{gw} && !$res->{ip}) {
2287         warn 'gateway specified without specifying an IP address';
2288         return undef;
2289     }
2290     if ($res->{gw6} && !$res->{ip6}) {
2291         warn 'IPv6 gateway specified without specifying an IPv6 address';
2292         return undef;
2293     }
2294     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2295         warn 'gateway specified together with DHCP';
2296         return undef;
2297     }
2298     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2299         # gw6 + auto/dhcp
2300         warn "IPv6 gateway specified together with $res->{ip6} address";
2301         return undef;
2302     }
2303
2304     if (!$res->{ip} && !$res->{ip6}) {
2305         return { ip => 'dhcp', ip6 => 'dhcp' };
2306     }
2307
2308     return $res;
2309 }
2310
2311 sub print_net {
2312     my $net = shift;
2313
2314     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2315 }
2316
2317 sub add_random_macs {
2318     my ($settings) = @_;
2319
2320     foreach my $opt (keys %$settings) {
2321         next if $opt !~ m/^net(\d+)$/;
2322         my $net = parse_net($settings->{$opt});
2323         next if !$net;
2324         $settings->{$opt} = print_net($net);
2325     }
2326 }
2327
2328 sub vm_is_volid_owner {
2329     my ($storecfg, $vmid, $volid) = @_;
2330
2331     if ($volid !~  m|^/|) {
2332         my ($path, $owner);
2333         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2334         if ($owner && ($owner == $vmid)) {
2335             return 1;
2336         }
2337     }
2338
2339     return undef;
2340 }
2341
2342 sub vmconfig_register_unused_drive {
2343     my ($storecfg, $vmid, $conf, $drive) = @_;
2344
2345     if (drive_is_cloudinit($drive)) {
2346         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2347         warn $@ if $@;
2348     } elsif (!drive_is_cdrom($drive)) {
2349         my $volid = $drive->{file};
2350         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2351             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2352         }
2353     }
2354 }
2355
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2357 my $smbios1_fmt = {
2358     uuid => {
2359         type => 'string',
2360         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361         format_description => 'UUID',
2362         description => "Set SMBIOS1 UUID.",
2363         optional => 1,
2364     },
2365     version => {
2366         type => 'string',
2367         pattern => '[A-Za-z0-9+\/]+={0,2}',
2368         format_description => 'Base64 encoded string',
2369         description => "Set SMBIOS1 version.",
2370         optional => 1,
2371     },
2372     serial => {
2373         type => 'string',
2374         pattern => '[A-Za-z0-9+\/]+={0,2}',
2375         format_description => 'Base64 encoded string',
2376         description => "Set SMBIOS1 serial number.",
2377         optional => 1,
2378     },
2379     manufacturer => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 manufacturer.",
2384         optional => 1,
2385     },
2386     product => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 product ID.",
2391         optional => 1,
2392     },
2393     sku => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 SKU string.",
2398         optional => 1,
2399     },
2400     family => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 family string.",
2405         optional => 1,
2406     },
2407     base64 => {
2408         type => 'boolean',
2409         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2410         optional => 1,
2411     },
2412 };
2413
2414 sub parse_smbios1 {
2415     my ($data) = @_;
2416
2417     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2418     warn $@ if $@;
2419     return $res;
2420 }
2421
2422 sub print_smbios1 {
2423     my ($smbios1) = @_;
2424     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2425 }
2426
2427 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2428
2429 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2430 sub verify_bootdisk {
2431     my ($value, $noerr) = @_;
2432
2433     return $value if is_valid_drivename($value);
2434
2435     return undef if $noerr;
2436
2437     die "invalid boot disk '$value'\n";
2438 }
2439
2440 sub parse_watchdog {
2441     my ($value) = @_;
2442
2443     return undef if !$value;
2444
2445     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2446     warn $@ if $@;
2447     return $res;
2448 }
2449
2450 sub parse_guest_agent {
2451     my ($value) = @_;
2452
2453     return {} if !defined($value->{agent});
2454
2455     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2456     warn $@ if $@;
2457
2458     # if the agent is disabled ignore the other potentially set properties
2459     return {} if !$res->{enabled};
2460     return $res;
2461 }
2462
2463 sub parse_vga {
2464     my ($value) = @_;
2465
2466     return {} if !$value;
2467     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2468     warn $@ if $@;
2469     return $res;
2470 }
2471
2472 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2473 sub verify_usb_device {
2474     my ($value, $noerr) = @_;
2475
2476     return $value if parse_usb_device($value);
2477
2478     return undef if $noerr;
2479
2480     die "unable to parse usb device\n";
2481 }
2482
2483 # add JSON properties for create and set function
2484 sub json_config_properties {
2485     my $prop = shift;
2486
2487     foreach my $opt (keys %$confdesc) {
2488         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489         $prop->{$opt} = $confdesc->{$opt};
2490     }
2491
2492     return $prop;
2493 }
2494
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties {
2497
2498     return dclone($confdesc_cloudinit);
2499 }
2500
2501 sub check_type {
2502     my ($key, $value) = @_;
2503
2504     die "unknown setting '$key'\n" if !$confdesc->{$key};
2505
2506     my $type = $confdesc->{$key}->{type};
2507
2508     if (!defined($value)) {
2509         die "got undefined value\n";
2510     }
2511
2512     if ($value =~ m/[\n\r]/) {
2513         die "property contains a line feed\n";
2514     }
2515
2516     if ($type eq 'boolean') {
2517         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519         die "type check ('boolean') failed - got '$value'\n";
2520     } elsif ($type eq 'integer') {
2521         return int($1) if $value =~ m/^(\d+)$/;
2522         die "type check ('integer') failed - got '$value'\n";
2523     } elsif ($type eq 'number') {
2524         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525         die "type check ('number') failed - got '$value'\n";
2526     } elsif ($type eq 'string') {
2527         if (my $fmt = $confdesc->{$key}->{format}) {
2528             PVE::JSONSchema::check_format($fmt, $value);
2529             return $value;
2530         }
2531         $value =~ s/^\"(.*)\"$/$1/;
2532         return $value;
2533     } else {
2534         die "internal error"
2535     }
2536 }
2537
2538 sub destroy_vm {
2539     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2540
2541     my $conf = PVE::QemuConfig->load_config($vmid);
2542
2543     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2544
2545     if ($conf->{template}) {
2546         # check if any base image is still used by a linked clone
2547         foreach_drive($conf, sub {
2548                 my ($ds, $drive) = @_;
2549                 return if drive_is_cdrom($drive);
2550
2551                 my $volid = $drive->{file};
2552                 return if !$volid || $volid =~ m|^/|;
2553
2554                 die "base volume '$volid' is still in use by linked cloned\n"
2555                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2556
2557         });
2558     }
2559
2560     # only remove disks owned by this VM
2561     foreach_drive($conf, sub {
2562         my ($ds, $drive) = @_;
2563         return if drive_is_cdrom($drive, 1);
2564
2565         my $volid = $drive->{file};
2566         return if !$volid || $volid =~ m|^/|;
2567
2568         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2569         return if !$path || !$owner || ($owner != $vmid);
2570
2571         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2572         warn "Could not remove disk '$volid', check manually: $@" if $@;
2573     });
2574
2575     # also remove unused disk
2576     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2577     PVE::Storage::foreach_volid($vmdisks, sub {
2578         my ($volid, $sid, $volname, $d) = @_;
2579         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2580         warn $@ if $@;
2581     });
2582
2583     if (defined $replacement_conf) {
2584         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2585     } else {
2586         PVE::QemuConfig->destroy_config($vmid);
2587     }
2588 }
2589
2590 sub parse_vm_config {
2591     my ($filename, $raw) = @_;
2592
2593     return undef if !defined($raw);
2594
2595     my $res = {
2596         digest => Digest::SHA::sha1_hex($raw),
2597         snapshots => {},
2598         pending => {},
2599     };
2600
2601     $filename =~ m|/qemu-server/(\d+)\.conf$|
2602         || die "got strange filename '$filename'";
2603
2604     my $vmid = $1;
2605
2606     my $conf = $res;
2607     my $descr;
2608     my $section = '';
2609
2610     my @lines = split(/\n/, $raw);
2611     foreach my $line (@lines) {
2612         next if $line =~ m/^\s*$/;
2613
2614         if ($line =~ m/^\[PENDING\]\s*$/i) {
2615             $section = 'pending';
2616             if (defined($descr)) {
2617                 $descr =~ s/\s+$//;
2618                 $conf->{description} = $descr;
2619             }
2620             $descr = undef;
2621             $conf = $res->{$section} = {};
2622             next;
2623
2624         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2625             $section = $1;
2626             if (defined($descr)) {
2627                 $descr =~ s/\s+$//;
2628                 $conf->{description} = $descr;
2629             }
2630             $descr = undef;
2631             $conf = $res->{snapshots}->{$section} = {};
2632             next;
2633         }
2634
2635         if ($line =~ m/^\#(.*)\s*$/) {
2636             $descr = '' if !defined($descr);
2637             $descr .= PVE::Tools::decode_text($1) . "\n";
2638             next;
2639         }
2640
2641         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642             $descr = '' if !defined($descr);
2643             $descr .= PVE::Tools::decode_text($2);
2644         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645             $conf->{snapstate} = $1;
2646         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2647             my $key = $1;
2648             my $value = $2;
2649             $conf->{$key} = $value;
2650         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2651             my $value = $1;
2652             if ($section eq 'pending') {
2653                 $conf->{delete} = $value; # we parse this later
2654             } else {
2655                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2656             }
2657         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2658             my $key = $1;
2659             my $value = $2;
2660             eval { $value = check_type($key, $value); };
2661             if ($@) {
2662                 warn "vm $vmid - unable to parse value of '$key' - $@";
2663             } else {
2664                 $key = 'ide2' if $key eq 'cdrom';
2665                 my $fmt = $confdesc->{$key}->{format};
2666                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667                     my $v = parse_drive($key, $value);
2668                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2669                         $v->{file} = $volid;
2670                         $value = print_drive($v);
2671                     } else {
2672                         warn "vm $vmid - unable to parse value of '$key'\n";
2673                         next;
2674                     }
2675                 }
2676
2677                 $conf->{$key} = $value;
2678             }
2679         }
2680     }
2681
2682     if (defined($descr)) {
2683         $descr =~ s/\s+$//;
2684         $conf->{description} = $descr;
2685     }
2686     delete $res->{snapstate}; # just to be sure
2687
2688     return $res;
2689 }
2690
2691 sub write_vm_config {
2692     my ($filename, $conf) = @_;
2693
2694     delete $conf->{snapstate}; # just to be sure
2695
2696     if ($conf->{cdrom}) {
2697         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2698         $conf->{ide2} = $conf->{cdrom};
2699         delete $conf->{cdrom};
2700     }
2701
2702     # we do not use 'smp' any longer
2703     if ($conf->{sockets}) {
2704         delete $conf->{smp};
2705     } elsif ($conf->{smp}) {
2706         $conf->{sockets} = $conf->{smp};
2707         delete $conf->{cores};
2708         delete $conf->{smp};
2709     }
2710
2711     my $used_volids = {};
2712
2713     my $cleanup_config = sub {
2714         my ($cref, $pending, $snapname) = @_;
2715
2716         foreach my $key (keys %$cref) {
2717             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718                 $key eq 'snapstate' || $key eq 'pending';
2719             my $value = $cref->{$key};
2720             if ($key eq 'delete') {
2721                 die "propertry 'delete' is only allowed in [PENDING]\n"
2722                     if !$pending;
2723                 # fixme: check syntax?
2724                 next;
2725             }
2726             eval { $value = check_type($key, $value); };
2727             die "unable to parse value of '$key' - $@" if $@;
2728
2729             $cref->{$key} = $value;
2730
2731             if (!$snapname && is_valid_drivename($key)) {
2732                 my $drive = parse_drive($key, $value);
2733                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2734             }
2735         }
2736     };
2737
2738     &$cleanup_config($conf);
2739
2740     &$cleanup_config($conf->{pending}, 1);
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2745     }
2746
2747     # remove 'unusedX' settings if we re-add a volume
2748     foreach my $key (keys %$conf) {
2749         my $value = $conf->{$key};
2750         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751             delete $conf->{$key};
2752         }
2753     }
2754
2755     my $generate_raw_config = sub {
2756         my ($conf, $pending) = @_;
2757
2758         my $raw = '';
2759
2760         # add description as comment to top of file
2761         if (defined(my $descr = $conf->{description})) {
2762             if ($descr) {
2763                 foreach my $cl (split(/\n/, $descr)) {
2764                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2765                 }
2766             } else {
2767                 $raw .= "#\n" if $pending;
2768             }
2769         }
2770
2771         foreach my $key (sort keys %$conf) {
2772             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773             $raw .= "$key: $conf->{$key}\n";
2774         }
2775         return $raw;
2776     };
2777
2778     my $raw = &$generate_raw_config($conf);
2779
2780     if (scalar(keys %{$conf->{pending}})){
2781         $raw .= "\n[PENDING]\n";
2782         $raw .= &$generate_raw_config($conf->{pending}, 1);
2783     }
2784
2785     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2786         $raw .= "\n[$snapname]\n";
2787         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2788     }
2789
2790     return $raw;
2791 }
2792
2793 sub load_defaults {
2794
2795     my $res = {};
2796
2797     # we use static defaults from our JSON schema configuration
2798     foreach my $key (keys %$confdesc) {
2799         if (defined(my $default = $confdesc->{$key}->{default})) {
2800             $res->{$key} = $default;
2801         }
2802     }
2803
2804     return $res;
2805 }
2806
2807 sub config_list {
2808     my $vmlist = PVE::Cluster::get_vmlist();
2809     my $res = {};
2810     return $res if !$vmlist || !$vmlist->{ids};
2811     my $ids = $vmlist->{ids};
2812     my $nodename = nodename();
2813
2814     foreach my $vmid (keys %$ids) {
2815         my $d = $ids->{$vmid};
2816         next if !$d->{node} || $d->{node} ne $nodename;
2817         next if !$d->{type} || $d->{type} ne 'qemu';
2818         $res->{$vmid}->{exists} = 1;
2819     }
2820     return $res;
2821 }
2822
2823 # test if VM uses local resources (to prevent migration)
2824 sub check_local_resources {
2825     my ($conf, $noerr) = @_;
2826
2827     my @loc_res = ();
2828
2829     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2830     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2831
2832     push @loc_res, "ivshmem" if $conf->{ivshmem};
2833
2834     foreach my $k (keys %$conf) {
2835         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2836         # sockets are safe: they will recreated be on the target side post-migrate
2837         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2838         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2839     }
2840
2841     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2842
2843     return \@loc_res;
2844 }
2845
2846 # check if used storages are available on all nodes (use by migrate)
2847 sub check_storage_availability {
2848     my ($storecfg, $conf, $node) = @_;
2849
2850     foreach_drive($conf, sub {
2851         my ($ds, $drive) = @_;
2852
2853         my $volid = $drive->{file};
2854         return if !$volid;
2855
2856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2857         return if !$sid;
2858
2859         # check if storage is available on both nodes
2860         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2861         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2862    });
2863 }
2864
2865 # list nodes where all VM images are available (used by has_feature API)
2866 sub shared_nodes {
2867     my ($conf, $storecfg) = @_;
2868
2869     my $nodelist = PVE::Cluster::get_nodelist();
2870     my $nodehash = { map { $_ => 1 } @$nodelist };
2871     my $nodename = nodename();
2872
2873     foreach_drive($conf, sub {
2874         my ($ds, $drive) = @_;
2875
2876         my $volid = $drive->{file};
2877         return if !$volid;
2878
2879         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2880         if ($storeid) {
2881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2882             if ($scfg->{disable}) {
2883                 $nodehash = {};
2884             } elsif (my $avail = $scfg->{nodes}) {
2885                 foreach my $node (keys %$nodehash) {
2886                     delete $nodehash->{$node} if !$avail->{$node};
2887                 }
2888             } elsif (!$scfg->{shared}) {
2889                 foreach my $node (keys %$nodehash) {
2890                     delete $nodehash->{$node} if $node ne $nodename
2891                 }
2892             }
2893         }
2894     });
2895
2896     return $nodehash
2897 }
2898
2899 sub check_local_storage_availability {
2900     my ($conf, $storecfg) = @_;
2901
2902     my $nodelist = PVE::Cluster::get_nodelist();
2903     my $nodehash = { map { $_ => {} } @$nodelist };
2904
2905     foreach_drive($conf, sub {
2906         my ($ds, $drive) = @_;
2907
2908         my $volid = $drive->{file};
2909         return if !$volid;
2910
2911         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2912         if ($storeid) {
2913             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2914
2915             if ($scfg->{disable}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2918                 }
2919             } elsif (my $avail = $scfg->{nodes}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     if (!$avail->{$node}) {
2922                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2923                     }
2924                 }
2925             }
2926         }
2927     });
2928
2929     foreach my $node (values %$nodehash) {
2930         if (my $unavail = $node->{unavailable_storages}) {
2931             $node->{unavailable_storages} = [ sort keys %$unavail ];
2932         }
2933     }
2934
2935     return $nodehash
2936 }
2937
2938 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2939 sub check_running {
2940     my ($vmid, $nocheck, $node) = @_;
2941
2942     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2943     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2944 }
2945
2946 sub vzlist {
2947
2948     my $vzlist = config_list();
2949
2950     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2951
2952     while (defined(my $de = $fd->read)) {
2953         next if $de !~ m/^(\d+)\.pid$/;
2954         my $vmid = $1;
2955         next if !defined($vzlist->{$vmid});
2956         if (my $pid = check_running($vmid)) {
2957             $vzlist->{$vmid}->{pid} = $pid;
2958         }
2959     }
2960
2961     return $vzlist;
2962 }
2963
2964 sub disksize {
2965     my ($storecfg, $conf) = @_;
2966
2967     my $bootdisk = $conf->{bootdisk};
2968     return undef if !$bootdisk;
2969     return undef if !is_valid_drivename($bootdisk);
2970
2971     return undef if !$conf->{$bootdisk};
2972
2973     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2974     return undef if !defined($drive);
2975
2976     return undef if drive_is_cdrom($drive);
2977
2978     my $volid = $drive->{file};
2979     return undef if !$volid;
2980
2981     return $drive->{size};
2982 }
2983
2984 our $vmstatus_return_properties = {
2985     vmid => get_standard_option('pve-vmid'),
2986     status => {
2987         description => "Qemu process status.",
2988         type => 'string',
2989         enum => ['stopped', 'running'],
2990     },
2991     maxmem => {
2992         description => "Maximum memory in bytes.",
2993         type => 'integer',
2994         optional => 1,
2995         renderer => 'bytes',
2996     },
2997     maxdisk => {
2998         description => "Root disk size in bytes.",
2999         type => 'integer',
3000         optional => 1,
3001         renderer => 'bytes',
3002     },
3003     name => {
3004         description => "VM name.",
3005         type => 'string',
3006         optional => 1,
3007     },
3008     qmpstatus => {
3009         description => "Qemu QMP agent status.",
3010         type => 'string',
3011         optional => 1,
3012     },
3013     pid => {
3014         description => "PID of running qemu process.",
3015         type => 'integer',
3016         optional => 1,
3017     },
3018     uptime => {
3019         description => "Uptime.",
3020         type => 'integer',
3021         optional => 1,
3022         renderer => 'duration',
3023     },
3024     cpus => {
3025         description => "Maximum usable CPUs.",
3026         type => 'number',
3027         optional => 1,
3028     },
3029     lock => {
3030         description => "The current config lock, if any.",
3031         type => 'string',
3032         optional => 1,
3033     },
3034     tags => {
3035         description  => "The current configured tags, if any",
3036         type => 'string',
3037         optional => 1,
3038     },
3039 };
3040
3041 my $last_proc_pid_stat;
3042
3043 # get VM status information
3044 # This must be fast and should not block ($full == false)
3045 # We only query KVM using QMP if $full == true (this can be slow)
3046 sub vmstatus {
3047     my ($opt_vmid, $full) = @_;
3048
3049     my $res = {};
3050
3051     my $storecfg = PVE::Storage::config();
3052
3053     my $list = vzlist();
3054     my $defaults = load_defaults();
3055
3056     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3057
3058     my $cpucount = $cpuinfo->{cpus} || 1;
3059
3060     foreach my $vmid (keys %$list) {
3061         next if $opt_vmid && ($vmid ne $opt_vmid);
3062
3063         my $conf = PVE::QemuConfig->load_config($vmid);
3064
3065         my $d = { vmid => $vmid };
3066         $d->{pid} = $list->{$vmid}->{pid};
3067
3068         # fixme: better status?
3069         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3070
3071         my $size = disksize($storecfg, $conf);
3072         if (defined($size)) {
3073             $d->{disk} = 0; # no info available
3074             $d->{maxdisk} = $size;
3075         } else {
3076             $d->{disk} = 0;
3077             $d->{maxdisk} = 0;
3078         }
3079
3080         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3081             * ($conf->{cores} || $defaults->{cores});
3082         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3083         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3084
3085         $d->{name} = $conf->{name} || "VM $vmid";
3086         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3087             : $defaults->{memory}*(1024*1024);
3088
3089         if ($conf->{balloon}) {
3090             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3091             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3092                 : $defaults->{shares};
3093         }
3094
3095         $d->{uptime} = 0;
3096         $d->{cpu} = 0;
3097         $d->{mem} = 0;
3098
3099         $d->{netout} = 0;
3100         $d->{netin} = 0;
3101
3102         $d->{diskread} = 0;
3103         $d->{diskwrite} = 0;
3104
3105         $d->{template} = PVE::QemuConfig->is_template($conf);
3106
3107         $d->{serial} = 1 if conf_has_serial($conf);
3108         $d->{lock} = $conf->{lock} if $conf->{lock};
3109         $d->{tags} = $conf->{tags} if defined($conf->{tags});
3110
3111         $res->{$vmid} = $d;
3112     }
3113
3114     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3115     foreach my $dev (keys %$netdev) {
3116         next if $dev !~ m/^tap([1-9]\d*)i/;
3117         my $vmid = $1;
3118         my $d = $res->{$vmid};
3119         next if !$d;
3120
3121         $d->{netout} += $netdev->{$dev}->{receive};
3122         $d->{netin} += $netdev->{$dev}->{transmit};
3123
3124         if ($full) {
3125             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3126             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3127         }
3128
3129     }
3130
3131     my $ctime = gettimeofday;
3132
3133     foreach my $vmid (keys %$list) {
3134
3135         my $d = $res->{$vmid};
3136         my $pid = $d->{pid};
3137         next if !$pid;
3138
3139         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3140         next if !$pstat; # not running
3141
3142         my $used = $pstat->{utime} + $pstat->{stime};
3143
3144         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3145
3146         if ($pstat->{vsize}) {
3147             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3148         }
3149
3150         my $old = $last_proc_pid_stat->{$pid};
3151         if (!$old) {
3152             $last_proc_pid_stat->{$pid} = {
3153                 time => $ctime,
3154                 used => $used,
3155                 cpu => 0,
3156             };
3157             next;
3158         }
3159
3160         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3161
3162         if ($dtime > 1000) {
3163             my $dutime = $used -  $old->{used};
3164
3165             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3166             $last_proc_pid_stat->{$pid} = {
3167                 time => $ctime,
3168                 used => $used,
3169                 cpu => $d->{cpu},
3170             };
3171         } else {
3172             $d->{cpu} = $old->{cpu};
3173         }
3174     }
3175
3176     return $res if !$full;
3177
3178     my $qmpclient = PVE::QMPClient->new();
3179
3180     my $ballooncb = sub {
3181         my ($vmid, $resp) = @_;
3182
3183         my $info = $resp->{'return'};
3184         return if !$info->{max_mem};
3185
3186         my $d = $res->{$vmid};
3187
3188         # use memory assigned to VM
3189         $d->{maxmem} = $info->{max_mem};
3190         $d->{balloon} = $info->{actual};
3191
3192         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3193             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3194             $d->{freemem} = $info->{free_mem};
3195         }
3196
3197         $d->{ballooninfo} = $info;
3198     };
3199
3200     my $blockstatscb = sub {
3201         my ($vmid, $resp) = @_;
3202         my $data = $resp->{'return'} || [];
3203         my $totalrdbytes = 0;
3204         my $totalwrbytes = 0;
3205
3206         for my $blockstat (@$data) {
3207             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3208             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3209
3210             $blockstat->{device} =~ s/drive-//;
3211             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3212         }
3213         $res->{$vmid}->{diskread} = $totalrdbytes;
3214         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3215     };
3216
3217     my $statuscb = sub {
3218         my ($vmid, $resp) = @_;
3219
3220         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3221         # this fails if ballon driver is not loaded, so this must be
3222         # the last commnand (following command are aborted if this fails).
3223         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3224
3225         my $status = 'unknown';
3226         if (!defined($status = $resp->{'return'}->{status})) {
3227             warn "unable to get VM status\n";
3228             return;
3229         }
3230
3231         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3232     };
3233
3234     foreach my $vmid (keys %$list) {
3235         next if $opt_vmid && ($vmid ne $opt_vmid);
3236         next if !$res->{$vmid}->{pid}; # not running
3237         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3238     }
3239
3240     $qmpclient->queue_execute(undef, 2);
3241
3242     foreach my $vmid (keys %$list) {
3243         next if $opt_vmid && ($vmid ne $opt_vmid);
3244         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3245     }
3246
3247     return $res;
3248 }
3249
3250 sub foreach_drive {
3251     my ($conf, $func, @param) = @_;
3252
3253     foreach my $ds (valid_drive_names()) {
3254         next if !defined($conf->{$ds});
3255
3256         my $drive = parse_drive($ds, $conf->{$ds});
3257         next if !$drive;
3258
3259         &$func($ds, $drive, @param);
3260     }
3261 }
3262
3263 sub foreach_volid {
3264     my ($conf, $func, @param) = @_;
3265
3266     my $volhash = {};
3267
3268     my $test_volid = sub {
3269         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3270
3271         return if !$volid;
3272
3273         $volhash->{$volid}->{cdrom} //= 1;
3274         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3275
3276         $volhash->{$volid}->{replicate} //= 0;
3277         $volhash->{$volid}->{replicate} = 1 if $replicate;
3278
3279         $volhash->{$volid}->{shared} //= 0;
3280         $volhash->{$volid}->{shared} = 1 if $shared;
3281
3282         $volhash->{$volid}->{referenced_in_config} //= 0;
3283         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3284
3285         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3286             if defined($snapname);
3287         $volhash->{$volid}->{size} = $size if $size;
3288     };
3289
3290     foreach_drive($conf, sub {
3291         my ($ds, $drive) = @_;
3292         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3293     });
3294
3295     foreach my $snapname (keys %{$conf->{snapshots}}) {
3296         my $snap = $conf->{snapshots}->{$snapname};
3297         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3298         foreach_drive($snap, sub {
3299             my ($ds, $drive) = @_;
3300             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3301         });
3302     }
3303
3304     foreach my $volid (keys %$volhash) {
3305         &$func($volid, $volhash->{$volid}, @param);
3306     }
3307 }
3308
3309 sub conf_has_serial {
3310     my ($conf) = @_;
3311
3312     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3313         if ($conf->{"serial$i"}) {
3314             return 1;
3315         }
3316     }
3317
3318     return 0;
3319 }
3320
3321 sub conf_has_audio {
3322     my ($conf, $id) = @_;
3323
3324     $id //= 0;
3325     my $audio = $conf->{"audio$id"};
3326     return undef if !defined($audio);
3327
3328     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3329     my $audiodriver = $audioproperties->{driver} // 'spice';
3330
3331     return {
3332         dev => $audioproperties->{device},
3333         dev_id => "audiodev$id",
3334         backend => $audiodriver,
3335         backend_id => "$audiodriver-backend${id}",
3336     };
3337 }
3338
3339 sub vga_conf_has_spice {
3340     my ($vga) = @_;
3341
3342     my $vgaconf = parse_vga($vga);
3343     my $vgatype = $vgaconf->{type};
3344     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3345
3346     return $1 || 1;
3347 }
3348
3349 sub is_native($) {
3350     my ($arch) = @_;
3351     return get_host_arch() eq $arch;
3352 }
3353
3354 sub get_vm_arch {
3355     my ($conf) = @_;
3356     return $conf->{arch} // get_host_arch();
3357 }
3358
3359 my $default_machines = {
3360     x86_64 => 'pc',
3361     aarch64 => 'virt',
3362 };
3363
3364 sub get_vm_machine {
3365     my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3366
3367     my $machine = $forcemachine || $conf->{machine};
3368
3369     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370         $arch //= 'x86_64';
3371         $machine ||= $default_machines->{$arch};
3372         $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3373     }
3374
3375     return $machine;
3376 }
3377
3378 sub get_ovmf_files($) {
3379     my ($arch) = @_;
3380
3381     my $ovmf = $OVMF->{$arch}
3382         or die "no OVMF images known for architecture '$arch'\n";
3383
3384     return @$ovmf;
3385 }
3386
3387 my $Arch2Qemu = {
3388     aarch64 => '/usr/bin/qemu-system-aarch64',
3389     x86_64 => '/usr/bin/qemu-system-x86_64',
3390 };
3391 sub get_command_for_arch($) {
3392     my ($arch) = @_;
3393     return '/usr/bin/kvm' if is_native($arch);
3394
3395     my $cmd = $Arch2Qemu->{$arch}
3396         or die "don't know how to emulate architecture '$arch'\n";
3397     return $cmd;
3398 }
3399
3400 sub get_cpu_options {
3401     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3402
3403     my $cpuFlags = [];
3404     my $ostype = $conf->{ostype};
3405
3406     my $cpu = $kvm ? "kvm64" : "qemu64";
3407     if ($arch eq 'aarch64') {
3408         $cpu = 'cortex-a57';
3409     }
3410     my $hv_vendor_id;
3411     if (my $cputype = $conf->{cpu}) {
3412         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3413             or die "Cannot parse cpu description: $cputype\n";
3414         $cpu = $cpuconf->{cputype};
3415         $kvm_off = 1 if $cpuconf->{hidden};
3416         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3417
3418         if (defined(my $flags = $cpuconf->{flags})) {
3419             push @$cpuFlags, split(";", $flags);
3420         }
3421     }
3422
3423     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3424
3425     push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3426
3427     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3428
3429     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3430
3431     if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
3432
3433         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3434         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3435     }
3436
3437     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3438
3439     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3440
3441     push @$cpuFlags, 'kvm=off' if $kvm_off;
3442
3443     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3444         push @$cpuFlags, "vendor=${cpu_vendor}"
3445             if $cpu_vendor ne 'default';
3446     } elsif ($arch ne 'aarch64') {
3447         die "internal error"; # should not happen
3448     }
3449
3450     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3451
3452     return ('-cpu', $cpu);
3453 }
3454
3455 sub config_to_command {
3456     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3457
3458     my $cmd = [];
3459     my $globalFlags = [];
3460     my $machineFlags = [];
3461     my $rtcFlags = [];
3462     my $devices = [];
3463     my $pciaddr = '';
3464     my $bridges = {};
3465     my $vernum = 0; # unknown
3466     my $ostype = $conf->{ostype};
3467     my $winversion = windows_version($ostype);
3468     my $kvm = $conf->{kvm};
3469     my $nodename = nodename();
3470
3471     my $arch = get_vm_arch($conf);
3472     my $kvm_binary = get_command_for_arch($arch);
3473     my $kvmver = kvm_user_version($kvm_binary);
3474
3475     my $add_pve_version = min_version($kvmver, 4, 1);
3476
3477     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3478     my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
3479     $kvm //= 1 if is_native($arch);
3480
3481     $machine_version =~ m/(\d+)\.(\d+)/;
3482     die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3483         if !PVE::QemuServer::min_version($kvmver, $1, $2);
3484
3485     if ($kvm) {
3486         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3487             if !defined kvm_version();
3488     }
3489
3490     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3491         $vernum = $1*1000000+$2*1000;
3492     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3493         $vernum = $1*1000000+$2*1000+$3;
3494     }
3495
3496     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3497
3498     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3499     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3500     my $use_old_bios_files = undef;
3501     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3502
3503     my $cpuunits = defined($conf->{cpuunits}) ?
3504             $conf->{cpuunits} : $defaults->{cpuunits};
3505
3506     push @$cmd, $kvm_binary;
3507
3508     push @$cmd, '-id', $vmid;
3509
3510     my $vmname = $conf->{name} || "vm$vmid";
3511
3512     push @$cmd, '-name', $vmname;
3513
3514     my $use_virtio = 0;
3515
3516     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3517     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3518     push @$cmd, '-mon', "chardev=qmp,mode=control";
3519
3520     if (min_version($machine_version, 2, 12)) {
3521         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3522         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3523     }
3524
3525     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3526
3527     push @$cmd, '-daemonize';
3528
3529     if ($conf->{smbios1}) {
3530         my $smbios_conf = parse_smbios1($conf->{smbios1});
3531         if ($smbios_conf->{base64}) {
3532             # Do not pass base64 flag to qemu
3533             delete $smbios_conf->{base64};
3534             my $smbios_string = "";
3535             foreach my $key (keys %$smbios_conf) {
3536                 my $value;
3537                 if ($key eq "uuid") {
3538                     $value = $smbios_conf->{uuid}
3539                 } else {
3540                     $value = decode_base64($smbios_conf->{$key});
3541                 }
3542                 # qemu accepts any binary data, only commas need escaping by double comma
3543                 $value =~ s/,/,,/g;
3544                 $smbios_string .= "," . $key . "=" . $value if $value;
3545             }
3546             push @$cmd, '-smbios', "type=1" . $smbios_string;
3547         } else {
3548             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3549         }
3550     }
3551
3552     if ($conf->{vmgenid}) {
3553         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3554     }
3555
3556     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3557     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3558         die "uefi base image not found\n" if ! -f $ovmf_code;
3559
3560         my $path;
3561         my $format;
3562         if (my $efidisk = $conf->{efidisk0}) {
3563             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3564             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3565             $format = $d->{format};
3566             if ($storeid) {
3567                 $path = PVE::Storage::path($storecfg, $d->{file});
3568                 if (!defined($format)) {
3569                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3570                     $format = qemu_img_format($scfg, $volname);
3571                 }
3572             } else {
3573                 $path = $d->{file};
3574                 die "efidisk format must be specified\n"
3575                     if !defined($format);
3576             }
3577         } else {
3578             warn "no efidisk configured! Using temporary efivars disk.\n";
3579             $path = "/tmp/$vmid-ovmf.fd";
3580             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3581             $format = 'raw';
3582         }
3583
3584         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3585         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3586     }
3587
3588     # load q35 config
3589     if ($q35) {
3590         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3591         if (min_version($machine_version, 4, 0)) {
3592             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3593         } else {
3594             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3595         }
3596     }
3597
3598     # add usb controllers
3599     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3600     push @$devices, @usbcontrollers if @usbcontrollers;
3601     my $vga = parse_vga($conf->{vga});
3602
3603     my $qxlnum = vga_conf_has_spice($conf->{vga});
3604     $vga->{type} = 'qxl' if $qxlnum;
3605
3606     if (!$vga->{type}) {
3607         if ($arch eq 'aarch64') {
3608             $vga->{type} = 'virtio';
3609         } elsif (min_version($machine_version, 2, 9)) {
3610             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3611         } else {
3612             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3613         }
3614     }
3615
3616     # enable absolute mouse coordinates (needed by vnc)
3617     my $tablet;
3618     if (defined($conf->{tablet})) {
3619         $tablet = $conf->{tablet};
3620     } else {
3621         $tablet = $defaults->{tablet};
3622         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3623         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3624     }
3625
3626     if ($tablet) {
3627         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3628         my $kbd = print_keyboarddevice_full($conf, $arch);
3629         push @$devices, '-device', $kbd if defined($kbd);
3630     }
3631
3632     my $kvm_off = 0;
3633     my $gpu_passthrough;
3634
3635     # host pci devices
3636     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3637         my $id = "hostpci$i";
3638         my $d = parse_hostpci($conf->{$id});
3639         next if !$d;
3640
3641         if (my $pcie = $d->{pcie}) {
3642             die "q35 machine model is not enabled" if !$q35;
3643             # win7 wants to have the pcie devices directly on the pcie bus
3644             # instead of in the root port
3645             if ($winversion == 7) {
3646                 $pciaddr = print_pcie_addr("${id}bus0");
3647             } else {
3648                 # add more root ports if needed, 4 are present by default
3649                 # by pve-q35 cfgs, rest added here on demand.
3650                 if ($i > 3) {
3651                     push @$devices, '-device', print_pcie_root_port($i);
3652                 }
3653                 $pciaddr = print_pcie_addr($id);
3654             }
3655         } else {
3656             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3657         }
3658
3659         my $xvga = '';
3660         if ($d->{'x-vga'}) {
3661             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3662             $kvm_off = 1;
3663             $vga->{type} = 'none' if !defined($conf->{vga});
3664             $gpu_passthrough = 1;
3665         }
3666
3667         my $pcidevices = $d->{pciid};
3668         my $multifunction = 1 if @$pcidevices > 1;
3669
3670         my $sysfspath;
3671         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3672             my $pci_id = $pcidevices->[0]->{id};
3673             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3674             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3675         } elsif ($d->{mdev}) {
3676             warn "ignoring mediated device '$id' with multifunction device\n";
3677         }
3678
3679         my $j=0;
3680         foreach my $pcidevice (@$pcidevices) {
3681             my $devicestr = "vfio-pci";
3682
3683             if ($sysfspath) {
3684                 $devicestr .= ",sysfsdev=$sysfspath";
3685             } else {
3686                 $devicestr .= ",host=$pcidevice->{id}";
3687             }
3688
3689             my $mf_addr = $multifunction ? ".$j" : '';
3690             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3691
3692             if ($j == 0) {
3693                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3694                 $devicestr .= "$xvga";
3695                 $devicestr .= ",multifunction=on" if $multifunction;
3696                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3697             }
3698
3699             push @$devices, '-device', $devicestr;
3700             $j++;
3701         }
3702     }
3703
3704     # usb devices
3705     my $usb_dev_features = {};
3706     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3707
3708     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3709     push @$devices, @usbdevices if @usbdevices;
3710     # serial devices
3711     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3712         if (my $path = $conf->{"serial$i"}) {
3713             if ($path eq 'socket') {
3714                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3715                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3716                 # On aarch64, serial0 is the UART device. Qemu only allows
3717                 # connecting UART devices via the '-serial' command line, as
3718                 # the device has a fixed slot on the hardware...
3719                 if ($arch eq 'aarch64' && $i == 0) {
3720                     push @$devices, '-serial', "chardev:serial$i";
3721                 } else {
3722                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3723                 }
3724             } else {
3725                 die "no such serial device\n" if ! -c $path;
3726                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3727                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3728             }
3729         }
3730     }
3731
3732     # parallel devices
3733     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3734         if (my $path = $conf->{"parallel$i"}) {
3735             die "no such parallel device\n" if ! -c $path;
3736             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3737             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3738             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3739         }
3740     }
3741
3742     if (my $audio = conf_has_audio($conf)) {
3743
3744         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3745
3746         my $id = $audio->{dev_id};
3747         if ($audio->{dev} eq 'AC97') {
3748             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3749         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3750             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3751             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3752             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3753         } else {
3754             die "unkown audio device '$audio->{dev}', implement me!";
3755         }
3756
3757         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3758     }
3759
3760     my $sockets = 1;
3761     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3762     $sockets = $conf->{sockets} if  $conf->{sockets};
3763
3764     my $cores = $conf->{cores} || 1;
3765
3766     my $maxcpus = $sockets * $cores;
3767
3768     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3769
3770     my $allowed_vcpus = $cpuinfo->{cpus};
3771
3772     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3773         if ($allowed_vcpus < $maxcpus);
3774
3775     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3776
3777         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3778         for (my $i = 2; $i <= $vcpus; $i++)  {
3779             my $cpustr = print_cpu_device($conf,$i);
3780             push @$cmd, '-device', $cpustr;
3781         }
3782
3783     } else {
3784
3785         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3786     }
3787     push @$cmd, '-nodefaults';
3788
3789     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3790
3791     my $bootindex_hash = {};
3792     my $i = 1;
3793     foreach my $o (split(//, $bootorder)) {
3794         $bootindex_hash->{$o} = $i*100;
3795         $i++;
3796     }
3797
3798     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3799
3800     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3801
3802     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3803
3804     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3805         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3806         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3807         push @$cmd,  '-vnc', "unix:$socket,password";
3808     } else {
3809         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3810         push @$cmd, '-nographic';
3811     }
3812
3813     # time drift fix
3814     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3815
3816     my $useLocaltime = $conf->{localtime};
3817
3818     if ($winversion >= 5) { # windows
3819         $useLocaltime = 1 if !defined($conf->{localtime});
3820
3821         # use time drift fix when acpi is enabled
3822         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3823             $tdf = 1 if !defined($conf->{tdf});
3824         }
3825     }
3826
3827     if ($winversion >= 6) {
3828         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3829         push @$cmd, '-no-hpet';
3830     }
3831
3832     push @$rtcFlags, 'driftfix=slew' if $tdf;
3833
3834     if (!$kvm) {
3835         push @$machineFlags, 'accel=tcg';
3836     }
3837
3838     if ($machine_type) {
3839         push @$machineFlags, "type=${machine_type}";
3840     }
3841
3842     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3843         push @$rtcFlags, "base=$conf->{startdate}";
3844     } elsif ($useLocaltime) {
3845         push @$rtcFlags, 'base=localtime';
3846     }
3847
3848     push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3849
3850     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3851
3852     push @$cmd, '-S' if $conf->{freeze};
3853
3854     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3855
3856     my $guest_agent = parse_guest_agent($conf);
3857
3858     if ($guest_agent->{enabled}) {
3859         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3860         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3861
3862         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3863             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3864             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3865             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3866         } elsif ($guest_agent->{type} eq 'isa') {
3867             push @$devices, '-device', "isa-serial,chardev=qga0";
3868         }
3869     }
3870
3871     my $spice_port;
3872
3873     if ($qxlnum) {
3874         if ($qxlnum > 1) {
3875             if ($winversion){
3876                 for(my $i = 1; $i < $qxlnum; $i++){
3877                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3878                 }
3879             } else {
3880                 # assume other OS works like Linux
3881                 my ($ram, $vram) = ("134217728", "67108864");
3882                 if ($vga->{memory}) {
3883                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3884                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3885                 }
3886                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3887                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3888             }
3889         }
3890
3891         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3892
3893         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3894         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3895         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3896
3897         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3898         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3899         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3900
3901         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3902         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3903
3904         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3905         if ($spice_enhancement->{foldersharing}) {
3906             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3907             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3908         }
3909
3910         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3911         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3912         push @$devices, '-spice', "$spice_opts";
3913     }
3914
3915     # enable balloon by default, unless explicitly disabled
3916     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3917         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3918         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3919     }
3920
3921     if ($conf->{watchdog}) {
3922         my $wdopts = parse_watchdog($conf->{watchdog});
3923         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3924         my $watchdog = $wdopts->{model} || 'i6300esb';
3925         push @$devices, '-device', "$watchdog$pciaddr";
3926         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3927     }
3928
3929     my $vollist = [];
3930     my $scsicontroller = {};
3931     my $ahcicontroller = {};
3932     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3933
3934     # Add iscsi initiator name if available
3935     if (my $initiator = get_initiator_name()) {
3936         push @$devices, '-iscsi', "initiator-name=$initiator";
3937     }
3938
3939     foreach_drive($conf, sub {
3940         my ($ds, $drive) = @_;
3941
3942         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3943             push @$vollist, $drive->{file};
3944         }
3945
3946         # ignore efidisk here, already added in bios/fw handling code above
3947         return if $drive->{interface} eq 'efidisk';
3948
3949         $use_virtio = 1 if $ds =~ m/^virtio/;
3950
3951         if (drive_is_cdrom ($drive)) {
3952             if ($bootindex_hash->{d}) {
3953                 $drive->{bootindex} = $bootindex_hash->{d};
3954                 $bootindex_hash->{d} += 1;
3955             }
3956         } else {
3957             if ($bootindex_hash->{c}) {
3958                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3959                 $bootindex_hash->{c} += 1;
3960             }
3961         }
3962
3963         if($drive->{interface} eq 'virtio'){
3964            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3965         }
3966
3967         if ($drive->{interface} eq 'scsi') {
3968
3969             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3970
3971             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3972             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3973
3974             my $iothread = '';
3975             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3976                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3977                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3978             } elsif ($drive->{iothread}) {
3979                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3980             }
3981
3982             my $queues = '';
3983             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3984                 $queues = ",num_queues=$drive->{queues}";
3985             }
3986
3987             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3988             $scsicontroller->{$controller}=1;
3989         }
3990
3991         if ($drive->{interface} eq 'sata') {
3992            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3993            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3994            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3995            $ahcicontroller->{$controller}=1;
3996         }
3997
3998         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3999         push @$devices, '-drive',$drive_cmd;
4000         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4001     });
4002
4003     for (my $i = 0; $i < $MAX_NETS; $i++) {
4004          next if !$conf->{"net$i"};
4005          my $d = parse_net($conf->{"net$i"});
4006          next if !$d;
4007
4008          $use_virtio = 1 if $d->{model} eq 'virtio';
4009
4010          if ($bootindex_hash->{n}) {
4011             $d->{bootindex} = $bootindex_hash->{n};
4012             $bootindex_hash->{n} += 1;
4013          }
4014
4015          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4016          push @$devices, '-netdev', $netdevfull;
4017
4018          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4019          push @$devices, '-device', $netdevicefull;
4020     }
4021
4022     if ($conf->{ivshmem}) {
4023         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4024
4025         my $bus;
4026         if ($q35) {
4027             $bus = print_pcie_addr("ivshmem");
4028         } else {
4029             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4030         }
4031
4032         my $ivshmem_name = $ivshmem->{name} // $vmid;
4033         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4034
4035         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4036         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4037     }
4038
4039     if (!$q35) {
4040         # add pci bridges
4041         if (min_version($machine_version, 2, 3)) {
4042            $bridges->{1} = 1;
4043            $bridges->{2} = 1;
4044         }
4045
4046         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4047
4048         for my $k (sort {$b cmp $a} keys %$bridges) {
4049             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4050             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4051         }
4052     }
4053
4054     push @$cmd, @$devices;
4055     push @$cmd, '-rtc', join(',', @$rtcFlags)
4056         if scalar(@$rtcFlags);
4057     push @$cmd, '-machine', join(',', @$machineFlags)
4058         if scalar(@$machineFlags);
4059     push @$cmd, '-global', join(',', @$globalFlags)
4060         if scalar(@$globalFlags);
4061
4062     if (my $vmstate = $conf->{vmstate}) {
4063         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4064         push @$vollist, $vmstate;
4065         push @$cmd, '-loadstate', $statepath;
4066         print "activating and using '$vmstate' as vmstate\n";
4067     }
4068
4069     # add custom args
4070     if ($conf->{args}) {
4071         my $aa = PVE::Tools::split_args($conf->{args});
4072         push @$cmd, @$aa;
4073     }
4074
4075     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4076 }
4077
4078 sub spice_port {
4079     my ($vmid) = @_;
4080
4081     my $res = mon_cmd($vmid, 'query-spice');
4082
4083     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4084 }
4085
4086 sub vm_devices_list {
4087     my ($vmid) = @_;
4088
4089     my $res = mon_cmd($vmid, 'query-pci');
4090     my $devices_to_check = [];
4091     my $devices = {};
4092     foreach my $pcibus (@$res) {
4093         push @$devices_to_check, @{$pcibus->{devices}},
4094     }
4095
4096     while (@$devices_to_check) {
4097         my $to_check = [];
4098         for my $d (@$devices_to_check) {
4099             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4100             next if !$d->{'pci_bridge'};
4101
4102             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4103             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4104         }
4105         $devices_to_check = $to_check;
4106     }
4107
4108     my $resblock = mon_cmd($vmid, 'query-block');
4109     foreach my $block (@$resblock) {
4110         if($block->{device} =~ m/^drive-(\S+)/){
4111                 $devices->{$1} = 1;
4112         }
4113     }
4114
4115     my $resmice = mon_cmd($vmid, 'query-mice');
4116     foreach my $mice (@$resmice) {
4117         if ($mice->{name} eq 'QEMU HID Tablet') {
4118             $devices->{tablet} = 1;
4119             last;
4120         }
4121     }
4122
4123     # for usb devices there is no query-usb
4124     # but we can iterate over the entries in
4125     # qom-list path=/machine/peripheral
4126     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4127     foreach my $per (@$resperipheral) {
4128         if ($per->{name} =~ m/^usb\d+$/) {
4129             $devices->{$per->{name}} = 1;
4130         }
4131     }
4132
4133     return $devices;
4134 }
4135
4136 sub vm_deviceplug {
4137     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4138
4139     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4140
4141     my $devices_list = vm_devices_list($vmid);
4142     return 1 if defined($devices_list->{$deviceid});
4143
4144     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4145
4146     if ($deviceid eq 'tablet') {
4147
4148         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4149
4150     } elsif ($deviceid eq 'keyboard') {
4151
4152         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4153
4154     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4155
4156         die "usb hotplug currently not reliable\n";
4157         # since we can't reliably hot unplug all added usb devices
4158         # and usb passthrough disables live migration
4159         # we disable usb hotplugging for now
4160         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4161
4162     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4163
4164         qemu_iothread_add($vmid, $deviceid, $device);
4165
4166         qemu_driveadd($storecfg, $vmid, $device);
4167         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4168
4169         qemu_deviceadd($vmid, $devicefull);
4170         eval { qemu_deviceaddverify($vmid, $deviceid); };
4171         if (my $err = $@) {
4172             eval { qemu_drivedel($vmid, $deviceid); };
4173             warn $@ if $@;
4174             die $err;
4175         }
4176
4177     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4178
4179
4180         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4181         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4182         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4183
4184         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4185
4186         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4187             qemu_iothread_add($vmid, $deviceid, $device);
4188             $devicefull .= ",iothread=iothread-$deviceid";
4189         }
4190
4191         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4192             $devicefull .= ",num_queues=$device->{queues}";
4193         }
4194
4195         qemu_deviceadd($vmid, $devicefull);
4196         qemu_deviceaddverify($vmid, $deviceid);
4197
4198     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4199
4200         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4201         qemu_driveadd($storecfg, $vmid, $device);
4202
4203         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4204         eval { qemu_deviceadd($vmid, $devicefull); };
4205         if (my $err = $@) {
4206             eval { qemu_drivedel($vmid, $deviceid); };
4207             warn $@ if $@;
4208             die $err;
4209         }
4210
4211     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4212
4213         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4214
4215         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4216         my $use_old_bios_files = undef;
4217         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4218
4219         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4220         qemu_deviceadd($vmid, $netdevicefull);
4221         eval {
4222             qemu_deviceaddverify($vmid, $deviceid);
4223             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4224         };
4225         if (my $err = $@) {
4226             eval { qemu_netdevdel($vmid, $deviceid); };
4227             warn $@ if $@;
4228             die $err;
4229         }
4230
4231     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4232
4233         my $bridgeid = $2;
4234         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4235         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4236
4237         qemu_deviceadd($vmid, $devicefull);
4238         qemu_deviceaddverify($vmid, $deviceid);
4239
4240     } else {
4241         die "can't hotplug device '$deviceid'\n";
4242     }
4243
4244     return 1;
4245 }
4246
4247 # fixme: this should raise exceptions on error!
4248 sub vm_deviceunplug {
4249     my ($vmid, $conf, $deviceid) = @_;
4250
4251     my $devices_list = vm_devices_list($vmid);
4252     return 1 if !defined($devices_list->{$deviceid});
4253
4254     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4255
4256     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4257
4258         qemu_devicedel($vmid, $deviceid);
4259
4260     } elsif ($deviceid =~ m/^usb\d+$/) {
4261
4262         die "usb hotplug currently not reliable\n";
4263         # when unplugging usb devices this way,
4264         # there may be remaining usb controllers/hubs
4265         # so we disable it for now
4266         qemu_devicedel($vmid, $deviceid);
4267         qemu_devicedelverify($vmid, $deviceid);
4268
4269     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4270
4271         qemu_devicedel($vmid, $deviceid);
4272         qemu_devicedelverify($vmid, $deviceid);
4273         qemu_drivedel($vmid, $deviceid);
4274         qemu_iothread_del($conf, $vmid, $deviceid);
4275
4276     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4277
4278         qemu_devicedel($vmid, $deviceid);
4279         qemu_devicedelverify($vmid, $deviceid);
4280         qemu_iothread_del($conf, $vmid, $deviceid);
4281
4282     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4283
4284         qemu_devicedel($vmid, $deviceid);
4285         qemu_drivedel($vmid, $deviceid);
4286         qemu_deletescsihw($conf, $vmid, $deviceid);
4287
4288     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4289
4290         qemu_devicedel($vmid, $deviceid);
4291         qemu_devicedelverify($vmid, $deviceid);
4292         qemu_netdevdel($vmid, $deviceid);
4293
4294     } else {
4295         die "can't unplug device '$deviceid'\n";
4296     }
4297
4298     return 1;
4299 }
4300
4301 sub qemu_deviceadd {
4302     my ($vmid, $devicefull) = @_;
4303
4304     $devicefull = "driver=".$devicefull;
4305     my %options =  split(/[=,]/, $devicefull);
4306
4307     mon_cmd($vmid, "device_add" , %options);
4308 }
4309
4310 sub qemu_devicedel {
4311     my ($vmid, $deviceid) = @_;
4312
4313     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4314 }
4315
4316 sub qemu_iothread_add {
4317     my($vmid, $deviceid, $device) = @_;
4318
4319     if ($device->{iothread}) {
4320         my $iothreads = vm_iothreads_list($vmid);
4321         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4322     }
4323 }
4324
4325 sub qemu_iothread_del {
4326     my($conf, $vmid, $deviceid) = @_;
4327
4328     my $confid = $deviceid;
4329     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4330         $confid = 'scsi' . $1;
4331     }
4332     my $device = parse_drive($confid, $conf->{$confid});
4333     if ($device->{iothread}) {
4334         my $iothreads = vm_iothreads_list($vmid);
4335         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4336     }
4337 }
4338
4339 sub qemu_objectadd {
4340     my($vmid, $objectid, $qomtype) = @_;
4341
4342     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4343
4344     return 1;
4345 }
4346
4347 sub qemu_objectdel {
4348     my($vmid, $objectid) = @_;
4349
4350     mon_cmd($vmid, "object-del", id => $objectid);
4351
4352     return 1;
4353 }
4354
4355 sub qemu_driveadd {
4356     my ($storecfg, $vmid, $device) = @_;
4357
4358     my $drive = print_drive_full($storecfg, $vmid, $device);
4359     $drive =~ s/\\/\\\\/g;
4360     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4361
4362     # If the command succeeds qemu prints: "OK"
4363     return 1 if $ret =~ m/OK/s;
4364
4365     die "adding drive failed: $ret\n";
4366 }
4367
4368 sub qemu_drivedel {
4369     my($vmid, $deviceid) = @_;
4370
4371     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4372     $ret =~ s/^\s+//;
4373
4374     return 1 if $ret eq "";
4375
4376     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4377     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4378
4379     die "deleting drive $deviceid failed : $ret\n";
4380 }
4381
4382 sub qemu_deviceaddverify {
4383     my ($vmid, $deviceid) = @_;
4384
4385     for (my $i = 0; $i <= 5; $i++) {
4386          my $devices_list = vm_devices_list($vmid);
4387          return 1 if defined($devices_list->{$deviceid});
4388          sleep 1;
4389     }
4390
4391     die "error on hotplug device '$deviceid'\n";
4392 }
4393
4394
4395 sub qemu_devicedelverify {
4396     my ($vmid, $deviceid) = @_;
4397
4398     # need to verify that the device is correctly removed as device_del
4399     # is async and empty return is not reliable
4400
4401     for (my $i = 0; $i <= 5; $i++) {
4402          my $devices_list = vm_devices_list($vmid);
4403          return 1 if !defined($devices_list->{$deviceid});
4404          sleep 1;
4405     }
4406
4407     die "error on hot-unplugging device '$deviceid'\n";
4408 }
4409
4410 sub qemu_findorcreatescsihw {
4411     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4412
4413     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4414
4415     my $scsihwid="$controller_prefix$controller";
4416     my $devices_list = vm_devices_list($vmid);
4417
4418     if(!defined($devices_list->{$scsihwid})) {
4419         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4420     }
4421
4422     return 1;
4423 }
4424
4425 sub qemu_deletescsihw {
4426     my ($conf, $vmid, $opt) = @_;
4427
4428     my $device = parse_drive($opt, $conf->{$opt});
4429
4430     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4431         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4432         return 1;
4433     }
4434
4435     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4436
4437     my $devices_list = vm_devices_list($vmid);
4438     foreach my $opt (keys %{$devices_list}) {
4439         if (PVE::QemuServer::is_valid_drivename($opt)) {
4440             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4441             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4442                 return 1;
4443             }
4444         }
4445     }
4446
4447     my $scsihwid="scsihw$controller";
4448
4449     vm_deviceunplug($vmid, $conf, $scsihwid);
4450
4451     return 1;
4452 }
4453
4454 sub qemu_add_pci_bridge {
4455     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4456
4457     my $bridges = {};
4458
4459     my $bridgeid;
4460
4461     print_pci_addr($device, $bridges, $arch, $machine_type);
4462
4463     while (my ($k, $v) = each %$bridges) {
4464         $bridgeid = $k;
4465     }
4466     return 1 if !defined($bridgeid) || $bridgeid < 1;
4467
4468     my $bridge = "pci.$bridgeid";
4469     my $devices_list = vm_devices_list($vmid);
4470
4471     if (!defined($devices_list->{$bridge})) {
4472         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4473     }
4474
4475     return 1;
4476 }
4477
4478 sub qemu_set_link_status {
4479     my ($vmid, $device, $up) = @_;
4480
4481     mon_cmd($vmid, "set_link", name => $device,
4482                up => $up ? JSON::true : JSON::false);
4483 }
4484
4485 sub qemu_netdevadd {
4486     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4487
4488     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4489     my %options =  split(/[=,]/, $netdev);
4490
4491     mon_cmd($vmid, "netdev_add",  %options);
4492     return 1;
4493 }
4494
4495 sub qemu_netdevdel {
4496     my ($vmid, $deviceid) = @_;
4497
4498     mon_cmd($vmid, "netdev_del", id => $deviceid);
4499 }
4500
4501 sub qemu_usb_hotplug {
4502     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4503
4504     return if !$device;
4505
4506     # remove the old one first
4507     vm_deviceunplug($vmid, $conf, $deviceid);
4508
4509     # check if xhci controller is necessary and available
4510     if ($device->{usb3}) {
4511
4512         my $devicelist = vm_devices_list($vmid);
4513
4514         if (!$devicelist->{xhci}) {
4515             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4516             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4517         }
4518     }
4519     my $d = parse_usb_device($device->{host});
4520     $d->{usb3} = $device->{usb3};
4521
4522     # add the new one
4523     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4524 }
4525
4526 sub qemu_cpu_hotplug {
4527     my ($vmid, $conf, $vcpus) = @_;
4528
4529     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4530
4531     my $sockets = 1;
4532     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4533     $sockets = $conf->{sockets} if  $conf->{sockets};
4534     my $cores = $conf->{cores} || 1;
4535     my $maxcpus = $sockets * $cores;
4536
4537     $vcpus = $maxcpus if !$vcpus;
4538
4539     die "you can't add more vcpus than maxcpus\n"
4540         if $vcpus > $maxcpus;
4541
4542     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4543
4544     if ($vcpus < $currentvcpus) {
4545
4546         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4547
4548             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4549                 qemu_devicedel($vmid, "cpu$i");
4550                 my $retry = 0;
4551                 my $currentrunningvcpus = undef;
4552                 while (1) {
4553                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4554                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4555                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4556                     $retry++;
4557                     sleep 1;
4558                 }
4559                 #update conf after each succesfull cpu unplug
4560                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4561                 PVE::QemuConfig->write_config($vmid, $conf);
4562             }
4563         } else {
4564             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4565         }
4566
4567         return;
4568     }
4569
4570     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4571     die "vcpus in running vm does not match its configuration\n"
4572         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4573
4574     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4575
4576         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4577             my $cpustr = print_cpu_device($conf, $i);
4578             qemu_deviceadd($vmid, $cpustr);
4579
4580             my $retry = 0;
4581             my $currentrunningvcpus = undef;
4582             while (1) {
4583                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4584                 last if scalar(@{$currentrunningvcpus}) == $i;
4585                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4586                 sleep 1;
4587                 $retry++;
4588             }
4589             #update conf after each succesfull cpu hotplug
4590             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4591             PVE::QemuConfig->write_config($vmid, $conf);
4592         }
4593     } else {
4594
4595         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4596             mon_cmd($vmid, "cpu-add", id => int($i));
4597         }
4598     }
4599 }
4600
4601 sub qemu_block_set_io_throttle {
4602     my ($vmid, $deviceid,
4603         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4604         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4605         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4606         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4607
4608     return if !check_running($vmid) ;
4609
4610     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4611         bps => int($bps),
4612         bps_rd => int($bps_rd),
4613         bps_wr => int($bps_wr),
4614         iops => int($iops),
4615         iops_rd => int($iops_rd),
4616         iops_wr => int($iops_wr),
4617         bps_max => int($bps_max),
4618         bps_rd_max => int($bps_rd_max),
4619         bps_wr_max => int($bps_wr_max),
4620         iops_max => int($iops_max),
4621         iops_rd_max => int($iops_rd_max),
4622         iops_wr_max => int($iops_wr_max),
4623         bps_max_length => int($bps_max_length),
4624         bps_rd_max_length => int($bps_rd_max_length),
4625         bps_wr_max_length => int($bps_wr_max_length),
4626         iops_max_length => int($iops_max_length),
4627         iops_rd_max_length => int($iops_rd_max_length),
4628         iops_wr_max_length => int($iops_wr_max_length),
4629     );
4630
4631 }
4632
4633 # old code, only used to shutdown old VM after update
4634 sub __read_avail {
4635     my ($fh, $timeout) = @_;
4636
4637     my $sel = new IO::Select;
4638     $sel->add($fh);
4639
4640     my $res = '';
4641     my $buf;
4642
4643     my @ready;
4644     while (scalar (@ready = $sel->can_read($timeout))) {
4645         my $count;
4646         if ($count = $fh->sysread($buf, 8192)) {
4647             if ($buf =~ /^(.*)\(qemu\) $/s) {
4648                 $res .= $1;
4649                 last;
4650             } else {
4651                 $res .= $buf;
4652             }
4653         } else {
4654             if (!defined($count)) {
4655                 die "$!\n";
4656             }
4657             last;
4658         }
4659     }
4660
4661     die "monitor read timeout\n" if !scalar(@ready);
4662
4663     return $res;
4664 }
4665
4666 sub qemu_block_resize {
4667     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4668
4669     my $running = check_running($vmid);
4670
4671     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4672
4673     return if !$running;
4674
4675     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4676
4677 }
4678
4679 sub qemu_volume_snapshot {
4680     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4681
4682     my $running = check_running($vmid);
4683
4684     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4685         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4686     } else {
4687         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4688     }
4689 }
4690
4691 sub qemu_volume_snapshot_delete {
4692     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4693
4694     my $running = check_running($vmid);
4695
4696     if($running) {
4697
4698         $running = undef;
4699         my $conf = PVE::QemuConfig->load_config($vmid);
4700         foreach_drive($conf, sub {
4701             my ($ds, $drive) = @_;
4702             $running = 1 if $drive->{file} eq $volid;
4703         });
4704     }
4705
4706     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4707         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4708     } else {
4709         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4710     }
4711 }
4712
4713 sub set_migration_caps {
4714     my ($vmid) = @_;
4715
4716     my $cap_ref = [];
4717
4718     my $enabled_cap = {
4719         "auto-converge" => 1,
4720         "xbzrle" => 1,
4721         "x-rdma-pin-all" => 0,
4722         "zero-blocks" => 0,
4723         "compress" => 0
4724     };
4725
4726     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4727
4728     for my $supported_capability (@$supported_capabilities) {
4729         push @$cap_ref, {
4730             capability => $supported_capability->{capability},
4731             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4732         };
4733     }
4734
4735     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4736 }
4737
4738 my $fast_plug_option = {
4739     'lock' => 1,
4740     'name' => 1,
4741     'onboot' => 1,
4742     'shares' => 1,
4743     'startup' => 1,
4744     'description' => 1,
4745     'protection' => 1,
4746     'vmstatestorage' => 1,
4747     'hookscript' => 1,
4748     'tags' => 1,
4749 };
4750
4751 # hotplug changes in [PENDING]
4752 # $selection hash can be used to only apply specified options, for
4753 # example: { cores => 1 } (only apply changed 'cores')
4754 # $errors ref is used to return error messages
4755 sub vmconfig_hotplug_pending {
4756     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4757
4758     my $defaults = load_defaults();
4759     my $arch = get_vm_arch($conf);
4760     my $machine_type = get_vm_machine($conf, undef, $arch);
4761
4762     # commit values which do not have any impact on running VM first
4763     # Note: those option cannot raise errors, we we do not care about
4764     # $selection and always apply them.
4765
4766     my $add_error = sub {
4767         my ($opt, $msg) = @_;
4768         $errors->{$opt} = "hotplug problem - $msg";
4769     };
4770
4771     my $changes = 0;
4772     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4773         if ($fast_plug_option->{$opt}) {
4774             $conf->{$opt} = $conf->{pending}->{$opt};
4775             delete $conf->{pending}->{$opt};
4776             $changes = 1;
4777         }
4778     }
4779
4780     if ($changes) {
4781         PVE::QemuConfig->write_config($vmid, $conf);
4782         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4783     }
4784
4785     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4786
4787     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4788     foreach my $opt (sort keys %$pending_delete_hash) {
4789         next if $selection && !$selection->{$opt};
4790         my $force = $pending_delete_hash->{$opt}->{force};
4791         eval {
4792             if ($opt eq 'hotplug') {
4793                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4794             } elsif ($opt eq 'tablet') {
4795                 die "skip\n" if !$hotplug_features->{usb};
4796                 if ($defaults->{tablet}) {
4797                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4798                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4799                         if $arch eq 'aarch64';
4800                 } else {
4801                     vm_deviceunplug($vmid, $conf, 'tablet');
4802                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4803                 }
4804             } elsif ($opt =~ m/^usb\d+/) {
4805                 die "skip\n";
4806                 # since we cannot reliably hot unplug usb devices
4807                 # we are disabling it
4808                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4809                 vm_deviceunplug($vmid, $conf, $opt);
4810             } elsif ($opt eq 'vcpus') {
4811                 die "skip\n" if !$hotplug_features->{cpu};
4812                 qemu_cpu_hotplug($vmid, $conf, undef);
4813             } elsif ($opt eq 'balloon') {
4814                 # enable balloon device is not hotpluggable
4815                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4816                 # here we reset the ballooning value to memory
4817                 my $balloon = $conf->{memory} || $defaults->{memory};
4818                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4819             } elsif ($fast_plug_option->{$opt}) {
4820                 # do nothing
4821             } elsif ($opt =~ m/^net(\d+)$/) {
4822                 die "skip\n" if !$hotplug_features->{network};
4823                 vm_deviceunplug($vmid, $conf, $opt);
4824             } elsif (is_valid_drivename($opt)) {
4825                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4826                 vm_deviceunplug($vmid, $conf, $opt);
4827                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4828             } elsif ($opt =~ m/^memory$/) {
4829                 die "skip\n" if !$hotplug_features->{memory};
4830                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4831             } elsif ($opt eq 'cpuunits') {
4832                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4833             } elsif ($opt eq 'cpulimit') {
4834                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4835             } else {
4836                 die "skip\n";
4837             }
4838         };
4839         if (my $err = $@) {
4840             &$add_error($opt, $err) if $err ne "skip\n";
4841         } else {
4842             # save new config if hotplug was successful
4843             delete $conf->{$opt};
4844             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4845             PVE::QemuConfig->write_config($vmid, $conf);
4846             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4847         }
4848     }
4849
4850     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4851     $apply_pending_cloudinit = sub {
4852         return if $apply_pending_cloudinit_done; # once is enough
4853         $apply_pending_cloudinit_done = 1; # once is enough
4854
4855         my ($key, $value) = @_;
4856
4857         my @cloudinit_opts = keys %$confdesc_cloudinit;
4858         foreach my $opt (keys %{$conf->{pending}}) {
4859             next if !grep { $_ eq $opt } @cloudinit_opts;
4860             $conf->{$opt} = delete $conf->{pending}->{$opt};
4861         }
4862
4863         my $new_conf = { %$conf };
4864         $new_conf->{$key} = $value;
4865         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4866     };
4867
4868     foreach my $opt (keys %{$conf->{pending}}) {
4869         next if $selection && !$selection->{$opt};
4870         my $value = $conf->{pending}->{$opt};
4871         eval {
4872             if ($opt eq 'hotplug') {
4873                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4874             } elsif ($opt eq 'tablet') {
4875                 die "skip\n" if !$hotplug_features->{usb};
4876                 if ($value == 1) {
4877                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4878                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4879                         if $arch eq 'aarch64';
4880                 } elsif ($value == 0) {
4881                     vm_deviceunplug($vmid, $conf, 'tablet');
4882                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4883                 }
4884             } elsif ($opt =~ m/^usb\d+$/) {
4885                 die "skip\n";
4886                 # since we cannot reliably hot unplug usb devices
4887                 # we are disabling it
4888                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4889                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4890                 die "skip\n" if !$d;
4891                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4892             } elsif ($opt eq 'vcpus') {
4893                 die "skip\n" if !$hotplug_features->{cpu};
4894                 qemu_cpu_hotplug($vmid, $conf, $value);
4895             } elsif ($opt eq 'balloon') {
4896                 # enable/disable balloning device is not hotpluggable
4897                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4898                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4899                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4900
4901                 # allow manual ballooning if shares is set to zero
4902                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4903                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4904                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4905                 }
4906             } elsif ($opt =~ m/^net(\d+)$/) {
4907                 # some changes can be done without hotplug
4908                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4909                                     $vmid, $opt, $value, $arch, $machine_type);
4910             } elsif (is_valid_drivename($opt)) {
4911                 die "skip\n" if $opt eq 'efidisk0';
4912                 # some changes can be done without hotplug
4913                 my $drive = parse_drive($opt, $value);
4914                 if (drive_is_cloudinit($drive)) {
4915                     &$apply_pending_cloudinit($opt, $value);
4916                 }
4917                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4918                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4919             } elsif ($opt =~ m/^memory$/) { #dimms
4920                 die "skip\n" if !$hotplug_features->{memory};
4921                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4922             } elsif ($opt eq 'cpuunits') {
4923                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4924             } elsif ($opt eq 'cpulimit') {
4925                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4926                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4927             } else {
4928                 die "skip\n";  # skip non-hot-pluggable options
4929             }
4930         };
4931         if (my $err = $@) {
4932             &$add_error($opt, $err) if $err ne "skip\n";
4933         } else {
4934             # save new config if hotplug was successful
4935             $conf->{$opt} = $value;
4936             delete $conf->{pending}->{$opt};
4937             PVE::QemuConfig->write_config($vmid, $conf);
4938             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4939         }
4940     }
4941 }
4942
4943 sub try_deallocate_drive {
4944     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4945
4946     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4947         my $volid = $drive->{file};
4948         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4949             my $sid = PVE::Storage::parse_volume_id($volid);
4950             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4951
4952             # check if the disk is really unused
4953             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4954                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4955             PVE::Storage::vdisk_free($storecfg, $volid);
4956             return 1;
4957         } else {
4958             # If vm is not owner of this disk remove from config
4959             return 1;
4960         }
4961     }
4962
4963     return undef;
4964 }
4965
4966 sub vmconfig_delete_or_detach_drive {
4967     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4968
4969     my $drive = parse_drive($opt, $conf->{$opt});
4970
4971     my $rpcenv = PVE::RPCEnvironment::get();
4972     my $authuser = $rpcenv->get_user();
4973
4974     if ($force) {
4975         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4976         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4977     } else {
4978         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4979     }
4980 }
4981
4982
4983
4984 sub vmconfig_apply_pending {
4985     my ($vmid, $conf, $storecfg) = @_;
4986
4987     # cold plug
4988
4989     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4990     foreach my $opt (sort keys %$pending_delete_hash) {
4991         die "internal error" if $opt =~ m/^unused/;
4992         my $force = $pending_delete_hash->{$opt}->{force};
4993         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4994         if (!defined($conf->{$opt})) {
4995             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4996             PVE::QemuConfig->write_config($vmid, $conf);
4997         } elsif (is_valid_drivename($opt)) {
4998             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4999             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5000             delete $conf->{$opt};
5001             PVE::QemuConfig->write_config($vmid, $conf);
5002         } else {
5003             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5004             delete $conf->{$opt};
5005             PVE::QemuConfig->write_config($vmid, $conf);
5006         }
5007     }
5008
5009     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5010
5011     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5012         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5013
5014         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5015             # skip if nothing changed
5016         } elsif (is_valid_drivename($opt)) {
5017             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5018                 if defined($conf->{$opt});
5019             $conf->{$opt} = $conf->{pending}->{$opt};
5020         } else {
5021             $conf->{$opt} = $conf->{pending}->{$opt};
5022         }
5023
5024         delete $conf->{pending}->{$opt};
5025         PVE::QemuConfig->write_config($vmid, $conf);
5026     }
5027 }
5028
5029 my $safe_num_ne = sub {
5030     my ($a, $b) = @_;
5031
5032     return 0 if !defined($a) && !defined($b);
5033     return 1 if !defined($a);
5034     return 1 if !defined($b);
5035
5036     return $a != $b;
5037 };
5038
5039 my $safe_string_ne = sub {
5040     my ($a, $b) = @_;
5041
5042     return 0 if !defined($a) && !defined($b);
5043     return 1 if !defined($a);
5044     return 1 if !defined($b);
5045
5046     return $a ne $b;
5047 };
5048
5049 sub vmconfig_update_net {
5050     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5051
5052     my $newnet = parse_net($value);
5053
5054     if ($conf->{$opt}) {
5055         my $oldnet = parse_net($conf->{$opt});
5056
5057         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5058             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5059             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5060             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5061
5062             # for non online change, we try to hot-unplug
5063             die "skip\n" if !$hotplug;
5064             vm_deviceunplug($vmid, $conf, $opt);
5065         } else {
5066
5067             die "internal error" if $opt !~ m/net(\d+)/;
5068             my $iface = "tap${vmid}i$1";
5069
5070             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5071                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5072                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5073                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5074                 PVE::Network::tap_unplug($iface);
5075                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5076             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5077                 # Rate can be applied on its own but any change above needs to
5078                 # include the rate in tap_plug since OVS resets everything.
5079                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5080             }
5081
5082             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5083                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5084             }
5085
5086             return 1;
5087         }
5088     }
5089
5090     if ($hotplug) {
5091         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5092     } else {
5093         die "skip\n";
5094     }
5095 }
5096
5097 sub vmconfig_update_disk {
5098     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5099
5100     # fixme: do we need force?
5101
5102     my $drive = parse_drive($opt, $value);
5103
5104     if ($conf->{$opt}) {
5105
5106         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5107
5108             my $media = $drive->{media} || 'disk';
5109             my $oldmedia = $old_drive->{media} || 'disk';
5110             die "unable to change media type\n" if $media ne $oldmedia;
5111
5112             if (!drive_is_cdrom($old_drive)) {
5113
5114                 if ($drive->{file} ne $old_drive->{file}) {
5115
5116                     die "skip\n" if !$hotplug;
5117
5118                     # unplug and register as unused
5119                     vm_deviceunplug($vmid, $conf, $opt);
5120                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5121
5122                 } else {
5123                     # update existing disk
5124
5125                     # skip non hotpluggable value
5126                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5127                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5128                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5129                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5130                         die "skip\n";
5131                     }
5132
5133                     # apply throttle
5134                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5135                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5136                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5137                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5138                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5139                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5140                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5141                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5142                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5143                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5144                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5145                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5146                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5147                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5148                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5149                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5150                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5151                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5152
5153                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5154                                                    ($drive->{mbps} || 0)*1024*1024,
5155                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5156                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5157                                                    $drive->{iops} || 0,
5158                                                    $drive->{iops_rd} || 0,
5159                                                    $drive->{iops_wr} || 0,
5160                                                    ($drive->{mbps_max} || 0)*1024*1024,
5161                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5162                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5163                                                    $drive->{iops_max} || 0,
5164                                                    $drive->{iops_rd_max} || 0,
5165                                                    $drive->{iops_wr_max} || 0,
5166                                                    $drive->{bps_max_length} || 1,
5167                                                    $drive->{bps_rd_max_length} || 1,
5168                                                    $drive->{bps_wr_max_length} || 1,
5169                                                    $drive->{iops_max_length} || 1,
5170                                                    $drive->{iops_rd_max_length} || 1,
5171                                                    $drive->{iops_wr_max_length} || 1);
5172
5173                     }
5174
5175                     return 1;
5176                 }
5177
5178             } else { # cdrom
5179
5180                 if ($drive->{file} eq 'none') {
5181                     mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5182                     if (drive_is_cloudinit($old_drive)) {
5183                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5184                     }
5185                 } else {
5186                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5187                     mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5188                     mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5189                 }
5190
5191                 return 1;
5192             }
5193         }
5194     }
5195
5196     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5197     # hotplug new disks
5198     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5199     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5200 }
5201
5202 sub vm_start {
5203     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5204         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5205
5206     PVE::QemuConfig->lock_config($vmid, sub {
5207         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5208
5209         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5210
5211         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5212
5213         PVE::QemuConfig->check_lock($conf)
5214             if !($skiplock || $is_suspended);
5215
5216         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5217
5218         # clean up leftover reboot request files
5219         eval { clear_reboot_request($vmid); };
5220         warn $@ if $@;
5221
5222         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5223             vmconfig_apply_pending($vmid, $conf, $storecfg);
5224             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5225         }
5226
5227         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5228
5229         my $defaults = load_defaults();
5230
5231         # set environment variable useful inside network script
5232         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5233
5234         my $local_volumes = {};
5235
5236         if ($targetstorage) {
5237             foreach_drive($conf, sub {
5238                 my ($ds, $drive) = @_;
5239
5240                 return if drive_is_cdrom($drive);
5241
5242                 my $volid = $drive->{file};
5243
5244                 return if !$volid;
5245
5246                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5247
5248                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5249                 return if $scfg->{shared};
5250                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5251             });
5252
5253             my $format = undef;
5254
5255             foreach my $opt (sort keys %$local_volumes) {
5256
5257                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5258                 my $drive = parse_drive($opt, $conf->{$opt});
5259
5260                 #if remote storage is specified, use default format
5261                 if ($targetstorage && $targetstorage ne "1") {
5262                     $storeid = $targetstorage;
5263                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5264                     $format = $defFormat;
5265                 } else {
5266                     #else we use same format than original
5267                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5268                     $format = qemu_img_format($scfg, $volid);
5269                 }
5270
5271                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5272                 my $newdrive = $drive;
5273                 $newdrive->{format} = $format;
5274                 $newdrive->{file} = $newvolid;
5275                 my $drivestr = print_drive($newdrive);
5276                 $local_volumes->{$opt} = $drivestr;
5277                 #pass drive to conf for command line
5278                 $conf->{$opt} = $drivestr;
5279             }
5280         }
5281
5282         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5283
5284         if ($is_suspended) {
5285             # enforce machine type on suspended vm to ensure HW compatibility
5286             $forcemachine = $conf->{runningmachine};
5287             print "Resuming suspended VM\n";
5288         }
5289
5290         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5291
5292         my $migration_ip;
5293         my $get_migration_ip = sub {
5294             my ($cidr, $nodename) = @_;
5295
5296             return $migration_ip if defined($migration_ip);
5297
5298             if (!defined($cidr)) {
5299                 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5300                 $cidr = $dc_conf->{migration}->{network};
5301             }
5302
5303             if (defined($cidr)) {
5304                 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5305
5306                 die "could not get IP: no address configured on local " .
5307                     "node for network '$cidr'\n" if scalar(@$ips) == 0;
5308
5309                 die "could not get IP: multiple addresses configured on local " .
5310                     "node for network '$cidr'\n" if scalar(@$ips) > 1;
5311
5312                 $migration_ip = @$ips[0];
5313             }
5314
5315             $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5316                 if !defined($migration_ip);
5317
5318             return $migration_ip;
5319         };
5320
5321         my $migrate_uri;
5322         if ($statefile) {
5323             if ($statefile eq 'tcp') {
5324                 my $localip = "localhost";
5325                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5326                 my $nodename = nodename();
5327
5328                 if (!defined($migration_type)) {
5329                     if (defined($datacenterconf->{migration}->{type})) {
5330                         $migration_type = $datacenterconf->{migration}->{type};
5331                     } else {
5332                         $migration_type = 'secure';
5333                     }
5334                 }
5335
5336                 if ($migration_type eq 'insecure') {
5337                     $localip = $get_migration_ip->($migration_network, $nodename);
5338                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5339                 }
5340
5341                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5342                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5343                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5344                 push @$cmd, '-incoming', $migrate_uri;
5345                 push @$cmd, '-S';
5346
5347             } elsif ($statefile eq 'unix') {
5348                 # should be default for secure migrations as a ssh TCP forward
5349                 # tunnel is not deterministic reliable ready and fails regurarly
5350                 # to set up in time, so use UNIX socket forwards
5351                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5352                 unlink $socket_addr;
5353
5354                 $migrate_uri = "unix:$socket_addr";
5355
5356                 push @$cmd, '-incoming', $migrate_uri;
5357                 push @$cmd, '-S';
5358
5359             } elsif (-e $statefile) {
5360                 push @$cmd, '-loadstate', $statefile;
5361             } else {
5362                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5363                 push @$vollist, $statefile;
5364                 push @$cmd, '-loadstate', $statepath;
5365             }
5366         } elsif ($paused) {
5367             push @$cmd, '-S';
5368         }
5369
5370         # host pci devices
5371         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5372           my $d = parse_hostpci($conf->{"hostpci$i"});
5373           next if !$d;
5374           my $pcidevices = $d->{pciid};
5375           foreach my $pcidevice (@$pcidevices) {
5376                 my $pciid = $pcidevice->{id};
5377
5378                 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5379                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5380                 die "no pci device info for device '$pciid'\n" if !$info;
5381
5382                 if ($d->{mdev}) {
5383                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5384                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5385                 } else {
5386                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5387                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5388                     die "can't reset pci device '$pciid'\n"
5389                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5390                 }
5391           }
5392         }
5393
5394         PVE::Storage::activate_volumes($storecfg, $vollist);
5395
5396         eval {
5397             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5398                 outfunc => sub {}, errfunc => sub {});
5399         };
5400         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5401         # timeout should be more than enough here...
5402         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5403
5404         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5405                                                   : $defaults->{cpuunits};
5406
5407         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5408         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5409
5410         my %properties = (
5411             Slice => 'qemu.slice',
5412             KillMode => 'none',
5413             CPUShares => $cpuunits
5414         );
5415
5416         if (my $cpulimit = $conf->{cpulimit}) {
5417             $properties{CPUQuota} = int($cpulimit * 100);
5418         }
5419         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5420
5421         my $run_qemu = sub {
5422             PVE::Tools::run_fork sub {
5423                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5424                 run_command($cmd, %run_params);
5425             };
5426         };
5427
5428         if ($conf->{hugepages}) {
5429
5430             my $code = sub {
5431                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5432                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5433
5434                 PVE::QemuServer::Memory::hugepages_mount();
5435                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5436
5437                 eval { $run_qemu->() };
5438                 if (my $err = $@) {
5439                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5440                     die $err;
5441                 }
5442
5443                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5444             };
5445             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5446
5447         } else {
5448             eval { $run_qemu->() };
5449         }
5450
5451         if (my $err = $@) {
5452             # deactivate volumes if start fails
5453             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5454             die "start failed: $err";
5455         }
5456
5457         print "migration listens on $migrate_uri\n" if $migrate_uri;
5458
5459         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5460             eval { mon_cmd($vmid, "cont"); };
5461             warn $@ if $@;
5462         }
5463
5464         #start nbd server for storage migration
5465         if ($targetstorage) {
5466             my $nodename = nodename();
5467             my $localip = $get_migration_ip->($migration_network, $nodename);
5468             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5469             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5470
5471             mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5472
5473             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5474
5475             foreach my $opt (sort keys %$local_volumes) {
5476                 my $volid = $local_volumes->{$opt};
5477                 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5478                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5479                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5480             }
5481         }
5482
5483         if ($migratedfrom) {
5484             eval {
5485                 set_migration_caps($vmid);
5486             };
5487             warn $@ if $@;
5488
5489             if ($spice_port) {
5490                 print "spice listens on port $spice_port\n";
5491                 if ($spice_ticket) {
5492                     mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5493                     mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5494                 }
5495             }
5496
5497         } else {
5498             mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5499                 if !$statefile && $conf->{balloon};
5500
5501             foreach my $opt (keys %$conf) {
5502                 next if $opt !~  m/^net\d+$/;
5503                 my $nicconf = parse_net($conf->{$opt});
5504                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5505             }
5506         }
5507
5508         mon_cmd($vmid, 'qom-set',
5509                     path => "machine/peripheral/balloon0",
5510                     property => "guest-stats-polling-interval",
5511                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5512
5513         if ($is_suspended) {
5514             print "Resumed VM, removing state\n";
5515             if (my $vmstate = $conf->{vmstate}) {
5516                 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5517                 PVE::Storage::vdisk_free($storecfg, $vmstate);
5518             }
5519             delete $conf->@{qw(lock vmstate runningmachine)};
5520             PVE::QemuConfig->write_config($vmid, $conf);
5521         }
5522
5523         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5524     });
5525 }
5526
5527 sub vm_commandline {
5528     my ($storecfg, $vmid, $snapname) = @_;
5529
5530     my $conf = PVE::QemuConfig->load_config($vmid);
5531     my $forcemachine;
5532
5533     if ($snapname) {
5534         my $snapshot = $conf->{snapshots}->{$snapname};
5535         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5536
5537         # check for a 'runningmachine' in snapshot
5538         $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5539
5540         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5541
5542         $conf = $snapshot;
5543     }
5544
5545     my $defaults = load_defaults();
5546
5547     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5548
5549     return PVE::Tools::cmd2string($cmd);
5550 }
5551
5552 sub vm_reset {
5553     my ($vmid, $skiplock) = @_;
5554
5555     PVE::QemuConfig->lock_config($vmid, sub {
5556
5557         my $conf = PVE::QemuConfig->load_config($vmid);
5558
5559         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5560
5561         mon_cmd($vmid, "system_reset");
5562     });
5563 }
5564
5565 sub get_vm_volumes {
5566     my ($conf) = @_;
5567
5568     my $vollist = [];
5569     foreach_volid($conf, sub {
5570         my ($volid, $attr) = @_;
5571
5572         return if $volid =~ m|^/|;
5573
5574         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5575         return if !$sid;
5576
5577         push @$vollist, $volid;
5578     });
5579
5580     return $vollist;
5581 }
5582
5583 sub vm_stop_cleanup {
5584     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5585
5586     eval {
5587
5588         if (!$keepActive) {
5589             my $vollist = get_vm_volumes($conf);
5590             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5591         }
5592
5593         foreach my $ext (qw(mon qmp pid vnc qga)) {
5594             unlink "/var/run/qemu-server/${vmid}.$ext";
5595         }
5596
5597         if ($conf->{ivshmem}) {
5598             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5599             # just delete it for now, VMs which have this already open do not
5600             # are affected, but new VMs will get a separated one. If this
5601             # becomes an issue we either add some sort of ref-counting or just
5602             # add a "don't delete on stop" flag to the ivshmem format.
5603             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5604         }
5605
5606         foreach my $key (keys %$conf) {
5607             next if $key !~ m/^hostpci(\d+)$/;
5608             my $hostpciindex = $1;
5609             my $d = parse_hostpci($conf->{$key});
5610             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5611
5612             foreach my $pci (@{$d->{pciid}}) {
5613                 my $pciid = $pci->{id};
5614                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5615             }
5616         }
5617
5618         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5619     };
5620     warn $@ if $@; # avoid errors - just warn
5621 }
5622
5623 # call only in locked context
5624 sub _do_vm_stop {
5625     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5626
5627     my $pid = check_running($vmid, $nocheck);
5628     return if !$pid;
5629
5630     my $conf;
5631     if (!$nocheck) {
5632         $conf = PVE::QemuConfig->load_config($vmid);
5633         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5634         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5635             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5636             $timeout = $opts->{down} if $opts->{down};
5637         }
5638         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5639     }
5640
5641     eval {
5642         if ($shutdown) {
5643             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5644                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5645             } else {
5646                 mon_cmd($vmid, "system_powerdown");
5647             }
5648         } else {
5649             mon_cmd($vmid, "quit");
5650         }
5651     };
5652     my $err = $@;
5653
5654     if (!$err) {
5655         $timeout = 60 if !defined($timeout);
5656
5657         my $count = 0;
5658         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5659             $count++;
5660             sleep 1;
5661         }
5662
5663         if ($count >= $timeout) {
5664             if ($force) {
5665                 warn "VM still running - terminating now with SIGTERM\n";
5666                 kill 15, $pid;
5667             } else {
5668                 die "VM quit/powerdown failed - got timeout\n";
5669             }
5670         } else {
5671             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5672             return;
5673         }
5674     } else {
5675         if ($force) {
5676             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5677             kill 15, $pid;
5678         } else {
5679             die "VM quit/powerdown failed\n";
5680         }
5681     }
5682
5683     # wait again
5684     $timeout = 10;
5685
5686     my $count = 0;
5687     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5688         $count++;
5689         sleep 1;
5690     }
5691
5692     if ($count >= $timeout) {
5693         warn "VM still running - terminating now with SIGKILL\n";
5694         kill 9, $pid;
5695         sleep 1;
5696     }
5697
5698     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5699 }
5700
5701 # Note: use $nocheck to skip tests if VM configuration file exists.
5702 # We need that when migration VMs to other nodes (files already moved)
5703 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5704 sub vm_stop {
5705     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5706
5707     $force = 1 if !defined($force) && !$shutdown;
5708
5709     if ($migratedfrom){
5710         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5711         kill 15, $pid if $pid;
5712         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5713         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5714         return;
5715     }
5716
5717     PVE::QemuConfig->lock_config($vmid, sub {
5718         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5719    });
5720 }
5721
5722 sub vm_reboot {
5723     my ($vmid, $timeout) = @_;
5724
5725     PVE::QemuConfig->lock_config($vmid, sub {
5726         eval {
5727
5728             # only reboot if running, as qmeventd starts it again on a stop event
5729             return if !check_running($vmid);
5730
5731             create_reboot_request($vmid);
5732
5733             my $storecfg = PVE::Storage::config();
5734             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5735
5736         };
5737         if (my $err = $@) {
5738             # avoid that the next normal shutdown will be confused for a reboot
5739             clear_reboot_request($vmid);
5740             die $err;
5741         }
5742    });
5743 }
5744
5745 # note: if using the statestorage parameter, the caller has to check privileges
5746 sub vm_suspend {
5747     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5748
5749     my $conf;
5750     my $path;
5751     my $storecfg;
5752     my $vmstate;
5753
5754     PVE::QemuConfig->lock_config($vmid, sub {
5755
5756         $conf = PVE::QemuConfig->load_config($vmid);
5757
5758         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5759         PVE::QemuConfig->check_lock($conf)
5760             if !($skiplock || $is_backing_up);
5761
5762         die "cannot suspend to disk during backup\n"
5763             if $is_backing_up && $includestate;
5764
5765         if ($includestate) {
5766             $conf->{lock} = 'suspending';
5767             my $date = strftime("%Y-%m-%d", localtime(time()));
5768             $storecfg = PVE::Storage::config();
5769             if (!$statestorage) {
5770                 $statestorage = find_vmstate_storage($conf, $storecfg);
5771                 # check permissions for the storage
5772                 my $rpcenv = PVE::RPCEnvironment::get();
5773                 if ($rpcenv->{type} ne 'cli') {
5774                     my $authuser = $rpcenv->get_user();
5775                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5776                 }
5777             }
5778
5779
5780             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5781             $path = PVE::Storage::path($storecfg, $vmstate);
5782             PVE::QemuConfig->write_config($vmid, $conf);
5783         } else {
5784             mon_cmd($vmid, "stop");
5785         }
5786     });
5787
5788     if ($includestate) {
5789         # save vm state
5790         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5791
5792         eval {
5793             mon_cmd($vmid, "savevm-start", statefile => $path);
5794             for(;;) {
5795                 my $state = mon_cmd($vmid, "query-savevm");
5796                 if (!$state->{status}) {
5797                     die "savevm not active\n";
5798                 } elsif ($state->{status} eq 'active') {
5799                     sleep(1);
5800                     next;
5801                 } elsif ($state->{status} eq 'completed') {
5802                     print "State saved, quitting\n";
5803                     last;
5804                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5805                     die "query-savevm failed with error '$state->{error}'\n"
5806                 } else {
5807                     die "query-savevm returned status '$state->{status}'\n";
5808                 }
5809             }
5810         };
5811         my $err = $@;
5812
5813         PVE::QemuConfig->lock_config($vmid, sub {
5814             $conf = PVE::QemuConfig->load_config($vmid);
5815             if ($err) {
5816                 # cleanup, but leave suspending lock, to indicate something went wrong
5817                 eval {
5818                     mon_cmd($vmid, "savevm-end");
5819                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5820                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5821                     delete $conf->@{qw(vmstate runningmachine)};
5822                     PVE::QemuConfig->write_config($vmid, $conf);
5823                 };
5824                 warn $@ if $@;
5825                 die $err;
5826             }
5827
5828             die "lock changed unexpectedly\n"
5829                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5830
5831             mon_cmd($vmid, "quit");
5832             $conf->{lock} = 'suspended';
5833             PVE::QemuConfig->write_config($vmid, $conf);
5834         });
5835     }
5836 }
5837
5838 sub vm_resume {
5839     my ($vmid, $skiplock, $nocheck) = @_;
5840
5841     PVE::QemuConfig->lock_config($vmid, sub {
5842         my $res = mon_cmd($vmid, 'query-status');
5843         my $resume_cmd = 'cont';
5844
5845         if ($res->{status} && $res->{status} eq 'suspended') {
5846             $resume_cmd = 'system_wakeup';
5847         }
5848
5849         if (!$nocheck) {
5850
5851             my $conf = PVE::QemuConfig->load_config($vmid);
5852
5853             PVE::QemuConfig->check_lock($conf)
5854                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5855         }
5856
5857         mon_cmd($vmid, $resume_cmd);
5858     });
5859 }
5860
5861 sub vm_sendkey {
5862     my ($vmid, $skiplock, $key) = @_;
5863
5864     PVE::QemuConfig->lock_config($vmid, sub {
5865
5866         my $conf = PVE::QemuConfig->load_config($vmid);
5867
5868         # there is no qmp command, so we use the human monitor command
5869         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5870         die $res if $res ne '';
5871     });
5872 }
5873
5874 # vzdump restore implementaion
5875
5876 sub tar_archive_read_firstfile {
5877     my $archive = shift;
5878
5879     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5880
5881     # try to detect archive type first
5882     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5883         die "unable to open file '$archive'\n";
5884     my $firstfile = <$fh>;
5885     kill 15, $pid;
5886     close $fh;
5887
5888     die "ERROR: archive contaions no data\n" if !$firstfile;
5889     chomp $firstfile;
5890
5891     return $firstfile;
5892 }
5893
5894 sub tar_restore_cleanup {
5895     my ($storecfg, $statfile) = @_;
5896
5897     print STDERR "starting cleanup\n";
5898
5899     if (my $fd = IO::File->new($statfile, "r")) {
5900         while (defined(my $line = <$fd>)) {
5901             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5902                 my $volid = $2;
5903                 eval {
5904                     if ($volid =~ m|^/|) {
5905                         unlink $volid || die 'unlink failed\n';
5906                     } else {
5907                         PVE::Storage::vdisk_free($storecfg, $volid);
5908                     }
5909                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5910                 };
5911                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5912             } else {
5913                 print STDERR "unable to parse line in statfile - $line";
5914             }
5915         }
5916         $fd->close();
5917     }
5918 }
5919
5920 sub restore_archive {
5921     my ($archive, $vmid, $user, $opts) = @_;
5922
5923     my $format = $opts->{format};
5924     my $comp;
5925
5926     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5927         $format = 'tar' if !$format;
5928         $comp = 'gzip';
5929     } elsif ($archive =~ m/\.tar$/) {
5930         $format = 'tar' if !$format;
5931     } elsif ($archive =~ m/.tar.lzo$/) {
5932         $format = 'tar' if !$format;
5933         $comp = 'lzop';
5934     } elsif ($archive =~ m/\.vma$/) {
5935         $format = 'vma' if !$format;
5936     } elsif ($archive =~ m/\.vma\.gz$/) {
5937         $format = 'vma' if !$format;
5938         $comp = 'gzip';
5939     } elsif ($archive =~ m/\.vma\.lzo$/) {
5940         $format = 'vma' if !$format;
5941         $comp = 'lzop';
5942     } else {
5943         $format = 'vma' if !$format; # default
5944     }
5945
5946     # try to detect archive format
5947     if ($format eq 'tar') {
5948         return restore_tar_archive($archive, $vmid, $user, $opts);
5949     } else {
5950         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5951     }
5952 }
5953
5954 sub restore_update_config_line {
5955     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5956
5957     return if $line =~ m/^\#qmdump\#/;
5958     return if $line =~ m/^\#vzdump\#/;
5959     return if $line =~ m/^lock:/;
5960     return if $line =~ m/^unused\d+:/;
5961     return if $line =~ m/^parent:/;
5962
5963     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5964     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5965         # try to convert old 1.X settings
5966         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5967         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5968             my ($model, $macaddr) = split(/\=/, $devconfig);
5969             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5970             my $net = {
5971                 model => $model,
5972                 bridge => "vmbr$ind",
5973                 macaddr => $macaddr,
5974             };
5975             my $netstr = print_net($net);
5976
5977             print $outfd "net$cookie->{netcount}: $netstr\n";
5978             $cookie->{netcount}++;
5979         }
5980     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5981         my ($id, $netstr) = ($1, $2);
5982         my $net = parse_net($netstr);
5983         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5984         $netstr = print_net($net);
5985         print $outfd "$id: $netstr\n";
5986     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5987         my $virtdev = $1;
5988         my $value = $3;
5989         my $di = parse_drive($virtdev, $value);
5990         if (defined($di->{backup}) && !$di->{backup}) {
5991             print $outfd "#$line";
5992         } elsif ($map->{$virtdev}) {
5993             delete $di->{format}; # format can change on restore
5994             $di->{file} = $map->{$virtdev};
5995             $value = print_drive($di);
5996             print $outfd "$virtdev: $value\n";
5997         } else {
5998             print $outfd $line;
5999         }
6000     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6001         my $vmgenid = $1;
6002         if ($vmgenid ne '0') {
6003             # always generate a new vmgenid if there was a valid one setup
6004             $vmgenid = generate_uuid();
6005         }
6006         print $outfd "vmgenid: $vmgenid\n";
6007     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6008         my ($uuid, $uuid_str);
6009         UUID::generate($uuid);
6010         UUID::unparse($uuid, $uuid_str);
6011         my $smbios1 = parse_smbios1($2);
6012         $smbios1->{uuid} = $uuid_str;
6013         print $outfd $1.print_smbios1($smbios1)."\n";
6014     } else {
6015         print $outfd $line;
6016     }
6017 }
6018
6019 sub scan_volids {
6020     my ($cfg, $vmid) = @_;
6021
6022     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6023
6024     my $volid_hash = {};
6025     foreach my $storeid (keys %$info) {
6026         foreach my $item (@{$info->{$storeid}}) {
6027             next if !($item->{volid} && $item->{size});
6028             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6029             $volid_hash->{$item->{volid}} = $item;
6030         }
6031     }
6032
6033     return $volid_hash;
6034 }
6035
6036 sub is_volume_in_use {
6037     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6038
6039     my $path = PVE::Storage::path($storecfg, $volid);
6040
6041     my $scan_config = sub {
6042         my ($cref, $snapname) = @_;
6043
6044         foreach my $key (keys %$cref) {
6045             my $value = $cref->{$key};
6046             if (is_valid_drivename($key)) {
6047                 next if $skip_drive && $key eq $skip_drive;
6048                 my $drive = parse_drive($key, $value);
6049                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6050                 return 1 if $volid eq $drive->{file};
6051                 if ($drive->{file} =~ m!^/!) {
6052                     return 1 if $drive->{file} eq $path;
6053                 } else {
6054                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6055                     next if !$storeid;
6056                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6057                     next if !$scfg;
6058                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6059                 }
6060             }
6061         }
6062
6063         return 0;
6064     };
6065
6066     return 1 if &$scan_config($conf);
6067
6068     undef $skip_drive;
6069
6070     foreach my $snapname (keys %{$conf->{snapshots}}) {
6071         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6072     }
6073
6074     return 0;
6075 }
6076
6077 sub update_disksize {
6078     my ($vmid, $conf, $volid_hash) = @_;
6079
6080     my $changes;
6081     my $prefix = "VM $vmid:";
6082
6083     # used and unused disks
6084     my $referenced = {};
6085
6086     # Note: it is allowed to define multiple storages with same path (alias), so
6087     # we need to check both 'volid' and real 'path' (two different volid can point
6088     # to the same path).
6089
6090     my $referencedpath = {};
6091
6092     # update size info
6093     foreach my $opt (keys %$conf) {
6094         if (is_valid_drivename($opt)) {
6095             my $drive = parse_drive($opt, $conf->{$opt});
6096             my $volid = $drive->{file};
6097             next if !$volid;
6098
6099             $referenced->{$volid} = 1;
6100             if ($volid_hash->{$volid} &&
6101                 (my $path = $volid_hash->{$volid}->{path})) {
6102                 $referencedpath->{$path} = 1;
6103             }
6104
6105             next if drive_is_cdrom($drive);
6106             next if !$volid_hash->{$volid};
6107
6108             $drive->{size} = $volid_hash->{$volid}->{size};
6109             my $new = print_drive($drive);
6110             if ($new ne $conf->{$opt}) {
6111                 $changes = 1;
6112                 $conf->{$opt} = $new;
6113                 print "$prefix update disk '$opt' information.\n";
6114             }
6115         }
6116     }
6117
6118     # remove 'unusedX' entry if volume is used
6119     foreach my $opt (keys %$conf) {
6120         next if $opt !~ m/^unused\d+$/;
6121         my $volid = $conf->{$opt};
6122         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6123         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6124             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6125             $changes = 1;
6126             delete $conf->{$opt};
6127         }
6128
6129         $referenced->{$volid} = 1;
6130         $referencedpath->{$path} = 1 if $path;
6131     }
6132
6133     foreach my $volid (sort keys %$volid_hash) {
6134         next if $volid =~ m/vm-$vmid-state-/;
6135         next if $referenced->{$volid};
6136         my $path = $volid_hash->{$volid}->{path};
6137         next if !$path; # just to be sure
6138         next if $referencedpath->{$path};
6139         $changes = 1;
6140         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6141         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6142         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6143     }
6144
6145     return $changes;
6146 }
6147
6148 sub rescan {
6149     my ($vmid, $nolock, $dryrun) = @_;
6150
6151     my $cfg = PVE::Storage::config();
6152
6153     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6154     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6155     foreach my $stor (keys %{$cfg->{ids}}) {
6156         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6157     }
6158
6159     print "rescan volumes...\n";
6160     my $volid_hash = scan_volids($cfg, $vmid);
6161
6162     my $updatefn =  sub {
6163         my ($vmid) = @_;
6164
6165         my $conf = PVE::QemuConfig->load_config($vmid);
6166
6167         PVE::QemuConfig->check_lock($conf);
6168
6169         my $vm_volids = {};
6170         foreach my $volid (keys %$volid_hash) {
6171             my $info = $volid_hash->{$volid};
6172             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6173         }
6174
6175         my $changes = update_disksize($vmid, $conf, $vm_volids);
6176
6177         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6178     };
6179
6180     if (defined($vmid)) {
6181         if ($nolock) {
6182             &$updatefn($vmid);
6183         } else {
6184             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6185         }
6186     } else {
6187         my $vmlist = config_list();
6188         foreach my $vmid (keys %$vmlist) {
6189             if ($nolock) {
6190                 &$updatefn($vmid);
6191             } else {
6192                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6193             }
6194         }
6195     }
6196 }
6197
6198 sub restore_vma_archive {
6199     my ($archive, $vmid, $user, $opts, $comp) = @_;
6200
6201     my $readfrom = $archive;
6202
6203     my $cfg = PVE::Storage::config();
6204     my $commands = [];
6205     my $bwlimit = $opts->{bwlimit};
6206
6207     my $dbg_cmdstring = '';
6208     my $add_pipe = sub {
6209         my ($cmd) = @_;
6210         push @$commands, $cmd;
6211         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6212         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6213         $readfrom = '-';
6214     };
6215
6216     my $input = undef;
6217     if ($archive eq '-') {
6218         $input = '<&STDIN';
6219     } else {
6220         # If we use a backup from a PVE defined storage we also consider that
6221         # storage's rate limit:
6222         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6223         if (defined($volid)) {
6224             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6225             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6226             if ($readlimit) {
6227                 print STDERR "applying read rate limit: $readlimit\n";
6228                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6229                 $add_pipe->($cstream);
6230             }
6231         }
6232     }
6233
6234     if ($comp) {
6235         my $cmd;
6236         if ($comp eq 'gzip') {
6237             $cmd = ['zcat', $readfrom];
6238         } elsif ($comp eq 'lzop') {
6239             $cmd = ['lzop', '-d', '-c', $readfrom];
6240         } else {
6241             die "unknown compression method '$comp'\n";
6242         }
6243         $add_pipe->($cmd);
6244     }
6245
6246     my $tmpdir = "/var/tmp/vzdumptmp$$";
6247     rmtree $tmpdir;
6248
6249     # disable interrupts (always do cleanups)
6250     local $SIG{INT} =
6251         local $SIG{TERM} =
6252         local $SIG{QUIT} =
6253         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6254
6255     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6256     POSIX::mkfifo($mapfifo, 0600);
6257     my $fifofh;
6258
6259     my $openfifo = sub {
6260         open($fifofh, '>', $mapfifo) || die $!;
6261     };
6262
6263     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6264
6265     my $oldtimeout;
6266     my $timeout = 5;
6267
6268     my $devinfo = {};
6269
6270     my $rpcenv = PVE::RPCEnvironment::get();
6271
6272     my $conffile = PVE::QemuConfig->config_file($vmid);
6273     my $tmpfn = "$conffile.$$.tmp";
6274
6275     # Note: $oldconf is undef if VM does not exists
6276     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6277     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6278
6279     my %storage_limits;
6280
6281     my $print_devmap = sub {
6282         my $virtdev_hash = {};
6283
6284         my $cfgfn = "$tmpdir/qemu-server.conf";
6285
6286         # we can read the config - that is already extracted
6287         my $fh = IO::File->new($cfgfn, "r") ||
6288             "unable to read qemu-server.conf - $!\n";
6289
6290         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6291         if (-f $fwcfgfn) {
6292             my $pve_firewall_dir = '/etc/pve/firewall';
6293             mkdir $pve_firewall_dir; # make sure the dir exists
6294             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6295         }
6296
6297         while (defined(my $line = <$fh>)) {
6298             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6299                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6300                 die "archive does not contain data for drive '$virtdev'\n"
6301                     if !$devinfo->{$devname};
6302                 if (defined($opts->{storage})) {
6303                     $storeid = $opts->{storage} || 'local';
6304                 } elsif (!$storeid) {
6305                     $storeid = 'local';
6306                 }
6307                 $format = 'raw' if !$format;
6308                 $devinfo->{$devname}->{devname} = $devname;
6309                 $devinfo->{$devname}->{virtdev} = $virtdev;
6310                 $devinfo->{$devname}->{format} = $format;
6311                 $devinfo->{$devname}->{storeid} = $storeid;
6312
6313                 # check permission on storage
6314                 my $pool = $opts->{pool}; # todo: do we need that?
6315                 if ($user ne 'root@pam') {
6316                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6317                 }
6318
6319                 $storage_limits{$storeid} = $bwlimit;
6320
6321                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6322             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6323                 my $virtdev = $1;
6324                 my $drive = parse_drive($virtdev, $2);
6325                 if (drive_is_cloudinit($drive)) {
6326                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6327                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6328                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6329
6330                     my $d = {
6331                         format => $format,
6332                         storeid => $opts->{storage} // $storeid,
6333                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6334                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6335                         name => "vm-$vmid-cloudinit",
6336                         is_cloudinit => 1,
6337                     };
6338                     $virtdev_hash->{$virtdev} = $d;
6339                 }
6340             }
6341         }
6342
6343         foreach my $key (keys %storage_limits) {
6344             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6345             next if !$limit;
6346             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6347             $storage_limits{$key} = $limit * 1024;
6348         }
6349
6350         foreach my $devname (keys %$devinfo) {
6351             die "found no device mapping information for device '$devname'\n"
6352                 if !$devinfo->{$devname}->{virtdev};
6353         }
6354
6355         # create empty/temp config
6356         if ($oldconf) {
6357             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6358             foreach_drive($oldconf, sub {
6359                 my ($ds, $drive) = @_;
6360
6361                 return if drive_is_cdrom($drive, 1);
6362
6363                 my $volid = $drive->{file};
6364                 return if !$volid || $volid =~ m|^/|;
6365
6366                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6367                 return if !$path || !$owner || ($owner != $vmid);
6368
6369                 # Note: only delete disk we want to restore
6370                 # other volumes will become unused
6371                 if ($virtdev_hash->{$ds}) {
6372                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6373                     if (my $err = $@) {
6374                         warn $err;
6375                     }
6376                 }
6377             });
6378
6379             # delete vmstate files, after the restore we have no snapshots anymore
6380             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6381                 my $snap = $oldconf->{snapshots}->{$snapname};
6382                 if ($snap->{vmstate}) {
6383                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6384                     if (my $err = $@) {
6385                         warn $err;
6386                     }
6387                 }
6388             }
6389         }
6390
6391         my $map = {};
6392         foreach my $virtdev (sort keys %$virtdev_hash) {
6393             my $d = $virtdev_hash->{$virtdev};
6394             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6395             my $storeid = $d->{storeid};
6396             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6397
6398             my $map_opts = '';
6399             if (my $limit = $storage_limits{$storeid}) {
6400                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6401             }
6402
6403             # test if requested format is supported
6404             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6405             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6406             $d->{format} = $defFormat if !$supported;
6407
6408             my $name;
6409             if ($d->{is_cloudinit}) {
6410                 $name = $d->{name};
6411                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6412             }
6413
6414             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6415             print STDERR "new volume ID is '$volid'\n";
6416             $d->{volid} = $volid;
6417
6418             PVE::Storage::activate_volumes($cfg, [$volid]);
6419
6420             my $write_zeros = 1;
6421             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6422                 $write_zeros = 0;
6423             }
6424
6425             if (!$d->{is_cloudinit}) {
6426                 my $path = PVE::Storage::path($cfg, $volid);
6427
6428                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6429
6430                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6431             }
6432             $map->{$virtdev} = $volid;
6433         }
6434
6435         $fh->seek(0, 0) || die "seek failed - $!\n";
6436
6437         my $outfd = new IO::File ($tmpfn, "w") ||
6438             die "unable to write config for VM $vmid\n";
6439
6440         my $cookie = { netcount => 0 };
6441         while (defined(my $line = <$fh>)) {
6442             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6443         }
6444
6445         $fh->close();
6446         $outfd->close();
6447     };
6448
6449     eval {
6450         # enable interrupts
6451         local $SIG{INT} =
6452             local $SIG{TERM} =
6453             local $SIG{QUIT} =
6454             local $SIG{HUP} =
6455             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6456         local $SIG{ALRM} = sub { die "got timeout\n"; };
6457
6458         $oldtimeout = alarm($timeout);
6459
6460         my $parser = sub {
6461             my $line = shift;
6462
6463             print "$line\n";
6464
6465             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6466                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6467                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6468             } elsif ($line =~ m/^CTIME: /) {
6469                 # we correctly received the vma config, so we can disable
6470                 # the timeout now for disk allocation (set to 10 minutes, so
6471                 # that we always timeout if something goes wrong)
6472                 alarm(600);
6473                 &$print_devmap();
6474                 print $fifofh "done\n";
6475                 my $tmp = $oldtimeout || 0;
6476                 $oldtimeout = undef;
6477                 alarm($tmp);
6478                 close($fifofh);
6479             }
6480         };
6481
6482         print "restore vma archive: $dbg_cmdstring\n";
6483         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6484     };
6485     my $err = $@;
6486
6487     alarm($oldtimeout) if $oldtimeout;
6488
6489     my $vollist = [];
6490     foreach my $devname (keys %$devinfo) {
6491         my $volid = $devinfo->{$devname}->{volid};
6492         push @$vollist, $volid if $volid;
6493     }
6494
6495     PVE::Storage::deactivate_volumes($cfg, $vollist);
6496
6497     unlink $mapfifo;
6498
6499     if ($err) {
6500         rmtree $tmpdir;
6501         unlink $tmpfn;
6502
6503         foreach my $devname (keys %$devinfo) {
6504             my $volid = $devinfo->{$devname}->{volid};
6505             next if !$volid;
6506             eval {
6507                 if ($volid =~ m|^/|) {
6508                     unlink $volid || die 'unlink failed\n';
6509                 } else {
6510                     PVE::Storage::vdisk_free($cfg, $volid);
6511                 }
6512                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6513             };
6514             print STDERR "unable to cleanup '$volid' - $@" if $@;
6515         }
6516         die $err;
6517     }
6518
6519     rmtree $tmpdir;
6520
6521     rename($tmpfn, $conffile) ||
6522         die "unable to commit configuration file '$conffile'\n";
6523
6524     PVE::Cluster::cfs_update(); # make sure we read new file
6525
6526     eval { rescan($vmid, 1); };
6527     warn $@ if $@;
6528 }
6529
6530 sub restore_tar_archive {
6531     my ($archive, $vmid, $user, $opts) = @_;
6532
6533     if ($archive ne '-') {
6534         my $firstfile = tar_archive_read_firstfile($archive);
6535         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6536             if $firstfile ne 'qemu-server.conf';
6537     }
6538
6539     my $storecfg = PVE::Storage::config();
6540
6541     # avoid zombie disks when restoring over an existing VM -> cleanup first
6542     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6543     # skiplock=1 because qmrestore has set the 'create' lock itself already
6544     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6545     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6546
6547     my $tocmd = "/usr/lib/qemu-server/qmextract";
6548
6549     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6550     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6551     $tocmd .= ' --prealloc' if $opts->{prealloc};
6552     $tocmd .= ' --info' if $opts->{info};
6553
6554     # tar option "xf" does not autodetect compression when read from STDIN,
6555     # so we pipe to zcat
6556     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6557         PVE::Tools::shellquote("--to-command=$tocmd");
6558
6559     my $tmpdir = "/var/tmp/vzdumptmp$$";
6560     mkpath $tmpdir;
6561
6562     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6563     local $ENV{VZDUMP_VMID} = $vmid;
6564     local $ENV{VZDUMP_USER} = $user;
6565
6566     my $conffile = PVE::QemuConfig->config_file($vmid);
6567     my $tmpfn = "$conffile.$$.tmp";
6568
6569     # disable interrupts (always do cleanups)
6570     local $SIG{INT} =
6571         local $SIG{TERM} =
6572         local $SIG{QUIT} =
6573         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6574
6575     eval {
6576         # enable interrupts
6577         local $SIG{INT} =
6578             local $SIG{TERM} =
6579             local $SIG{QUIT} =
6580             local $SIG{HUP} =
6581             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6582
6583         if ($archive eq '-') {
6584             print "extracting archive from STDIN\n";
6585             run_command($cmd, input => "<&STDIN");
6586         } else {
6587             print "extracting archive '$archive'\n";
6588             run_command($cmd);
6589         }
6590
6591         return if $opts->{info};
6592
6593         # read new mapping
6594         my $map = {};
6595         my $statfile = "$tmpdir/qmrestore.stat";
6596         if (my $fd = IO::File->new($statfile, "r")) {
6597             while (defined (my $line = <$fd>)) {
6598                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6599                     $map->{$1} = $2 if $1;
6600                 } else {
6601                     print STDERR "unable to parse line in statfile - $line\n";
6602                 }
6603             }
6604             $fd->close();
6605         }
6606
6607         my $confsrc = "$tmpdir/qemu-server.conf";
6608
6609         my $srcfd = new IO::File($confsrc, "r") ||
6610             die "unable to open file '$confsrc'\n";
6611
6612         my $outfd = new IO::File ($tmpfn, "w") ||
6613             die "unable to write config for VM $vmid\n";
6614
6615         my $cookie = { netcount => 0 };
6616         while (defined (my $line = <$srcfd>)) {
6617             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6618         }
6619
6620         $srcfd->close();
6621         $outfd->close();
6622     };
6623     if (my $err = $@) {
6624         unlink $tmpfn;
6625         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6626         die $err;
6627     }
6628
6629     rmtree $tmpdir;
6630
6631     rename $tmpfn, $conffile ||
6632         die "unable to commit configuration file '$conffile'\n";
6633
6634     PVE::Cluster::cfs_update(); # make sure we read new file
6635
6636     eval { rescan($vmid, 1); };
6637     warn $@ if $@;
6638 };
6639
6640 sub foreach_storage_used_by_vm {
6641     my ($conf, $func) = @_;
6642
6643     my $sidhash = {};
6644
6645     foreach_drive($conf, sub {
6646         my ($ds, $drive) = @_;
6647         return if drive_is_cdrom($drive);
6648
6649         my $volid = $drive->{file};
6650
6651         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6652         $sidhash->{$sid} = $sid if $sid;
6653     });
6654
6655     foreach my $sid (sort keys %$sidhash) {
6656         &$func($sid);
6657     }
6658 }
6659
6660 my $qemu_snap_storage = {
6661     rbd => 1,
6662 };
6663 sub do_snapshots_with_qemu {
6664     my ($storecfg, $volid) = @_;
6665
6666     my $storage_name = PVE::Storage::parse_volume_id($volid);
6667     my $scfg = $storecfg->{ids}->{$storage_name};
6668
6669     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6670         return 1;
6671     }
6672
6673     if ($volid =~ m/\.(qcow2|qed)$/){
6674         return 1;
6675     }
6676
6677     return undef;
6678 }
6679
6680 sub qga_check_running {
6681     my ($vmid, $nowarn) = @_;
6682
6683     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6684     if ($@) {
6685         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6686         return 0;
6687     }
6688     return 1;
6689 }
6690
6691 sub template_create {
6692     my ($vmid, $conf, $disk) = @_;
6693
6694     my $storecfg = PVE::Storage::config();
6695
6696     foreach_drive($conf, sub {
6697         my ($ds, $drive) = @_;
6698
6699         return if drive_is_cdrom($drive);
6700         return if $disk && $ds ne $disk;
6701
6702         my $volid = $drive->{file};
6703         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6704
6705         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6706         $drive->{file} = $voliddst;
6707         $conf->{$ds} = print_drive($drive);
6708         PVE::QemuConfig->write_config($vmid, $conf);
6709     });
6710 }
6711
6712 sub convert_iscsi_path {
6713     my ($path) = @_;
6714
6715     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6716         my $portal = $1;
6717         my $target = $2;
6718         my $lun = $3;
6719
6720         my $initiator_name = get_initiator_name();
6721
6722         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6723                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6724     }
6725
6726     die "cannot convert iscsi path '$path', unkown format\n";
6727 }
6728
6729 sub qemu_img_convert {
6730     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6731
6732     my $storecfg = PVE::Storage::config();
6733     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6734     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6735
6736     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6737
6738     my $cachemode;
6739     my $src_path;
6740     my $src_is_iscsi = 0;
6741     my $src_format;
6742
6743     if ($src_storeid) {
6744         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6745         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6746         $src_format = qemu_img_format($src_scfg, $src_volname);
6747         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6748         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6749         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6750     } elsif (-f $src_volid) {
6751         $src_path = $src_volid;
6752         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6753             $src_format = $1;
6754         }
6755     }
6756
6757     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6758
6759     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6760     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6761     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6762     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6763
6764     my $cmd = [];
6765     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6766     push @$cmd, '-l', "snapshot.name=$snapname"
6767         if $snapname && $src_format && $src_format eq "qcow2";
6768     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6769     push @$cmd, '-T', $cachemode if defined($cachemode);
6770
6771     if ($src_is_iscsi) {
6772         push @$cmd, '--image-opts';
6773         $src_path = convert_iscsi_path($src_path);
6774     } elsif ($src_format) {
6775         push @$cmd, '-f', $src_format;
6776     }
6777
6778     if ($dst_is_iscsi) {
6779         push @$cmd, '--target-image-opts';
6780         $dst_path = convert_iscsi_path($dst_path);
6781     } else {
6782         push @$cmd, '-O', $dst_format;
6783     }
6784
6785     push @$cmd, $src_path;
6786
6787     if (!$dst_is_iscsi && $is_zero_initialized) {
6788         push @$cmd, "zeroinit:$dst_path";
6789     } else {
6790         push @$cmd, $dst_path;
6791     }
6792
6793     my $parser = sub {
6794         my $line = shift;
6795         if($line =~ m/\((\S+)\/100\%\)/){
6796             my $percent = $1;
6797             my $transferred = int($size * $percent / 100);
6798             my $remaining = $size - $transferred;
6799
6800             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6801         }
6802
6803     };
6804
6805     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6806     my $err = $@;
6807     die "copy failed: $err" if $err;
6808 }
6809
6810 sub qemu_img_format {
6811     my ($scfg, $volname) = @_;
6812
6813     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6814         return $1;
6815     } else {
6816         return "raw";
6817     }
6818 }
6819
6820 sub qemu_drive_mirror {
6821     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6822
6823     $jobs = {} if !$jobs;
6824
6825     my $qemu_target;
6826     my $format;
6827     $jobs->{"drive-$drive"} = {};
6828
6829     if ($dst_volid =~ /^nbd:/) {
6830         $qemu_target = $dst_volid;
6831         $format = "nbd";
6832     } else {
6833         my $storecfg = PVE::Storage::config();
6834         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6835
6836         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6837
6838         $format = qemu_img_format($dst_scfg, $dst_volname);
6839
6840         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6841
6842         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6843     }
6844
6845     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6846     $opts->{format} = $format if $format;
6847
6848     if (defined($bwlimit)) {
6849         $opts->{speed} = $bwlimit * 1024;
6850         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6851     } else {
6852         print "drive mirror is starting for drive-$drive\n";
6853     }
6854
6855     # if a job already runs for this device we get an error, catch it for cleanup
6856     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6857     if (my $err = $@) {
6858         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6859         warn "$@\n" if $@;
6860         die "mirroring error: $err\n";
6861     }
6862
6863     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6864 }
6865
6866 sub qemu_drive_mirror_monitor {
6867     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6868
6869     eval {
6870         my $err_complete = 0;
6871
6872         while (1) {
6873             die "storage migration timed out\n" if $err_complete > 300;
6874
6875             my $stats = mon_cmd($vmid, "query-block-jobs");
6876
6877             my $running_mirror_jobs = {};
6878             foreach my $stat (@$stats) {
6879                 next if $stat->{type} ne 'mirror';
6880                 $running_mirror_jobs->{$stat->{device}} = $stat;
6881             }
6882
6883             my $readycounter = 0;
6884
6885             foreach my $job (keys %$jobs) {
6886
6887                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6888                     print "$job : finished\n";
6889                     delete $jobs->{$job};
6890                     next;
6891                 }
6892
6893                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6894
6895                 my $busy = $running_mirror_jobs->{$job}->{busy};
6896                 my $ready = $running_mirror_jobs->{$job}->{ready};
6897                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6898                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6899                     my $remaining = $total - $transferred;
6900                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6901
6902                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6903                 }
6904
6905                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6906             }
6907
6908             last if scalar(keys %$jobs) == 0;
6909
6910             if ($readycounter == scalar(keys %$jobs)) {
6911                 print "all mirroring jobs are ready \n";
6912                 last if $skipcomplete; #do the complete later
6913
6914                 if ($vmiddst && $vmiddst != $vmid) {
6915                     my $agent_running = $qga && qga_check_running($vmid);
6916                     if ($agent_running) {
6917                         print "freeze filesystem\n";
6918                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6919                     } else {
6920                         print "suspend vm\n";
6921                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6922                     }
6923
6924                     # if we clone a disk for a new target vm, we don't switch the disk
6925                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6926
6927                     if ($agent_running) {
6928                         print "unfreeze filesystem\n";
6929                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6930                     } else {
6931                         print "resume vm\n";
6932                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6933                     }
6934
6935                     last;
6936                 } else {
6937
6938                     foreach my $job (keys %$jobs) {
6939                         # try to switch the disk if source and destination are on the same guest
6940                         print "$job: Completing block job...\n";
6941
6942                         eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6943                         if ($@ =~ m/cannot be completed/) {
6944                             print "$job: Block job cannot be completed, try again.\n";
6945                             $err_complete++;
6946                         }else {
6947                             print "$job: Completed successfully.\n";
6948                             $jobs->{$job}->{complete} = 1;
6949                         }
6950                     }
6951                 }
6952             }
6953             sleep 1;
6954         }
6955     };
6956     my $err = $@;
6957
6958     if ($err) {
6959         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6960         die "mirroring error: $err";
6961     }
6962
6963 }
6964
6965 sub qemu_blockjobs_cancel {
6966     my ($vmid, $jobs) = @_;
6967
6968     foreach my $job (keys %$jobs) {
6969         print "$job: Cancelling block job\n";
6970         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6971         $jobs->{$job}->{cancel} = 1;
6972     }
6973
6974     while (1) {
6975         my $stats = mon_cmd($vmid, "query-block-jobs");
6976
6977         my $running_jobs = {};
6978         foreach my $stat (@$stats) {
6979             $running_jobs->{$stat->{device}} = $stat;
6980         }
6981
6982         foreach my $job (keys %$jobs) {
6983
6984             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6985                 print "$job: Done.\n";
6986                 delete $jobs->{$job};
6987             }
6988         }
6989
6990         last if scalar(keys %$jobs) == 0;
6991
6992         sleep 1;
6993     }
6994 }
6995
6996 sub clone_disk {
6997     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6998         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6999
7000     my $newvolid;
7001
7002     if (!$full) {
7003         print "create linked clone of drive $drivename ($drive->{file})\n";
7004         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7005         push @$newvollist, $newvolid;
7006     } else {
7007
7008         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7009         $storeid = $storage if $storage;
7010
7011         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7012         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7013
7014         print "create full clone of drive $drivename ($drive->{file})\n";
7015         my $name = undef;
7016         if (drive_is_cloudinit($drive)) {
7017             $name = "vm-$newvmid-cloudinit";
7018             $name .= ".$dst_format" if $dst_format ne 'raw';
7019             $snapname = undef;
7020             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7021         }
7022         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7023         push @$newvollist, $newvolid;
7024
7025         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7026
7027         if (drive_is_cloudinit($drive)) {
7028             goto no_data_clone;
7029         }
7030
7031         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7032         if (!$running || $snapname) {
7033             # TODO: handle bwlimits
7034             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7035         } else {
7036
7037             my $kvmver = get_running_qemu_version ($vmid);
7038             if (!min_version($kvmver, 2, 7)) {
7039                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7040                     if $drive->{iothread};
7041             }
7042
7043             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7044         }
7045     }
7046
7047 no_data_clone:
7048     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7049
7050     my $disk = $drive;
7051     $disk->{format} = undef;
7052     $disk->{file} = $newvolid;
7053     $disk->{size} = $size;
7054
7055     return $disk;
7056 }
7057
7058 sub get_running_qemu_version {
7059     my ($vmid) = @_;
7060     my $res = mon_cmd($vmid, "query-version");
7061     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7062 }
7063
7064 sub qemu_use_old_bios_files {
7065     my ($machine_type) = @_;
7066
7067     return if !$machine_type;
7068
7069     my $use_old_bios_files = undef;
7070
7071     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7072         $machine_type = $1;
7073         $use_old_bios_files = 1;
7074     } else {
7075         my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
7076         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7077         # load new efi bios files on migration. So this hack is required to allow
7078         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7079         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7080         $use_old_bios_files = !min_version($version, 2, 4);
7081     }
7082
7083     return ($use_old_bios_files, $machine_type);
7084 }
7085
7086 sub create_efidisk($$$$$) {
7087     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7088
7089     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7090     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7091
7092     my $vars_size_b = -s $ovmf_vars;
7093     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7094     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7095     PVE::Storage::activate_volumes($storecfg, [$volid]);
7096
7097     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7098     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7099
7100     return ($volid, $size/1024);
7101 }
7102
7103 sub vm_iothreads_list {
7104     my ($vmid) = @_;
7105
7106     my $res = mon_cmd($vmid, 'query-iothreads');
7107
7108     my $iothreads = {};
7109     foreach my $iothread (@$res) {
7110         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7111     }
7112
7113     return $iothreads;
7114 }
7115
7116 sub scsihw_infos {
7117     my ($conf, $drive) = @_;
7118
7119     my $maxdev = 0;
7120
7121     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7122         $maxdev = 7;
7123     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7124         $maxdev = 1;
7125     } else {
7126         $maxdev = 256;
7127     }
7128
7129     my $controller = int($drive->{index} / $maxdev);
7130     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7131
7132     return ($maxdev, $controller, $controller_prefix);
7133 }
7134
7135 sub add_hyperv_enlightenments {
7136     my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7137
7138     return if $winversion < 6;
7139     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7140
7141     if ($gpu_passthrough || defined($hv_vendor_id)) {
7142         $hv_vendor_id //= 'proxmox';
7143         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7144     }
7145
7146     if (min_version($machine_version, 2, 3)) {
7147         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7148         push @$cpuFlags , 'hv_vapic';
7149         push @$cpuFlags , 'hv_time';
7150     } else {
7151         push @$cpuFlags , 'hv_spinlocks=0xffff';
7152     }
7153
7154     if (min_version($machine_version, 2, 6)) {
7155         push @$cpuFlags , 'hv_reset';
7156         push @$cpuFlags , 'hv_vpindex';
7157         push @$cpuFlags , 'hv_runtime';
7158     }
7159
7160     if ($winversion >= 7) {
7161         push @$cpuFlags , 'hv_relaxed';
7162
7163         if (min_version($machine_version, 2, 12)) {
7164             push @$cpuFlags , 'hv_synic';
7165             push @$cpuFlags , 'hv_stimer';
7166         }
7167
7168         if (min_version($machine_version, 3, 1)) {
7169             push @$cpuFlags , 'hv_ipi';
7170         }
7171     }
7172 }
7173
7174 sub windows_version {
7175     my ($ostype) = @_;
7176
7177     return 0 if !$ostype;
7178
7179     my $winversion = 0;
7180
7181     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7182         $winversion = 5;
7183     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7184         $winversion = 6;
7185     } elsif ($ostype =~ m/^win(\d+)$/) {
7186         $winversion = $1;
7187     }
7188
7189     return $winversion;
7190 }
7191
7192 sub resolve_dst_disk_format {
7193         my ($storecfg, $storeid, $src_volname, $format) = @_;
7194         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7195
7196         if (!$format) {
7197             # if no target format is specified, use the source disk format as hint
7198             if ($src_volname) {
7199                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7200                 $format = qemu_img_format($scfg, $src_volname);
7201             } else {
7202                 return $defFormat;
7203             }
7204         }
7205
7206         # test if requested format is supported - else use default
7207         my $supported = grep { $_ eq $format } @$validFormats;
7208         $format = $defFormat if !$supported;
7209         return $format;
7210 }
7211
7212 sub resolve_first_disk {
7213     my $conf = shift;
7214     my @disks = PVE::QemuServer::valid_drive_names();
7215     my $firstdisk;
7216     foreach my $ds (reverse @disks) {
7217         next if !$conf->{$ds};
7218         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7219         next if PVE::QemuServer::drive_is_cdrom($disk);
7220         $firstdisk = $ds;
7221     }
7222     return $firstdisk;
7223 }
7224
7225 # NOTE: if this logic changes, please update docs & possibly gui logic
7226 sub find_vmstate_storage {
7227     my ($conf, $storecfg) = @_;
7228
7229     # first, return storage from conf if set
7230     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7231
7232     my ($target, $shared, $local);
7233
7234     foreach_storage_used_by_vm($conf, sub {
7235         my ($sid) = @_;
7236         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7237         my $dst = $scfg->{shared} ? \$shared : \$local;
7238         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7239     });
7240
7241     # second, use shared storage where VM has at least one disk
7242     # third, use local storage where VM has at least one disk
7243     # fall back to local storage
7244     $target = $shared // $local // 'local';
7245
7246     return $target;
7247 }
7248
7249 sub generate_uuid {
7250     my ($uuid, $uuid_str);
7251     UUID::generate($uuid);
7252     UUID::unparse($uuid, $uuid_str);
7253     return $uuid_str;
7254 }
7255
7256 sub generate_smbios1_uuid {
7257     return "uuid=".generate_uuid();
7258 }
7259
7260 sub nbd_stop {
7261     my ($vmid) = @_;
7262
7263     mon_cmd($vmid, 'nbd-server-stop');
7264 }
7265
7266 sub create_reboot_request {
7267     my ($vmid) = @_;
7268     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7269         or die "failed to create reboot trigger file: $!\n";
7270     close($fh);
7271 }
7272
7273 sub clear_reboot_request {
7274     my ($vmid) = @_;
7275     my $path = "/run/qemu-server/$vmid.reboot";
7276     my $res = 0;
7277
7278     $res = unlink($path);
7279     die "could not remove reboot request for $vmid: $!"
7280         if !$res && $! != POSIX::ENOENT;
7281
7282     return $res;
7283 }
7284
7285 # bash completion helper
7286
7287 sub complete_backup_archives {
7288     my ($cmdname, $pname, $cvalue) = @_;
7289
7290     my $cfg = PVE::Storage::config();
7291
7292     my $storeid;
7293
7294     if ($cvalue =~ m/^([^:]+):/) {
7295         $storeid = $1;
7296     }
7297
7298     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7299
7300     my $res = [];
7301     foreach my $id (keys %$data) {
7302         foreach my $item (@{$data->{$id}}) {
7303             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7304             push @$res, $item->{volid} if defined($item->{volid});
7305         }
7306     }
7307
7308     return $res;
7309 }
7310
7311 my $complete_vmid_full = sub {
7312     my ($running) = @_;
7313
7314     my $idlist = vmstatus();
7315
7316     my $res = [];
7317
7318     foreach my $id (keys %$idlist) {
7319         my $d = $idlist->{$id};
7320         if (defined($running)) {
7321             next if $d->{template};
7322             next if $running && $d->{status} ne 'running';
7323             next if !$running && $d->{status} eq 'running';
7324         }
7325         push @$res, $id;
7326
7327     }
7328     return $res;
7329 };
7330
7331 sub complete_vmid {
7332     return &$complete_vmid_full();
7333 }
7334
7335 sub complete_vmid_stopped {
7336     return &$complete_vmid_full(0);
7337 }
7338
7339 sub complete_vmid_running {
7340     return &$complete_vmid_full(1);
7341 }
7342
7343 sub complete_storage {
7344
7345     my $cfg = PVE::Storage::config();
7346     my $ids = $cfg->{ids};
7347
7348     my $res = [];
7349     foreach my $sid (keys %$ids) {
7350         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7351         next if !$ids->{$sid}->{content}->{images};
7352         push @$res, $sid;
7353     }
7354
7355     return $res;
7356 }
7357
7358 sub complete_migration_storage {
7359     my ($cmd, $param, $current_value, $all_args) = @_;
7360
7361     my $targetnode = @$all_args[1];
7362
7363     my $cfg = PVE::Storage::config();
7364     my $ids = $cfg->{ids};
7365
7366     my $res = [];
7367     foreach my $sid (keys %$ids) {
7368         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7369         next if !$ids->{$sid}->{content}->{images};
7370         push @$res, $sid;
7371     }
7372
7373     return $res;
7374 }
7375
7376 1;