PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     },
 641     audio0 => {
 642         type => 'string',
 643         enum => [qw(ich9-intel-hda intel-hda AC97)],
 644         description => "Configure a audio device, useful in combination with QXL/Spice.",
 645         optional => 1
 646     },
 647 };
 648
 649 my $cicustom_fmt = {
 650     meta => {
 651         type => 'string',
 652         optional => 1,
 653         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 654         format => 'pve-volume-id',
 655         format_description => 'volume',
 656     },
 657     network => {
 658         type => 'string',
 659         optional => 1,
 660         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 661         format => 'pve-volume-id',
 662         format_description => 'volume',
 663     },
 664     user => {
 665         type => 'string',
 666         optional => 1,
 667         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 668         format => 'pve-volume-id',
 669         format_description => 'volume',
 670     },
 671 };
 672 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 673
 674 my $confdesc_cloudinit = {
 675     citype => {
 676         optional => 1,
 677         type => 'string',
 678         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 679         enum => ['configdrive2', 'nocloud'],
 680     },
 681     ciuser => {
 682         optional => 1,
 683         type => 'string',
 684         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 685     },
 686     cipassword => {
 687         optional => 1,
 688         type => 'string',
 689         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 690     },
 691     cicustom => {
 692         optional => 1,
 693         type => 'string',
 694         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 695         format => 'pve-qm-cicustom',
 696     },
 697     searchdomain => {
 698         optional => 1,
 699         type => 'string',
 700         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 701     },
 702     nameserver => {
 703         optional => 1,
 704         type => 'string', format => 'address-list',
 705         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 706     },
 707     sshkeys => {
 708         optional => 1,
 709         type => 'string',
 710         format => 'urlencoded',
 711         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 712     },
 713 };
 714
 715 # what about other qemu settings ?
 716 #cpu => 'string',
 717 #machine => 'string',
 718 #fda => 'file',
 719 #fdb => 'file',
 720 #mtdblock => 'file',
 721 #sd => 'file',
 722 #pflash => 'file',
 723 #snapshot => 'bool',
 724 #bootp => 'file',
 725 ##tftp => 'dir',
 726 ##smb => 'dir',
 727 #kernel => 'file',
 728 #append => 'string',
 729 #initrd => 'file',
 730 ##soundhw => 'string',
 731
 732 while (my ($k, $v) = each %$confdesc) {
 733     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 734 }
 735
 736 my $MAX_IDE_DISKS = 4;
 737 my $MAX_SCSI_DISKS = 14;
 738 my $MAX_VIRTIO_DISKS = 16;
 739 my $MAX_SATA_DISKS = 6;
 740 my $MAX_USB_DEVICES = 5;
 741 my $MAX_NETS = 32;
 742 my $MAX_UNUSED_DISKS = 256;
 743 my $MAX_HOSTPCI_DEVICES = 4;
 744 my $MAX_SERIAL_PORTS = 4;
 745 my $MAX_PARALLEL_PORTS = 3;
 746 my $MAX_NUMA = 8;
 747
 748 my $numa_fmt = {
 749     cpus => {
 750         type => "string",
 751         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 752         description => "CPUs accessing this NUMA node.",
 753         format_description => "id[-id];...",
 754     },
 755     memory => {
 756         type => "number",
 757         description => "Amount of memory this NUMA node provides.",
 758         optional => 1,
 759     },
 760     hostnodes => {
 761         type => "string",
 762         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 763         description => "Host NUMA nodes to use.",
 764         format_description => "id[-id];...",
 765         optional => 1,
 766     },
 767     policy => {
 768         type => 'string',
 769         enum => [qw(preferred bind interleave)],
 770         description => "NUMA allocation policy.",
 771         optional => 1,
 772     },
 773 };
 774 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 775 my $numadesc = {
 776     optional => 1,
 777     type => 'string', format => $numa_fmt,
 778     description => "NUMA topology.",
 779 };
 780 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 781
 782 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 783     $confdesc->{"numa$i"} = $numadesc;
 784 }
 785
 786 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 787                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 788                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 789 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 790
 791 my $net_fmt_bridge_descr = <<__EOD__;
 792 Bridge to attach the network device to. The Proxmox VE standard bridge
 793 is called 'vmbr0'.
 794
 795 If you do not specify a bridge, we create a kvm user (NATed) network
 796 device, which provides DHCP and DNS services. The following addresses
 797 are used:
 798
 799  10.0.2.2   Gateway
 800  10.0.2.3   DNS Server
 801  10.0.2.4   SMB Server
 802
 803 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 804 __EOD__
 805
 806 my $net_fmt = {
 807     macaddr  => get_standard_option('mac-addr', {
 808         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 809     }),
 810     model => {
 811         type => 'string',
 812         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 813         enum => $nic_model_list,
 814         default_key => 1,
 815     },
 816     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 817     bridge => {
 818         type => 'string',
 819         description => $net_fmt_bridge_descr,
 820         format_description => 'bridge',
 821         optional => 1,
 822     },
 823     queues => {
 824         type => 'integer',
 825         minimum => 0, maximum => 16,
 826         description => 'Number of packet queues to be used on the device.',
 827         optional => 1,
 828     },
 829     rate => {
 830         type => 'number',
 831         minimum => 0,
 832         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 833         optional => 1,
 834     },
 835     tag => {
 836         type => 'integer',
 837         minimum => 1, maximum => 4094,
 838         description => 'VLAN tag to apply to packets on this interface.',
 839         optional => 1,
 840     },
 841     trunks => {
 842         type => 'string',
 843         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 844         description => 'VLAN trunks to pass through this interface.',
 845         format_description => 'vlanid[;vlanid...]',
 846         optional => 1,
 847     },
 848     firewall => {
 849         type => 'boolean',
 850         description => 'Whether this interface should be protected by the firewall.',
 851         optional => 1,
 852     },
 853     link_down => {
 854         type => 'boolean',
 855         description => 'Whether this interface should be disconnected (like pulling the plug).',
 856         optional => 1,
 857     },
 858 };
 859
 860 my $netdesc = {
 861     optional => 1,
 862     type => 'string', format => $net_fmt,
 863     description => "Specify network devices.",
 864 };
 865
 866 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 867
 868 my $ipconfig_fmt = {
 869     ip => {
 870         type => 'string',
 871         format => 'pve-ipv4-config',
 872         format_description => 'IPv4Format/CIDR',
 873         description => 'IPv4 address in CIDR format.',
 874         optional => 1,
 875         default => 'dhcp',
 876     },
 877     gw => {
 878         type => 'string',
 879         format => 'ipv4',
 880         format_description => 'GatewayIPv4',
 881         description => 'Default gateway for IPv4 traffic.',
 882         optional => 1,
 883         requires => 'ip',
 884     },
 885     ip6 => {
 886         type => 'string',
 887         format => 'pve-ipv6-config',
 888         format_description => 'IPv6Format/CIDR',
 889         description => 'IPv6 address in CIDR format.',
 890         optional => 1,
 891         default => 'dhcp',
 892     },
 893     gw6 => {
 894         type => 'string',
 895         format => 'ipv6',
 896         format_description => 'GatewayIPv6',
 897         description => 'Default gateway for IPv6 traffic.',
 898         optional => 1,
 899         requires => 'ip6',
 900     },
 901 };
 902 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 903 my $ipconfigdesc = {
 904     optional => 1,
 905     type => 'string', format => 'pve-qm-ipconfig',
 906     description => <<'EODESCR',
 907 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 908
 909 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 910
 911 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 912 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 913
 914 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 915 EODESCR
 916 };
 917 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 918
 919 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 920     $confdesc->{"net$i"} = $netdesc;
 921     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 922 }
 923
 924 foreach my $key (keys %$confdesc_cloudinit) {
 925     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 926 }
 927
 928 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 929 sub verify_volume_id_or_qm_path {
 930     my ($volid, $noerr) = @_;
 931
 932     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 933         return $volid;
 934     }
 935
 936     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 937     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 938     if ($@) {
 939         return undef if $noerr;
 940         die $@;
 941     }
 942     return $volid;
 943 }
 944
 945 my $drivename_hash;
 946
 947 my %drivedesc_base = (
 948     volume => { alias => 'file' },
 949     file => {
 950         type => 'string',
 951         format => 'pve-volume-id-or-qm-path',
 952         default_key => 1,
 953         format_description => 'volume',
 954         description => "The drive's backing volume.",
 955     },
 956     media => {
 957         type => 'string',
 958         enum => [qw(cdrom disk)],
 959         description => "The drive's media type.",
 960         default => 'disk',
 961         optional => 1
 962     },
 963     cyls => {
 964         type => 'integer',
 965         description => "Force the drive's physical geometry to have a specific cylinder count.",
 966         optional => 1
 967     },
 968     heads => {
 969         type => 'integer',
 970         description => "Force the drive's physical geometry to have a specific head count.",
 971         optional => 1
 972     },
 973     secs => {
 974         type => 'integer',
 975         description => "Force the drive's physical geometry to have a specific sector count.",
 976         optional => 1
 977     },
 978     trans => {
 979         type => 'string',
 980         enum => [qw(none lba auto)],
 981         description => "Force disk geometry bios translation mode.",
 982         optional => 1,
 983     },
 984     snapshot => {
 985         type => 'boolean',
 986         description => "Controls qemu's snapshot mode feature."
 987             . " If activated, changes made to the disk are temporary and will"
 988             . " be discarded when the VM is shutdown.",
 989         optional => 1,
 990     },
 991     cache => {
 992         type => 'string',
 993         enum => [qw(none writethrough writeback unsafe directsync)],
 994         description => "The drive's cache mode",
 995         optional => 1,
 996     },
 997     format => get_standard_option('pve-qm-image-format'),
 998     size => {
 999         type => 'string',
1000         format => 'disk-size',
1001         format_description => 'DiskSize',
1002         description => "Disk size. This is purely informational and has no effect.",
1003         optional => 1,
1004     },
1005     backup => {
1006         type => 'boolean',
1007         description => "Whether the drive should be included when making backups.",
1008         optional => 1,
1009     },
1010     replicate => {
1011         type => 'boolean',
1012         description => 'Whether the drive should considered for replication jobs.',
1013         optional => 1,
1014         default => 1,
1015     },
1016     rerror => {
1017         type => 'string',
1018         enum => [qw(ignore report stop)],
1019         description => 'Read error action.',
1020         optional => 1,
1021     },
1022     werror => {
1023         type => 'string',
1024         enum => [qw(enospc ignore report stop)],
1025         description => 'Write error action.',
1026         optional => 1,
1027     },
1028     aio => {
1029         type => 'string',
1030         enum => [qw(native threads)],
1031         description => 'AIO type to use.',
1032         optional => 1,
1033     },
1034     discard => {
1035         type => 'string',
1036         enum => [qw(ignore on)],
1037         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1038         optional => 1,
1039     },
1040     detect_zeroes => {
1041         type => 'boolean',
1042         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1043         optional => 1,
1044     },
1045     serial => {
1046         type => 'string',
1047         format => 'urlencoded',
1048         format_description => 'serial',
1049         maxLength => 20*3, # *3 since it's %xx url enoded
1050         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1051         optional => 1,
1052     },
1053     shared => {
1054         type => 'boolean',
1055         description => 'Mark this locally-managed volume as available on all nodes',
1056         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1057         optional => 1,
1058         default => 0,
1059     }
1060 );
1061
1062 my %iothread_fmt = ( iothread => {
1063         type => 'boolean',
1064         description => "Whether to use iothreads for this drive",
1065         optional => 1,
1066 });
1067
1068 my %model_fmt = (
1069     model => {
1070         type => 'string',
1071         format => 'urlencoded',
1072         format_description => 'model',
1073         maxLength => 40*3, # *3 since it's %xx url enoded
1074         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1075         optional => 1,
1076     },
1077 );
1078
1079 my %queues_fmt = (
1080     queues => {
1081         type => 'integer',
1082         description => "Number of queues.",
1083         minimum => 2,
1084         optional => 1
1085     }
1086 );
1087
1088 my %scsiblock_fmt = (
1089     scsiblock => {
1090         type => 'boolean',
1091         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1092         optional => 1,
1093         default => 0,
1094     },
1095 );
1096
1097 my %ssd_fmt = (
1098     ssd => {
1099         type => 'boolean',
1100         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1101         optional => 1,
1102     },
1103 );
1104
1105 my %wwn_fmt = (
1106     wwn => {
1107         type => 'string',
1108         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1109         format_description => 'wwn',
1110         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1111         optional => 1,
1112     },
1113 );
1114
1115 my $add_throttle_desc = sub {
1116     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1117     my $d = {
1118         type => $type,
1119         format_description => $unit,
1120         description => "Maximum $what in $longunit.",
1121         optional => 1,
1122     };
1123     $d->{minimum} = $minimum if defined($minimum);
1124     $drivedesc_base{$key} = $d;
1125 };
1126 # throughput: (leaky bucket)
1127 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1128 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1129 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1130 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1131 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1132 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1133 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1134 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1135 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1136
1137 # pools: (pool of IO before throttling starts taking effect)
1138 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1139 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1140 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1141 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1142 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1143 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1144
1145 # burst lengths
1146 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1147 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1148 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1150 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1151 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1152
1153 # legacy support
1154 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1155 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1156 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1157 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1158
1159 my $ide_fmt = {
1160     %drivedesc_base,
1161     %model_fmt,
1162     %ssd_fmt,
1163     %wwn_fmt,
1164 };
1165 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1166
1167 my $idedesc = {
1168     optional => 1,
1169     type => 'string', format => $ide_fmt,
1170     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1171 };
1172 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1173
1174 my $scsi_fmt = {
1175     %drivedesc_base,
1176     %iothread_fmt,
1177     %queues_fmt,
1178     %scsiblock_fmt,
1179     %ssd_fmt,
1180     %wwn_fmt,
1181 };
1182 my $scsidesc = {
1183     optional => 1,
1184     type => 'string', format => $scsi_fmt,
1185     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1186 };
1187 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1188
1189 my $sata_fmt = {
1190     %drivedesc_base,
1191     %ssd_fmt,
1192     %wwn_fmt,
1193 };
1194 my $satadesc = {
1195     optional => 1,
1196     type => 'string', format => $sata_fmt,
1197     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1198 };
1199 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1200
1201 my $virtio_fmt = {
1202     %drivedesc_base,
1203     %iothread_fmt,
1204 };
1205 my $virtiodesc = {
1206     optional => 1,
1207     type => 'string', format => $virtio_fmt,
1208     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1209 };
1210 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1211
1212 my $alldrive_fmt = {
1213     %drivedesc_base,
1214     %iothread_fmt,
1215     %model_fmt,
1216     %queues_fmt,
1217     %scsiblock_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221
1222 my $efidisk_fmt = {
1223     volume => { alias => 'file' },
1224     file => {
1225         type => 'string',
1226         format => 'pve-volume-id-or-qm-path',
1227         default_key => 1,
1228         format_description => 'volume',
1229         description => "The drive's backing volume.",
1230     },
1231     format => get_standard_option('pve-qm-image-format'),
1232     size => {
1233         type => 'string',
1234         format => 'disk-size',
1235         format_description => 'DiskSize',
1236         description => "Disk size. This is purely informational and has no effect.",
1237         optional => 1,
1238     },
1239 };
1240
1241 my $efidisk_desc = {
1242     optional => 1,
1243     type => 'string', format => $efidisk_fmt,
1244     description => "Configure a Disk for storing EFI vars",
1245 };
1246
1247 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1248
1249 my $usb_fmt = {
1250     host => {
1251         default_key => 1,
1252         type => 'string', format => 'pve-qm-usb-device',
1253         format_description => 'HOSTUSBDEVICE|spice',
1254         description => <<EODESCR,
1255 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1256
1257  'bus-port(.port)*' (decimal numbers) or
1258  'vendor_id:product_id' (hexadeciaml numbers) or
1259  'spice'
1260
1261 You can use the 'lsusb -t' command to list existing usb devices.
1262
1263 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1264
1265 The value 'spice' can be used to add a usb redirection devices for spice.
1266 EODESCR
1267     },
1268     usb3 => {
1269         optional => 1,
1270         type => 'boolean',
1271         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1272         default => 0,
1273     },
1274 };
1275
1276 my $usbdesc = {
1277     optional => 1,
1278     type => 'string', format => $usb_fmt,
1279     description => "Configure an USB device (n is 0 to 4).",
1280 };
1281 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1282
1283 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1284 my $hostpci_fmt = {
1285     host => {
1286         default_key => 1,
1287         type => 'string',
1288         pattern => qr/$PCIRE(;$PCIRE)*/,
1289         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1290         description => <<EODESCR,
1291 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1292 of PCI virtual functions of the host. HOSTPCIID syntax is:
1293
1294 'bus:dev.func' (hexadecimal numbers)
1295
1296 You can us the 'lspci' command to list existing PCI devices.
1297 EODESCR
1298     },
1299     rombar => {
1300         type => 'boolean',
1301         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1302         optional => 1,
1303         default => 1,
1304     },
1305     romfile => {
1306         type => 'string',
1307         pattern => '[^,;]+',
1308         format_description => 'string',
1309         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1310         optional => 1,
1311     },
1312     pcie => {
1313         type => 'boolean',
1314         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'x-vga' => {
1319         type => 'boolean',
1320         description =>  "Enable vfio-vga device support.",
1321         optional => 1,
1322         default => 0,
1323     },
1324     'mdev' => {
1325         type => 'string',
1326         format_description => 'string',
1327         pattern => '[^/\.:]+',
1328         optional => 1,
1329         description => <<EODESCR
1330 The type of mediated device to use.
1331 An instance of this type will be created on startup of the VM and
1332 will be cleaned up when the VM stops.
1333 EODESCR
1334     }
1335 };
1336 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1337
1338 my $hostpcidesc = {
1339         optional => 1,
1340         type => 'string', format => 'pve-qm-hostpci',
1341         description => "Map host PCI devices into guest.",
1342         verbose_description =>  <<EODESCR,
1343 Map host PCI devices into guest.
1344
1345 NOTE: This option allows direct access to host hardware. So it is no longer
1346 possible to migrate such machines - use with special care.
1347
1348 CAUTION: Experimental! User reported problems with this option.
1349 EODESCR
1350 };
1351 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1352
1353 my $serialdesc = {
1354         optional => 1,
1355         type => 'string',
1356         pattern => '(/dev/.+|socket)',
1357         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1358         verbose_description =>  <<EODESCR,
1359 Create a serial device inside the VM (n is 0 to 3), and pass through a
1360 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1361 host side (use 'qm terminal' to open a terminal connection).
1362
1363 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1364
1365 CAUTION: Experimental! User reported problems with this option.
1366 EODESCR
1367 };
1368
1369 my $paralleldesc= {
1370         optional => 1,
1371         type => 'string',
1372         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1373         description =>  "Map host parallel devices (n is 0 to 2).",
1374         verbose_description =>  <<EODESCR,
1375 Map host parallel devices (n is 0 to 2).
1376
1377 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1378
1379 CAUTION: Experimental! User reported problems with this option.
1380 EODESCR
1381 };
1382
1383 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1384     $confdesc->{"parallel$i"} = $paralleldesc;
1385 }
1386
1387 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1388     $confdesc->{"serial$i"} = $serialdesc;
1389 }
1390
1391 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1392     $confdesc->{"hostpci$i"} = $hostpcidesc;
1393 }
1394
1395 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1396     $drivename_hash->{"ide$i"} = 1;
1397     $confdesc->{"ide$i"} = $idedesc;
1398 }
1399
1400 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1401     $drivename_hash->{"sata$i"} = 1;
1402     $confdesc->{"sata$i"} = $satadesc;
1403 }
1404
1405 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1406     $drivename_hash->{"scsi$i"} = 1;
1407     $confdesc->{"scsi$i"} = $scsidesc ;
1408 }
1409
1410 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1411     $drivename_hash->{"virtio$i"} = 1;
1412     $confdesc->{"virtio$i"} = $virtiodesc;
1413 }
1414
1415 $drivename_hash->{efidisk0} = 1;
1416 $confdesc->{efidisk0} = $efidisk_desc;
1417
1418 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1419     $confdesc->{"usb$i"} = $usbdesc;
1420 }
1421
1422 my $unuseddesc = {
1423     optional => 1,
1424     type => 'string', format => 'pve-volume-id',
1425     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1426 };
1427
1428 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1429     $confdesc->{"unused$i"} = $unuseddesc;
1430 }
1431
1432 my $kvm_api_version = 0;
1433
1434 sub kvm_version {
1435     return $kvm_api_version if $kvm_api_version;
1436
1437     open my $fh, '<', '/dev/kvm'
1438         or return undef;
1439
1440     # 0xae00 => KVM_GET_API_VERSION
1441     $kvm_api_version = ioctl($fh, 0xae00, 0);
1442
1443     return $kvm_api_version;
1444 }
1445
1446 my $kvm_user_version;
1447
1448 sub kvm_user_version {
1449
1450     return $kvm_user_version if $kvm_user_version;
1451
1452     $kvm_user_version = 'unknown';
1453
1454     my $code = sub {
1455         my $line = shift;
1456         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1457             $kvm_user_version = $2;
1458         }
1459     };
1460
1461     eval { run_command("kvm -version", outfunc => $code); };
1462     warn $@ if $@;
1463
1464     return $kvm_user_version;
1465
1466 }
1467
1468 sub kernel_has_vhost_net {
1469     return -c '/dev/vhost-net';
1470 }
1471
1472 sub valid_drive_names {
1473     # order is important - used to autoselect boot disk
1474     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1475             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1476             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1477             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1478             'efidisk0');
1479 }
1480
1481 sub is_valid_drivename {
1482     my $dev = shift;
1483
1484     return defined($drivename_hash->{$dev});
1485 }
1486
1487 sub option_exists {
1488     my $key = shift;
1489     return defined($confdesc->{$key});
1490 }
1491
1492 sub nic_models {
1493     return $nic_model_list;
1494 }
1495
1496 sub os_list_description {
1497
1498     return {
1499         other => 'Other',
1500         wxp => 'Windows XP',
1501         w2k => 'Windows 2000',
1502         w2k3 =>, 'Windows 2003',
1503         w2k8 => 'Windows 2008',
1504         wvista => 'Windows Vista',
1505         win7 => 'Windows 7',
1506         win8 => 'Windows 8/2012',
1507         win10 => 'Windows 10/2016',
1508         l24 => 'Linux 2.4',
1509         l26 => 'Linux 2.6',
1510     };
1511 }
1512
1513 my $cdrom_path;
1514
1515 sub get_cdrom_path {
1516
1517     return  $cdrom_path if $cdrom_path;
1518
1519     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1520     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1521     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1522 }
1523
1524 sub get_iso_path {
1525     my ($storecfg, $vmid, $cdrom) = @_;
1526
1527     if ($cdrom eq 'cdrom') {
1528         return get_cdrom_path();
1529     } elsif ($cdrom eq 'none') {
1530         return '';
1531     } elsif ($cdrom =~ m|^/|) {
1532         return $cdrom;
1533     } else {
1534         return PVE::Storage::path($storecfg, $cdrom);
1535     }
1536 }
1537
1538 # try to convert old style file names to volume IDs
1539 sub filename_to_volume_id {
1540     my ($vmid, $file, $media) = @_;
1541
1542      if (!($file eq 'none' || $file eq 'cdrom' ||
1543           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1544
1545         return undef if $file =~ m|/|;
1546
1547         if ($media && $media eq 'cdrom') {
1548             $file = "local:iso/$file";
1549         } else {
1550             $file = "local:$vmid/$file";
1551         }
1552     }
1553
1554     return $file;
1555 }
1556
1557 sub verify_media_type {
1558     my ($opt, $vtype, $media) = @_;
1559
1560     return if !$media;
1561
1562     my $etype;
1563     if ($media eq 'disk') {
1564         $etype = 'images';
1565     } elsif ($media eq 'cdrom') {
1566         $etype = 'iso';
1567     } else {
1568         die "internal error";
1569     }
1570
1571     return if ($vtype eq $etype);
1572
1573     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1574 }
1575
1576 sub cleanup_drive_path {
1577     my ($opt, $storecfg, $drive) = @_;
1578
1579     # try to convert filesystem paths to volume IDs
1580
1581     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1582         ($drive->{file} !~ m|^/dev/.+|) &&
1583         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1584         ($drive->{file} !~ m/^\d+$/)) {
1585         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1586         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1587         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1588         verify_media_type($opt, $vtype, $drive->{media});
1589         $drive->{file} = $volid;
1590     }
1591
1592     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1593 }
1594
1595 sub parse_hotplug_features {
1596     my ($data) = @_;
1597
1598     my $res = {};
1599
1600     return $res if $data eq '0';
1601
1602     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1603
1604     foreach my $feature (PVE::Tools::split_list($data)) {
1605         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1606             $res->{$1} = 1;
1607         } else {
1608             die "invalid hotplug feature '$feature'\n";
1609         }
1610     }
1611     return $res;
1612 }
1613
1614 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1615 sub pve_verify_hotplug_features {
1616     my ($value, $noerr) = @_;
1617
1618     return $value if parse_hotplug_features($value);
1619
1620     return undef if $noerr;
1621
1622     die "unable to parse hotplug option\n";
1623 }
1624
1625 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1626 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1627 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1628 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1629 #        [,iothread=on][,serial=serial][,model=model]
1630
1631 sub parse_drive {
1632     my ($key, $data) = @_;
1633
1634     my ($interface, $index);
1635
1636     if ($key =~ m/^([^\d]+)(\d+)$/) {
1637         $interface = $1;
1638         $index = $2;
1639     } else {
1640         return undef;
1641     }
1642
1643     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1644                                      : $confdesc->{$key}->{format};
1645     if (!$desc) {
1646         warn "invalid drive key: $key\n";
1647         return undef;
1648     }
1649     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1650     return undef if !$res;
1651     $res->{interface} = $interface;
1652     $res->{index} = $index;
1653
1654     my $error = 0;
1655     foreach my $opt (qw(bps bps_rd bps_wr)) {
1656         if (my $bps = defined(delete $res->{$opt})) {
1657             if (defined($res->{"m$opt"})) {
1658                 warn "both $opt and m$opt specified\n";
1659                 ++$error;
1660                 next;
1661             }
1662             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1663         }
1664     }
1665
1666     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1667     for my $requirement (
1668         [mbps_max => 'mbps'],
1669         [mbps_rd_max => 'mbps_rd'],
1670         [mbps_wr_max => 'mbps_wr'],
1671         [miops_max => 'miops'],
1672         [miops_rd_max => 'miops_rd'],
1673         [miops_wr_max => 'miops_wr'],
1674         [bps_max_length => 'mbps_max'],
1675         [bps_rd_max_length => 'mbps_rd_max'],
1676         [bps_wr_max_length => 'mbps_wr_max'],
1677         [iops_max_length => 'iops_max'],
1678         [iops_rd_max_length => 'iops_rd_max'],
1679         [iops_wr_max_length => 'iops_wr_max']) {
1680         my ($option, $requires) = @$requirement;
1681         if ($res->{$option} && !$res->{$requires}) {
1682             warn "$option requires $requires\n";
1683             ++$error;
1684         }
1685     }
1686
1687     return undef if $error;
1688
1689     return undef if $res->{mbps_rd} && $res->{mbps};
1690     return undef if $res->{mbps_wr} && $res->{mbps};
1691     return undef if $res->{iops_rd} && $res->{iops};
1692     return undef if $res->{iops_wr} && $res->{iops};
1693
1694     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1695         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1696         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1697         return undef if $res->{interface} eq 'virtio';
1698     }
1699
1700     if (my $size = $res->{size}) {
1701         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1702     }
1703
1704     return $res;
1705 }
1706
1707 sub print_drive {
1708     my ($vmid, $drive) = @_;
1709     my $data = { %$drive };
1710     delete $data->{$_} for qw(index interface);
1711     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1712 }
1713
1714 sub scsi_inquiry {
1715     my($fh, $noerr) = @_;
1716
1717     my $SG_IO = 0x2285;
1718     my $SG_GET_VERSION_NUM = 0x2282;
1719
1720     my $versionbuf = "\x00" x 8;
1721     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1722     if (!$ret) {
1723         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1724         return undef;
1725     }
1726     my $version = unpack("I", $versionbuf);
1727     if ($version < 30000) {
1728         die "scsi generic interface too old\n"  if !$noerr;
1729         return undef;
1730     }
1731
1732     my $buf = "\x00" x 36;
1733     my $sensebuf = "\x00" x 8;
1734     my $cmd = pack("C x3 C x1", 0x12, 36);
1735
1736     # see /usr/include/scsi/sg.h
1737     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1738
1739     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1740                       length($sensebuf), 0, length($buf), $buf,
1741                       $cmd, $sensebuf, 6000);
1742
1743     $ret = ioctl($fh, $SG_IO, $packet);
1744     if (!$ret) {
1745         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my @res = unpack($sg_io_hdr_t, $packet);
1750     if ($res[17] || $res[18]) {
1751         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1752         return undef;
1753     }
1754
1755     my $res = {};
1756     (my $byte0, my $byte1, $res->{vendor},
1757      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1758
1759     $res->{removable} = $byte1 & 128 ? 1 : 0;
1760     $res->{type} = $byte0 & 31;
1761
1762     return $res;
1763 }
1764
1765 sub path_is_scsi {
1766     my ($path) = @_;
1767
1768     my $fh = IO::File->new("+<$path") || return undef;
1769     my $res = scsi_inquiry($fh, 1);
1770     close($fh);
1771
1772     return $res;
1773 }
1774
1775 sub machine_type_is_q35 {
1776     my ($conf) = @_;
1777
1778     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1779 }
1780
1781 sub print_tabletdevice_full {
1782     my ($conf, $arch) = @_;
1783
1784     my $q35 = machine_type_is_q35($conf);
1785
1786     # we use uhci for old VMs because tablet driver was buggy in older qemu
1787     my $usbbus;
1788     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1789         $usbbus = 'ehci';
1790     } else {
1791         $usbbus = 'uhci';
1792     }
1793
1794     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1795 }
1796
1797 sub print_keyboarddevice_full {
1798     my ($conf, $arch, $machine) = @_;
1799
1800     return undef if $arch ne 'aarch64';
1801
1802     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1803 }
1804
1805 sub print_drivedevice_full {
1806     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1807
1808     my $device = '';
1809     my $maxdev = 0;
1810
1811     if ($drive->{interface} eq 'virtio') {
1812         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1813         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1814         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1815     } elsif ($drive->{interface} eq 'scsi') {
1816
1817         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1818         my $unit = $drive->{index} % $maxdev;
1819         my $devicetype = 'hd';
1820         my $path = '';
1821         if (drive_is_cdrom($drive)) {
1822             $devicetype = 'cd';
1823         } else {
1824             if ($drive->{file} =~ m|^/|) {
1825                 $path = $drive->{file};
1826                 if (my $info = path_is_scsi($path)) {
1827                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1828                         $devicetype = 'block';
1829                     } elsif ($info->{type} == 1) { # tape
1830                         $devicetype = 'generic';
1831                     }
1832                 }
1833             } else {
1834                  $path = PVE::Storage::path($storecfg, $drive->{file});
1835             }
1836
1837             if($path =~ m/^iscsi\:\/\//){
1838                 $devicetype = 'generic';
1839             }
1840         }
1841
1842         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         } else {
1845             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1846         }
1847
1848         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1849             $device .= ",rotation_rate=1";
1850         }
1851         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1852
1853     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1854         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1855         my $controller = int($drive->{index} / $maxdev);
1856         my $unit = $drive->{index} % $maxdev;
1857         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1858
1859         $device = "ide-$devicetype";
1860         if ($drive->{interface} eq 'ide') {
1861             $device .= ",bus=ide.$controller,unit=$unit";
1862         } else {
1863             $device .= ",bus=ahci$controller.$unit";
1864         }
1865         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1866
1867         if ($devicetype eq 'hd') {
1868             if (my $model = $drive->{model}) {
1869                 $model = URI::Escape::uri_unescape($model);
1870                 $device .= ",model=$model";
1871             }
1872             if ($drive->{ssd}) {
1873                 $device .= ",rotation_rate=1";
1874             }
1875         }
1876         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1877     } elsif ($drive->{interface} eq 'usb') {
1878         die "implement me";
1879         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1880     } else {
1881         die "unsupported interface type";
1882     }
1883
1884     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1885
1886     if (my $serial = $drive->{serial}) {
1887         $serial = URI::Escape::uri_unescape($serial);
1888         $device .= ",serial=$serial";
1889     }
1890
1891
1892     return $device;
1893 }
1894
1895 sub get_initiator_name {
1896     my $initiator;
1897
1898     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1899     while (defined(my $line = <$fh>)) {
1900         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1901         $initiator = $1;
1902         last;
1903     }
1904     $fh->close();
1905
1906     return $initiator;
1907 }
1908
1909 sub print_drive_full {
1910     my ($storecfg, $vmid, $drive) = @_;
1911
1912     my $path;
1913     my $volid = $drive->{file};
1914     my $format;
1915
1916     if (drive_is_cdrom($drive)) {
1917         $path = get_iso_path($storecfg, $vmid, $volid);
1918     } else {
1919         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1920         if ($storeid) {
1921             $path = PVE::Storage::path($storecfg, $volid);
1922             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1923             $format = qemu_img_format($scfg, $volname);
1924         } else {
1925             $path = $volid;
1926             $format = "raw";
1927         }
1928    }
1929
1930     my $opts = '';
1931     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1932     foreach my $o (@qemu_drive_options) {
1933         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1934     }
1935
1936     # snapshot only accepts on|off
1937     if (defined($drive->{snapshot})) {
1938         my $v = $drive->{snapshot} ? 'on' : 'off';
1939         $opts .= ",snapshot=$v";
1940     }
1941
1942     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1943         my ($dir, $qmpname) = @$type;
1944         if (my $v = $drive->{"mbps$dir"}) {
1945             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1946         }
1947         if (my $v = $drive->{"mbps${dir}_max"}) {
1948             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1949         }
1950         if (my $v = $drive->{"bps${dir}_max_length"}) {
1951             $opts .= ",throttling.bps$qmpname-max-length=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}"}) {
1954             $opts .= ",throttling.iops$qmpname=$v";
1955         }
1956         if (my $v = $drive->{"iops${dir}_max"}) {
1957             $opts .= ",throttling.iops$qmpname-max=$v";
1958         }
1959         if (my $v = $drive->{"iops${dir}_max_length"}) {
1960             $opts .= ",throttling.iops$qmpname-max-length=$v";
1961         }
1962     }
1963
1964     $opts .= ",format=$format" if $format && !$drive->{format};
1965
1966     my $cache_direct = 0;
1967
1968     if (my $cache = $drive->{cache}) {
1969         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1970     } elsif (!drive_is_cdrom($drive)) {
1971         $opts .= ",cache=none";
1972         $cache_direct = 1;
1973     }
1974
1975     # aio native works only with O_DIRECT
1976     if (!$drive->{aio}) {
1977         if($cache_direct) {
1978             $opts .= ",aio=native";
1979         } else {
1980             $opts .= ",aio=threads";
1981         }
1982     }
1983
1984     if (!drive_is_cdrom($drive)) {
1985         my $detectzeroes;
1986         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1987             $detectzeroes = 'off';
1988         } elsif ($drive->{discard}) {
1989             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1990         } else {
1991             # This used to be our default with discard not being specified:
1992             $detectzeroes = 'on';
1993         }
1994         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1995     }
1996
1997     my $pathinfo = $path ? "file=$path," : '';
1998
1999     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2000 }
2001
2002 sub print_netdevice_full {
2003     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2004
2005     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2006
2007     my $device = $net->{model};
2008     if ($net->{model} eq 'virtio') {
2009          $device = 'virtio-net-pci';
2010      };
2011
2012     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2013     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2014     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2015         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2016         my $vectors = $net->{queues} * 2 + 2;
2017         $tmpstr .= ",vectors=$vectors,mq=on";
2018     }
2019     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2020
2021     if ($use_old_bios_files) {
2022         my $romfile;
2023         if ($device eq 'virtio-net-pci') {
2024             $romfile = 'pxe-virtio.rom';
2025         } elsif ($device eq 'e1000') {
2026             $romfile = 'pxe-e1000.rom';
2027         } elsif ($device eq 'ne2k') {
2028             $romfile = 'pxe-ne2k_pci.rom';
2029         } elsif ($device eq 'pcnet') {
2030             $romfile = 'pxe-pcnet.rom';
2031         } elsif ($device eq 'rtl8139') {
2032             $romfile = 'pxe-rtl8139.rom';
2033         }
2034         $tmpstr .= ",romfile=$romfile" if $romfile;
2035     }
2036
2037     return $tmpstr;
2038 }
2039
2040 sub print_netdev_full {
2041     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2042
2043     my $i = '';
2044     if ($netid =~ m/^net(\d+)$/) {
2045         $i = int($1);
2046     }
2047
2048     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2049
2050     my $ifname = "tap${vmid}i$i";
2051
2052     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2053     die "interface name '$ifname' is too long (max 15 character)\n"
2054         if length($ifname) >= 16;
2055
2056     my $vhostparam = '';
2057     if (is_native($arch)) {
2058         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2059     }
2060
2061     my $vmname = $conf->{name} || "vm$vmid";
2062
2063     my $netdev = "";
2064     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2065
2066     if ($net->{bridge}) {
2067         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2068     } else {
2069         $netdev = "type=user,id=$netid,hostname=$vmname";
2070     }
2071
2072     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2073
2074     return $netdev;
2075 }
2076
2077
2078 sub print_cpu_device {
2079     my ($conf, $id) = @_;
2080
2081     my $kvm = $conf->{kvm} // 1;
2082     my $cpu = $kvm ? "kvm64" : "qemu64";
2083     if (my $cputype = $conf->{cpu}) {
2084         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2085             or die "Cannot parse cpu description: $cputype\n";
2086         $cpu = $cpuconf->{cputype};
2087     }
2088
2089     my $cores = $conf->{cores} || 1;
2090
2091     my $current_core = ($id - 1) % $cores;
2092     my $current_socket = int(($id - 1 - $current_core)/$cores);
2093
2094     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2095 }
2096
2097 my $vga_map = {
2098     'cirrus' => 'cirrus-vga',
2099     'std' => 'VGA',
2100     'vmware' => 'vmware-svga',
2101     'virtio' => 'virtio-vga',
2102 };
2103
2104 sub print_vga_device {
2105     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2106
2107     my $type = $vga_map->{$vga->{type}};
2108     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2109         $type = 'virtio-gpu';
2110     }
2111     my $vgamem_mb = $vga->{memory};
2112     if ($qxlnum) {
2113         $type = $id ? 'qxl' : 'qxl-vga';
2114     }
2115     die "no devicetype for $vga->{type}\n" if !$type;
2116
2117     my $memory = "";
2118     if ($vgamem_mb) {
2119         if ($vga->{type} eq 'virtio') {
2120             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2121             $memory = ",max_hostmem=$bytes";
2122         } elsif ($qxlnum) {
2123             # from https://www.spice-space.org/multiple-monitors.html
2124             $memory = ",vgamem_mb=$vga->{memory}";
2125             my $ram = $vgamem_mb * 4;
2126             my $vram = $vgamem_mb * 2;
2127             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2128         } else {
2129             $memory = ",vgamem_mb=$vga->{memory}";
2130         }
2131     } elsif ($qxlnum && $id) {
2132         $memory = ",ram_size=67108864,vram_size=33554432";
2133     }
2134
2135     my $q35 = machine_type_is_q35($conf);
2136     my $vgaid = "vga" . ($id // '');
2137     my $pciaddr;
2138
2139     if ($q35 && $vgaid eq 'vga') {
2140         # the first display uses pcie.0 bus on q35 machines
2141         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2142     } else {
2143         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2144     }
2145
2146     return "$type,id=${vgaid}${memory}${pciaddr}";
2147 }
2148
2149 sub drive_is_cloudinit {
2150     my ($drive) = @_;
2151     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2152 }
2153
2154 sub drive_is_cdrom {
2155     my ($drive, $exclude_cloudinit) = @_;
2156
2157     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2158
2159     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2160
2161 }
2162
2163 sub parse_number_sets {
2164     my ($set) = @_;
2165     my $res = [];
2166     foreach my $part (split(/;/, $set)) {
2167         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2168             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2169             push @$res, [ $1, $2 ];
2170         } else {
2171             die "invalid range: $part\n";
2172         }
2173     }
2174     return $res;
2175 }
2176
2177 sub parse_numa {
2178     my ($data) = @_;
2179
2180     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2181     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2182     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2183     return $res;
2184 }
2185
2186 sub parse_hostpci {
2187     my ($value) = @_;
2188
2189     return undef if !$value;
2190
2191     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2192
2193     my @idlist = split(/;/, $res->{host});
2194     delete $res->{host};
2195     foreach my $id (@idlist) {
2196         if ($id =~ m/\./) { # full id 00:00.1
2197             push @{$res->{pciid}}, {
2198                 id => $id,
2199             };
2200         } else { # partial id 00:00
2201             $res->{pciid} = PVE::SysFSTools::lspci($id);
2202         }
2203     }
2204     return $res;
2205 }
2206
2207 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2208 sub parse_net {
2209     my ($data) = @_;
2210
2211     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2212     if ($@) {
2213         warn $@;
2214         return undef;
2215     }
2216     if (!defined($res->{macaddr})) {
2217         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2218         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2219     }
2220     return $res;
2221 }
2222
2223 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2224 sub parse_ipconfig {
2225     my ($data) = @_;
2226
2227     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2228     if ($@) {
2229         warn $@;
2230         return undef;
2231     }
2232
2233     if ($res->{gw} && !$res->{ip}) {
2234         warn 'gateway specified without specifying an IP address';
2235         return undef;
2236     }
2237     if ($res->{gw6} && !$res->{ip6}) {
2238         warn 'IPv6 gateway specified without specifying an IPv6 address';
2239         return undef;
2240     }
2241     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2242         warn 'gateway specified together with DHCP';
2243         return undef;
2244     }
2245     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2246         # gw6 + auto/dhcp
2247         warn "IPv6 gateway specified together with $res->{ip6} address";
2248         return undef;
2249     }
2250
2251     if (!$res->{ip} && !$res->{ip6}) {
2252         return { ip => 'dhcp', ip6 => 'dhcp' };
2253     }
2254
2255     return $res;
2256 }
2257
2258 sub print_net {
2259     my $net = shift;
2260
2261     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2262 }
2263
2264 sub add_random_macs {
2265     my ($settings) = @_;
2266
2267     foreach my $opt (keys %$settings) {
2268         next if $opt !~ m/^net(\d+)$/;
2269         my $net = parse_net($settings->{$opt});
2270         next if !$net;
2271         $settings->{$opt} = print_net($net);
2272     }
2273 }
2274
2275 sub vm_is_volid_owner {
2276     my ($storecfg, $vmid, $volid) = @_;
2277
2278     if ($volid !~  m|^/|) {
2279         my ($path, $owner);
2280         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2281         if ($owner && ($owner == $vmid)) {
2282             return 1;
2283         }
2284     }
2285
2286     return undef;
2287 }
2288
2289 sub split_flagged_list {
2290     my $text = shift || '';
2291     $text =~ s/[,;]/ /g;
2292     $text =~ s/^\s+//;
2293     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2294 }
2295
2296 sub join_flagged_list {
2297     my ($how, $lst) = @_;
2298     join $how, map { $lst->{$_} . $_ } keys %$lst;
2299 }
2300
2301 sub vmconfig_delete_pending_option {
2302     my ($conf, $key, $force) = @_;
2303
2304     delete $conf->{pending}->{$key};
2305     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2306     $pending_delete_hash->{$key} = $force ? '!' : '';
2307     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2308 }
2309
2310 sub vmconfig_undelete_pending_option {
2311     my ($conf, $key) = @_;
2312
2313     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2314     delete $pending_delete_hash->{$key};
2315
2316     if (%$pending_delete_hash) {
2317         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2318     } else {
2319         delete $conf->{pending}->{delete};
2320     }
2321 }
2322
2323 sub vmconfig_register_unused_drive {
2324     my ($storecfg, $vmid, $conf, $drive) = @_;
2325
2326     if (drive_is_cloudinit($drive)) {
2327         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2328         warn $@ if $@;
2329     } elsif (!drive_is_cdrom($drive)) {
2330         my $volid = $drive->{file};
2331         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2332             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2333         }
2334     }
2335 }
2336
2337 sub vmconfig_cleanup_pending {
2338     my ($conf) = @_;
2339
2340     # remove pending changes when nothing changed
2341     my $changes;
2342     foreach my $opt (keys %{$conf->{pending}}) {
2343         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2344             $changes = 1;
2345             delete $conf->{pending}->{$opt};
2346         }
2347     }
2348
2349     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2350     my $pending_delete_hash = {};
2351     while (my ($opt, $force) = each %$current_delete_hash) {
2352         if (defined($conf->{$opt})) {
2353             $pending_delete_hash->{$opt} = $force;
2354         } else {
2355             $changes = 1;
2356         }
2357     }
2358
2359     if (%$pending_delete_hash) {
2360         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2361     } else {
2362         delete $conf->{pending}->{delete};
2363     }
2364
2365     return $changes;
2366 }
2367
2368 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2369 my $smbios1_fmt = {
2370     uuid => {
2371         type => 'string',
2372         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2373         format_description => 'UUID',
2374         description => "Set SMBIOS1 UUID.",
2375         optional => 1,
2376     },
2377     version => {
2378         type => 'string',
2379         pattern => '[A-Za-z0-9+\/]+={0,2}',
2380         format_description => 'Base64 encoded string',
2381         description => "Set SMBIOS1 version.",
2382         optional => 1,
2383     },
2384     serial => {
2385         type => 'string',
2386         pattern => '[A-Za-z0-9+\/]+={0,2}',
2387         format_description => 'Base64 encoded string',
2388         description => "Set SMBIOS1 serial number.",
2389         optional => 1,
2390     },
2391     manufacturer => {
2392         type => 'string',
2393         pattern => '[A-Za-z0-9+\/]+={0,2}',
2394         format_description => 'Base64 encoded string',
2395         description => "Set SMBIOS1 manufacturer.",
2396         optional => 1,
2397     },
2398     product => {
2399         type => 'string',
2400         pattern => '[A-Za-z0-9+\/]+={0,2}',
2401         format_description => 'Base64 encoded string',
2402         description => "Set SMBIOS1 product ID.",
2403         optional => 1,
2404     },
2405     sku => {
2406         type => 'string',
2407         pattern => '[A-Za-z0-9+\/]+={0,2}',
2408         format_description => 'Base64 encoded string',
2409         description => "Set SMBIOS1 SKU string.",
2410         optional => 1,
2411     },
2412     family => {
2413         type => 'string',
2414         pattern => '[A-Za-z0-9+\/]+={0,2}',
2415         format_description => 'Base64 encoded string',
2416         description => "Set SMBIOS1 family string.",
2417         optional => 1,
2418     },
2419     base64 => {
2420         type => 'boolean',
2421         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2422         optional => 1,
2423     },
2424 };
2425
2426 sub parse_smbios1 {
2427     my ($data) = @_;
2428
2429     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2430     warn $@ if $@;
2431     return $res;
2432 }
2433
2434 sub print_smbios1 {
2435     my ($smbios1) = @_;
2436     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2437 }
2438
2439 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2440
2441 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2442 sub verify_bootdisk {
2443     my ($value, $noerr) = @_;
2444
2445     return $value if is_valid_drivename($value);
2446
2447     return undef if $noerr;
2448
2449     die "invalid boot disk '$value'\n";
2450 }
2451
2452 sub parse_watchdog {
2453     my ($value) = @_;
2454
2455     return undef if !$value;
2456
2457     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2458     warn $@ if $@;
2459     return $res;
2460 }
2461
2462 sub parse_guest_agent {
2463     my ($value) = @_;
2464
2465     return {} if !defined($value->{agent});
2466
2467     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2468     warn $@ if $@;
2469
2470     # if the agent is disabled ignore the other potentially set properties
2471     return {} if !$res->{enabled};
2472     return $res;
2473 }
2474
2475 sub parse_vga {
2476     my ($value) = @_;
2477
2478     return {} if !$value;
2479     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2480     warn $@ if $@;
2481     return $res;
2482 }
2483
2484 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2485 sub verify_usb_device {
2486     my ($value, $noerr) = @_;
2487
2488     return $value if parse_usb_device($value);
2489
2490     return undef if $noerr;
2491
2492     die "unable to parse usb device\n";
2493 }
2494
2495 # add JSON properties for create and set function
2496 sub json_config_properties {
2497     my $prop = shift;
2498
2499     foreach my $opt (keys %$confdesc) {
2500         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2501         $prop->{$opt} = $confdesc->{$opt};
2502     }
2503
2504     return $prop;
2505 }
2506
2507 # return copy of $confdesc_cloudinit to generate documentation
2508 sub cloudinit_config_properties {
2509
2510     return dclone($confdesc_cloudinit);
2511 }
2512
2513 sub check_type {
2514     my ($key, $value) = @_;
2515
2516     die "unknown setting '$key'\n" if !$confdesc->{$key};
2517
2518     my $type = $confdesc->{$key}->{type};
2519
2520     if (!defined($value)) {
2521         die "got undefined value\n";
2522     }
2523
2524     if ($value =~ m/[\n\r]/) {
2525         die "property contains a line feed\n";
2526     }
2527
2528     if ($type eq 'boolean') {
2529         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2530         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2531         die "type check ('boolean') failed - got '$value'\n";
2532     } elsif ($type eq 'integer') {
2533         return int($1) if $value =~ m/^(\d+)$/;
2534         die "type check ('integer') failed - got '$value'\n";
2535     } elsif ($type eq 'number') {
2536         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2537         die "type check ('number') failed - got '$value'\n";
2538     } elsif ($type eq 'string') {
2539         if (my $fmt = $confdesc->{$key}->{format}) {
2540             PVE::JSONSchema::check_format($fmt, $value);
2541             return $value;
2542         }
2543         $value =~ s/^\"(.*)\"$/$1/;
2544         return $value;
2545     } else {
2546         die "internal error"
2547     }
2548 }
2549
2550 sub touch_config {
2551     my ($vmid) = @_;
2552
2553     my $conf = PVE::QemuConfig->config_file($vmid);
2554     utime undef, undef, $conf;
2555 }
2556
2557 sub destroy_vm {
2558     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2559
2560     my $conffile = PVE::QemuConfig->config_file($vmid);
2561
2562     my $conf = PVE::QemuConfig->load_config($vmid);
2563
2564     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2565
2566     if ($conf->{template}) {
2567         # check if any base image is still used by a linked clone
2568         foreach_drive($conf, sub {
2569                 my ($ds, $drive) = @_;
2570
2571                 return if drive_is_cdrom($drive);
2572
2573                 my $volid = $drive->{file};
2574
2575                 return if !$volid || $volid =~ m|^/|;
2576
2577                 die "base volume '$volid' is still in use by linked cloned\n"
2578                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2579
2580         });
2581     }
2582
2583     # only remove disks owned by this VM
2584     foreach_drive($conf, sub {
2585         my ($ds, $drive) = @_;
2586
2587         return if drive_is_cdrom($drive, 1);
2588
2589         my $volid = $drive->{file};
2590
2591         return if !$volid || $volid =~ m|^/|;
2592
2593         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2594         return if !$path || !$owner || ($owner != $vmid);
2595
2596         eval {
2597             PVE::Storage::vdisk_free($storecfg, $volid);
2598         };
2599         warn "Could not remove disk '$volid', check manually: $@" if $@;
2600
2601     });
2602
2603     if ($keep_empty_config) {
2604         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2605     } else {
2606         unlink $conffile;
2607     }
2608
2609     # also remove unused disk
2610     eval {
2611         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2612
2613         eval {
2614             PVE::Storage::foreach_volid($dl, sub {
2615                 my ($volid, $sid, $volname, $d) = @_;
2616                 PVE::Storage::vdisk_free($storecfg, $volid);
2617             });
2618         };
2619         warn $@ if $@;
2620
2621     };
2622     warn $@ if $@;
2623 }
2624
2625 sub parse_vm_config {
2626     my ($filename, $raw) = @_;
2627
2628     return undef if !defined($raw);
2629
2630     my $res = {
2631         digest => Digest::SHA::sha1_hex($raw),
2632         snapshots => {},
2633         pending => {},
2634     };
2635
2636     $filename =~ m|/qemu-server/(\d+)\.conf$|
2637         || die "got strange filename '$filename'";
2638
2639     my $vmid = $1;
2640
2641     my $conf = $res;
2642     my $descr;
2643     my $section = '';
2644
2645     my @lines = split(/\n/, $raw);
2646     foreach my $line (@lines) {
2647         next if $line =~ m/^\s*$/;
2648
2649         if ($line =~ m/^\[PENDING\]\s*$/i) {
2650             $section = 'pending';
2651             if (defined($descr)) {
2652                 $descr =~ s/\s+$//;
2653                 $conf->{description} = $descr;
2654             }
2655             $descr = undef;
2656             $conf = $res->{$section} = {};
2657             next;
2658
2659         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2660             $section = $1;
2661             if (defined($descr)) {
2662                 $descr =~ s/\s+$//;
2663                 $conf->{description} = $descr;
2664             }
2665             $descr = undef;
2666             $conf = $res->{snapshots}->{$section} = {};
2667             next;
2668         }
2669
2670         if ($line =~ m/^\#(.*)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($1) . "\n";
2673             next;
2674         }
2675
2676         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2677             $descr = '' if !defined($descr);
2678             $descr .= PVE::Tools::decode_text($2);
2679         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2680             $conf->{snapstate} = $1;
2681         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2682             my $key = $1;
2683             my $value = $2;
2684             $conf->{$key} = $value;
2685         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2686             my $value = $1;
2687             if ($section eq 'pending') {
2688                 $conf->{delete} = $value; # we parse this later
2689             } else {
2690                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2691             }
2692         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2693             my $key = $1;
2694             my $value = $2;
2695             eval { $value = check_type($key, $value); };
2696             if ($@) {
2697                 warn "vm $vmid - unable to parse value of '$key' - $@";
2698             } else {
2699                 $key = 'ide2' if $key eq 'cdrom';
2700                 my $fmt = $confdesc->{$key}->{format};
2701                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2702                     my $v = parse_drive($key, $value);
2703                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2704                         $v->{file} = $volid;
2705                         $value = print_drive($vmid, $v);
2706                     } else {
2707                         warn "vm $vmid - unable to parse value of '$key'\n";
2708                         next;
2709                     }
2710                 }
2711
2712                 $conf->{$key} = $value;
2713             }
2714         }
2715     }
2716
2717     if (defined($descr)) {
2718         $descr =~ s/\s+$//;
2719         $conf->{description} = $descr;
2720     }
2721     delete $res->{snapstate}; # just to be sure
2722
2723     return $res;
2724 }
2725
2726 sub write_vm_config {
2727     my ($filename, $conf) = @_;
2728
2729     delete $conf->{snapstate}; # just to be sure
2730
2731     if ($conf->{cdrom}) {
2732         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2733         $conf->{ide2} = $conf->{cdrom};
2734         delete $conf->{cdrom};
2735     }
2736
2737     # we do not use 'smp' any longer
2738     if ($conf->{sockets}) {
2739         delete $conf->{smp};
2740     } elsif ($conf->{smp}) {
2741         $conf->{sockets} = $conf->{smp};
2742         delete $conf->{cores};
2743         delete $conf->{smp};
2744     }
2745
2746     my $used_volids = {};
2747
2748     my $cleanup_config = sub {
2749         my ($cref, $pending, $snapname) = @_;
2750
2751         foreach my $key (keys %$cref) {
2752             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2753                 $key eq 'snapstate' || $key eq 'pending';
2754             my $value = $cref->{$key};
2755             if ($key eq 'delete') {
2756                 die "propertry 'delete' is only allowed in [PENDING]\n"
2757                     if !$pending;
2758                 # fixme: check syntax?
2759                 next;
2760             }
2761             eval { $value = check_type($key, $value); };
2762             die "unable to parse value of '$key' - $@" if $@;
2763
2764             $cref->{$key} = $value;
2765
2766             if (!$snapname && is_valid_drivename($key)) {
2767                 my $drive = parse_drive($key, $value);
2768                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2769             }
2770         }
2771     };
2772
2773     &$cleanup_config($conf);
2774
2775     &$cleanup_config($conf->{pending}, 1);
2776
2777     foreach my $snapname (keys %{$conf->{snapshots}}) {
2778         die "internal error" if $snapname eq 'pending';
2779         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2780     }
2781
2782     # remove 'unusedX' settings if we re-add a volume
2783     foreach my $key (keys %$conf) {
2784         my $value = $conf->{$key};
2785         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2786             delete $conf->{$key};
2787         }
2788     }
2789
2790     my $generate_raw_config = sub {
2791         my ($conf, $pending) = @_;
2792
2793         my $raw = '';
2794
2795         # add description as comment to top of file
2796         if (defined(my $descr = $conf->{description})) {
2797             if ($descr) {
2798                 foreach my $cl (split(/\n/, $descr)) {
2799                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2800                 }
2801             } else {
2802                 $raw .= "#\n" if $pending;
2803             }
2804         }
2805
2806         foreach my $key (sort keys %$conf) {
2807             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2808             $raw .= "$key: $conf->{$key}\n";
2809         }
2810         return $raw;
2811     };
2812
2813     my $raw = &$generate_raw_config($conf);
2814
2815     if (scalar(keys %{$conf->{pending}})){
2816         $raw .= "\n[PENDING]\n";
2817         $raw .= &$generate_raw_config($conf->{pending}, 1);
2818     }
2819
2820     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2821         $raw .= "\n[$snapname]\n";
2822         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2823     }
2824
2825     return $raw;
2826 }
2827
2828 sub load_defaults {
2829
2830     my $res = {};
2831
2832     # we use static defaults from our JSON schema configuration
2833     foreach my $key (keys %$confdesc) {
2834         if (defined(my $default = $confdesc->{$key}->{default})) {
2835             $res->{$key} = $default;
2836         }
2837     }
2838
2839     return $res;
2840 }
2841
2842 sub config_list {
2843     my $vmlist = PVE::Cluster::get_vmlist();
2844     my $res = {};
2845     return $res if !$vmlist || !$vmlist->{ids};
2846     my $ids = $vmlist->{ids};
2847
2848     foreach my $vmid (keys %$ids) {
2849         my $d = $ids->{$vmid};
2850         next if !$d->{node} || $d->{node} ne $nodename;
2851         next if !$d->{type} || $d->{type} ne 'qemu';
2852         $res->{$vmid}->{exists} = 1;
2853     }
2854     return $res;
2855 }
2856
2857 # test if VM uses local resources (to prevent migration)
2858 sub check_local_resources {
2859     my ($conf, $noerr) = @_;
2860
2861     my @loc_res = ();
2862
2863     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2864     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2865
2866     push @loc_res, "ivshmem" if $conf->{ivshmem};
2867
2868     foreach my $k (keys %$conf) {
2869         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2870         # sockets are safe: they will recreated be on the target side post-migrate
2871         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2872         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2873     }
2874
2875     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2876
2877     return \@loc_res;
2878 }
2879
2880 # check if used storages are available on all nodes (use by migrate)
2881 sub check_storage_availability {
2882     my ($storecfg, $conf, $node) = @_;
2883
2884     foreach_drive($conf, sub {
2885         my ($ds, $drive) = @_;
2886
2887         my $volid = $drive->{file};
2888         return if !$volid;
2889
2890         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2891         return if !$sid;
2892
2893         # check if storage is available on both nodes
2894         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2895         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2896    });
2897 }
2898
2899 # list nodes where all VM images are available (used by has_feature API)
2900 sub shared_nodes {
2901     my ($conf, $storecfg) = @_;
2902
2903     my $nodelist = PVE::Cluster::get_nodelist();
2904     my $nodehash = { map { $_ => 1 } @$nodelist };
2905     my $nodename = PVE::INotify::nodename();
2906
2907     foreach_drive($conf, sub {
2908         my ($ds, $drive) = @_;
2909
2910         my $volid = $drive->{file};
2911         return if !$volid;
2912
2913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2914         if ($storeid) {
2915             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2916             if ($scfg->{disable}) {
2917                 $nodehash = {};
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     delete $nodehash->{$node} if !$avail->{$node};
2921                 }
2922             } elsif (!$scfg->{shared}) {
2923                 foreach my $node (keys %$nodehash) {
2924                     delete $nodehash->{$node} if $node ne $nodename
2925                 }
2926             }
2927         }
2928     });
2929
2930     return $nodehash
2931 }
2932
2933 sub check_local_storage_availability {
2934     my ($conf, $storecfg) = @_;
2935
2936     my $nodelist = PVE::Cluster::get_nodelist();
2937     my $nodehash = { map { $_ => {} } @$nodelist };
2938
2939     foreach_drive($conf, sub {
2940         my ($ds, $drive) = @_;
2941
2942         my $volid = $drive->{file};
2943         return if !$volid;
2944
2945         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2946         if ($storeid) {
2947             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2948
2949             if ($scfg->{disable}) {
2950                 foreach my $node (keys %$nodehash) {
2951                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2952                 }
2953             } elsif (my $avail = $scfg->{nodes}) {
2954                 foreach my $node (keys %$nodehash) {
2955                     if (!$avail->{$node}) {
2956                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2957                     }
2958                 }
2959             }
2960         }
2961     });
2962
2963     foreach my $node (values %$nodehash) {
2964         if (my $unavail = $node->{unavailable_storages}) {
2965             $node->{unavailable_storages} = [ sort keys %$unavail ];
2966         }
2967     }
2968
2969     return $nodehash
2970 }
2971
2972 sub check_cmdline {
2973     my ($pidfile, $pid) = @_;
2974
2975     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2976     if (defined($fh)) {
2977         my $line = <$fh>;
2978         $fh->close;
2979         return undef if !$line;
2980         my @param = split(/\0/, $line);
2981
2982         my $cmd = $param[0];
2983         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2984
2985         for (my $i = 0; $i < scalar (@param); $i++) {
2986             my $p = $param[$i];
2987             next if !$p;
2988             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2989                 my $p = $param[$i+1];
2990                 return 1 if $p && ($p eq $pidfile);
2991                 return undef;
2992             }
2993         }
2994     }
2995     return undef;
2996 }
2997
2998 sub check_running {
2999     my ($vmid, $nocheck, $node) = @_;
3000
3001     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3002
3003     die "unable to find configuration file for VM $vmid - no such machine\n"
3004         if !$nocheck && ! -f $filename;
3005
3006     my $pidfile = pidfile_name($vmid);
3007
3008     if (my $fd = IO::File->new("<$pidfile")) {
3009         my $st = stat($fd);
3010         my $line = <$fd>;
3011         close($fd);
3012
3013         my $mtime = $st->mtime;
3014         if ($mtime > time()) {
3015             warn "file '$filename' modified in future\n";
3016         }
3017
3018         if ($line =~ m/^(\d+)$/) {
3019             my $pid = $1;
3020             if (check_cmdline($pidfile, $pid)) {
3021                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3022                     return $pid;
3023                 }
3024             }
3025         }
3026     }
3027
3028     return undef;
3029 }
3030
3031 sub vzlist {
3032
3033     my $vzlist = config_list();
3034
3035     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3036
3037     while (defined(my $de = $fd->read)) {
3038         next if $de !~ m/^(\d+)\.pid$/;
3039         my $vmid = $1;
3040         next if !defined($vzlist->{$vmid});
3041         if (my $pid = check_running($vmid)) {
3042             $vzlist->{$vmid}->{pid} = $pid;
3043         }
3044     }
3045
3046     return $vzlist;
3047 }
3048
3049 sub disksize {
3050     my ($storecfg, $conf) = @_;
3051
3052     my $bootdisk = $conf->{bootdisk};
3053     return undef if !$bootdisk;
3054     return undef if !is_valid_drivename($bootdisk);
3055
3056     return undef if !$conf->{$bootdisk};
3057
3058     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3059     return undef if !defined($drive);
3060
3061     return undef if drive_is_cdrom($drive);
3062
3063     my $volid = $drive->{file};
3064     return undef if !$volid;
3065
3066     return $drive->{size};
3067 }
3068
3069 our $vmstatus_return_properties = {
3070     vmid => get_standard_option('pve-vmid'),
3071     status => {
3072         description => "Qemu process status.",
3073         type => 'string',
3074         enum => ['stopped', 'running'],
3075     },
3076     maxmem => {
3077         description => "Maximum memory in bytes.",
3078         type => 'integer',
3079         optional => 1,
3080         renderer => 'bytes',
3081     },
3082     maxdisk => {
3083         description => "Root disk size in bytes.",
3084         type => 'integer',
3085         optional => 1,
3086         renderer => 'bytes',
3087     },
3088     name => {
3089         description => "VM name.",
3090         type => 'string',
3091         optional => 1,
3092     },
3093     qmpstatus => {
3094         description => "Qemu QMP agent status.",
3095         type => 'string',
3096         optional => 1,
3097     },
3098     pid => {
3099         description => "PID of running qemu process.",
3100         type => 'integer',
3101         optional => 1,
3102     },
3103     uptime => {
3104         description => "Uptime.",
3105         type => 'integer',
3106         optional => 1,
3107         renderer => 'duration',
3108     },
3109     cpus => {
3110         description => "Maximum usable CPUs.",
3111         type => 'number',
3112         optional => 1,
3113     },
3114     lock => {
3115         description => "The current config lock, if any.",
3116         type => 'string',
3117         optional => 1,
3118     }
3119 };
3120
3121 my $last_proc_pid_stat;
3122
3123 # get VM status information
3124 # This must be fast and should not block ($full == false)
3125 # We only query KVM using QMP if $full == true (this can be slow)
3126 sub vmstatus {
3127     my ($opt_vmid, $full) = @_;
3128
3129     my $res = {};
3130
3131     my $storecfg = PVE::Storage::config();
3132
3133     my $list = vzlist();
3134     my $defaults = load_defaults();
3135
3136     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3137
3138     my $cpucount = $cpuinfo->{cpus} || 1;
3139
3140     foreach my $vmid (keys %$list) {
3141         next if $opt_vmid && ($vmid ne $opt_vmid);
3142
3143         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3144         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3145
3146         my $d = { vmid => $vmid };
3147         $d->{pid} = $list->{$vmid}->{pid};
3148
3149         # fixme: better status?
3150         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3151
3152         my $size = disksize($storecfg, $conf);
3153         if (defined($size)) {
3154             $d->{disk} = 0; # no info available
3155             $d->{maxdisk} = $size;
3156         } else {
3157             $d->{disk} = 0;
3158             $d->{maxdisk} = 0;
3159         }
3160
3161         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3162             * ($conf->{cores} || $defaults->{cores});
3163         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3164         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3165
3166         $d->{name} = $conf->{name} || "VM $vmid";
3167         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3168             : $defaults->{memory}*(1024*1024);
3169
3170         if ($conf->{balloon}) {
3171             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3172             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3173                 : $defaults->{shares};
3174         }
3175
3176         $d->{uptime} = 0;
3177         $d->{cpu} = 0;
3178         $d->{mem} = 0;
3179
3180         $d->{netout} = 0;
3181         $d->{netin} = 0;
3182
3183         $d->{diskread} = 0;
3184         $d->{diskwrite} = 0;
3185
3186         $d->{template} = PVE::QemuConfig->is_template($conf);
3187
3188         $d->{serial} = 1 if conf_has_serial($conf);
3189         $d->{lock} = $conf->{lock} if $conf->{lock};
3190
3191         $res->{$vmid} = $d;
3192     }
3193
3194     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3195     foreach my $dev (keys %$netdev) {
3196         next if $dev !~ m/^tap([1-9]\d*)i/;
3197         my $vmid = $1;
3198         my $d = $res->{$vmid};
3199         next if !$d;
3200
3201         $d->{netout} += $netdev->{$dev}->{receive};
3202         $d->{netin} += $netdev->{$dev}->{transmit};
3203
3204         if ($full) {
3205             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3206             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3207         }
3208
3209     }
3210
3211     my $ctime = gettimeofday;
3212
3213     foreach my $vmid (keys %$list) {
3214
3215         my $d = $res->{$vmid};
3216         my $pid = $d->{pid};
3217         next if !$pid;
3218
3219         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3220         next if !$pstat; # not running
3221
3222         my $used = $pstat->{utime} + $pstat->{stime};
3223
3224         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3225
3226         if ($pstat->{vsize}) {
3227             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3228         }
3229
3230         my $old = $last_proc_pid_stat->{$pid};
3231         if (!$old) {
3232             $last_proc_pid_stat->{$pid} = {
3233                 time => $ctime,
3234                 used => $used,
3235                 cpu => 0,
3236             };
3237             next;
3238         }
3239
3240         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3241
3242         if ($dtime > 1000) {
3243             my $dutime = $used -  $old->{used};
3244
3245             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3246             $last_proc_pid_stat->{$pid} = {
3247                 time => $ctime,
3248                 used => $used,
3249                 cpu => $d->{cpu},
3250             };
3251         } else {
3252             $d->{cpu} = $old->{cpu};
3253         }
3254     }
3255
3256     return $res if !$full;
3257
3258     my $qmpclient = PVE::QMPClient->new();
3259
3260     my $ballooncb = sub {
3261         my ($vmid, $resp) = @_;
3262
3263         my $info = $resp->{'return'};
3264         return if !$info->{max_mem};
3265
3266         my $d = $res->{$vmid};
3267
3268         # use memory assigned to VM
3269         $d->{maxmem} = $info->{max_mem};
3270         $d->{balloon} = $info->{actual};
3271
3272         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3273             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3274             $d->{freemem} = $info->{free_mem};
3275         }
3276
3277         $d->{ballooninfo} = $info;
3278     };
3279
3280     my $blockstatscb = sub {
3281         my ($vmid, $resp) = @_;
3282         my $data = $resp->{'return'} || [];
3283         my $totalrdbytes = 0;
3284         my $totalwrbytes = 0;
3285
3286         for my $blockstat (@$data) {
3287             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3288             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3289
3290             $blockstat->{device} =~ s/drive-//;
3291             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3292         }
3293         $res->{$vmid}->{diskread} = $totalrdbytes;
3294         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3295     };
3296
3297     my $statuscb = sub {
3298         my ($vmid, $resp) = @_;
3299
3300         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3301         # this fails if ballon driver is not loaded, so this must be
3302         # the last commnand (following command are aborted if this fails).
3303         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3304
3305         my $status = 'unknown';
3306         if (!defined($status = $resp->{'return'}->{status})) {
3307             warn "unable to get VM status\n";
3308             return;
3309         }
3310
3311         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3312     };
3313
3314     foreach my $vmid (keys %$list) {
3315         next if $opt_vmid && ($vmid ne $opt_vmid);
3316         next if !$res->{$vmid}->{pid}; # not running
3317         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3318     }
3319
3320     $qmpclient->queue_execute(undef, 2);
3321
3322     foreach my $vmid (keys %$list) {
3323         next if $opt_vmid && ($vmid ne $opt_vmid);
3324         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3325     }
3326
3327     return $res;
3328 }
3329
3330 sub foreach_drive {
3331     my ($conf, $func, @param) = @_;
3332
3333     foreach my $ds (valid_drive_names()) {
3334         next if !defined($conf->{$ds});
3335
3336         my $drive = parse_drive($ds, $conf->{$ds});
3337         next if !$drive;
3338
3339         &$func($ds, $drive, @param);
3340     }
3341 }
3342
3343 sub foreach_volid {
3344     my ($conf, $func, @param) = @_;
3345
3346     my $volhash = {};
3347
3348     my $test_volid = sub {
3349         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3350
3351         return if !$volid;
3352
3353         $volhash->{$volid}->{cdrom} //= 1;
3354         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3355
3356         $volhash->{$volid}->{replicate} //= 0;
3357         $volhash->{$volid}->{replicate} = 1 if $replicate;
3358
3359         $volhash->{$volid}->{shared} //= 0;
3360         $volhash->{$volid}->{shared} = 1 if $shared;
3361
3362         $volhash->{$volid}->{referenced_in_config} //= 0;
3363         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3364
3365         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3366             if defined($snapname);
3367         $volhash->{$volid}->{size} = $size if $size;
3368     };
3369
3370     foreach_drive($conf, sub {
3371         my ($ds, $drive) = @_;
3372         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3373     });
3374
3375     foreach my $snapname (keys %{$conf->{snapshots}}) {
3376         my $snap = $conf->{snapshots}->{$snapname};
3377         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3378         foreach_drive($snap, sub {
3379             my ($ds, $drive) = @_;
3380             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3381         });
3382     }
3383
3384     foreach my $volid (keys %$volhash) {
3385         &$func($volid, $volhash->{$volid}, @param);
3386     }
3387 }
3388
3389 sub conf_has_serial {
3390     my ($conf) = @_;
3391
3392     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3393         if ($conf->{"serial$i"}) {
3394             return 1;
3395         }
3396     }
3397
3398     return 0;
3399 }
3400
3401 sub vga_conf_has_spice {
3402     my ($vga) = @_;
3403
3404     my $vgaconf = parse_vga($vga);
3405     my $vgatype = $vgaconf->{type};
3406     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3407
3408     return $1 || 1;
3409 }
3410
3411 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3412 sub get_host_arch() {
3413     $host_arch = (POSIX::uname())[4] if !$host_arch;
3414     return $host_arch;
3415 }
3416
3417 sub is_native($) {
3418     my ($arch) = @_;
3419     return get_host_arch() eq $arch;
3420 }
3421
3422 my $default_machines = {
3423     x86_64 => 'pc',
3424     aarch64 => 'virt',
3425 };
3426
3427 sub get_basic_machine_info {
3428     my ($conf, $forcemachine) = @_;
3429
3430     my $arch = $conf->{arch} // get_host_arch();
3431     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3432     return ($arch, $machine);
3433 }
3434
3435 sub get_ovmf_files($) {
3436     my ($arch) = @_;
3437
3438     my $ovmf = $OVMF->{$arch}
3439         or die "no OVMF images known for architecture '$arch'\n";
3440
3441     return @$ovmf;
3442 }
3443
3444 my $Arch2Qemu = {
3445     aarch64 => '/usr/bin/qemu-system-aarch64',
3446     x86_64 => '/usr/bin/qemu-system-x86_64',
3447 };
3448 sub get_command_for_arch($) {
3449     my ($arch) = @_;
3450     return '/usr/bin/kvm' if is_native($arch);
3451
3452     my $cmd = $Arch2Qemu->{$arch}
3453         or die "don't know how to emulate architecture '$arch'\n";
3454     return $cmd;
3455 }
3456
3457 sub get_cpu_options {
3458     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3459
3460     my $cpuFlags = [];
3461     my $ostype = $conf->{ostype};
3462
3463     my $cpu = $kvm ? "kvm64" : "qemu64";
3464     if ($arch eq 'aarch64') {
3465         $cpu = 'cortex-a57';
3466     }
3467     my $hv_vendor_id;
3468     if (my $cputype = $conf->{cpu}) {
3469         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3470             or die "Cannot parse cpu description: $cputype\n";
3471         $cpu = $cpuconf->{cputype};
3472         $kvm_off = 1 if $cpuconf->{hidden};
3473         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3474
3475         if (defined(my $flags = $cpuconf->{flags})) {
3476             push @$cpuFlags, split(";", $flags);
3477         }
3478     }
3479
3480     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3481
3482     push @$cpuFlags , '-x2apic'
3483         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3484
3485     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3486
3487     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3488
3489     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3490
3491         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3492         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3493     }
3494
3495     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3496
3497     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3498
3499     push @$cpuFlags, 'kvm=off' if $kvm_off;
3500
3501     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3502         push @$cpuFlags, "vendor=${cpu_vendor}"
3503             if $cpu_vendor ne 'default';
3504     } elsif ($arch ne 'aarch64') {
3505         die "internal error"; # should not happen
3506     }
3507
3508     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3509
3510     return ('-cpu', $cpu);
3511 }
3512
3513 sub config_to_command {
3514     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3515
3516     my $cmd = [];
3517     my $globalFlags = [];
3518     my $machineFlags = [];
3519     my $rtcFlags = [];
3520     my $devices = [];
3521     my $pciaddr = '';
3522     my $bridges = {};
3523     my $kvmver = kvm_user_version();
3524     my $vernum = 0; # unknown
3525     my $ostype = $conf->{ostype};
3526     my $winversion = windows_version($ostype);
3527     my $kvm = $conf->{kvm};
3528
3529     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3530     $kvm //= 1 if is_native($arch);
3531
3532     if ($kvm) {
3533         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3534             if !defined kvm_version();
3535     }
3536
3537     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3538         $vernum = $1*1000000+$2*1000;
3539     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3540         $vernum = $1*1000000+$2*1000+$3;
3541     }
3542
3543     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3544
3545     my $have_ovz = -f '/proc/vz/vestat';
3546
3547     my $q35 = machine_type_is_q35($conf);
3548     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3549     my $use_old_bios_files = undef;
3550     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3551
3552     my $cpuunits = defined($conf->{cpuunits}) ?
3553             $conf->{cpuunits} : $defaults->{cpuunits};
3554
3555     push @$cmd, get_command_for_arch($arch);
3556
3557     push @$cmd, '-id', $vmid;
3558
3559     my $vmname = $conf->{name} || "vm$vmid";
3560
3561     push @$cmd, '-name', $vmname;
3562
3563     my $use_virtio = 0;
3564
3565     my $qmpsocket = qmp_socket($vmid);
3566     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3567     push @$cmd, '-mon', "chardev=qmp,mode=control";
3568
3569     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3570         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3571         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3572     }
3573
3574     push @$cmd, '-pidfile' , pidfile_name($vmid);
3575
3576     push @$cmd, '-daemonize';
3577
3578     if ($conf->{smbios1}) {
3579         my $smbios_conf = parse_smbios1($conf->{smbios1});
3580         if ($smbios_conf->{base64}) {
3581             # Do not pass base64 flag to qemu
3582             delete $smbios_conf->{base64};
3583             my $smbios_string = "";
3584             foreach my $key (keys %$smbios_conf) {
3585                 my $value;
3586                 if ($key eq "uuid") {
3587                     $value = $smbios_conf->{uuid}
3588                 } else {
3589                     $value = decode_base64($smbios_conf->{$key});
3590                 }
3591                 # qemu accepts any binary data, only commas need escaping by double comma
3592                 $value =~ s/,/,,/g;
3593                 $smbios_string .= "," . $key . "=" . $value if $value;
3594             }
3595             push @$cmd, '-smbios', "type=1" . $smbios_string;
3596         } else {
3597             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3598         }
3599     }
3600
3601     if ($conf->{vmgenid}) {
3602         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3603     }
3604
3605     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3606     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3607         die "uefi base image not found\n" if ! -f $ovmf_code;
3608
3609         my $path;
3610         my $format;
3611         if (my $efidisk = $conf->{efidisk0}) {
3612             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3613             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3614             $format = $d->{format};
3615             if ($storeid) {
3616                 $path = PVE::Storage::path($storecfg, $d->{file});
3617                 if (!defined($format)) {
3618                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3619                     $format = qemu_img_format($scfg, $volname);
3620                 }
3621             } else {
3622                 $path = $d->{file};
3623                 die "efidisk format must be specified\n"
3624                     if !defined($format);
3625             }
3626         } else {
3627             warn "no efidisk configured! Using temporary efivars disk.\n";
3628             $path = "/tmp/$vmid-ovmf.fd";
3629             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3630             $format = 'raw';
3631         }
3632
3633         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3634         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3635     }
3636
3637     # load q35 config
3638     if ($q35) {
3639         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3640         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3641             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3642         } else {
3643             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3644         }
3645     }
3646
3647     # add usb controllers
3648     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3649     push @$devices, @usbcontrollers if @usbcontrollers;
3650     my $vga = parse_vga($conf->{vga});
3651
3652     my $qxlnum = vga_conf_has_spice($conf->{vga});
3653     $vga->{type} = 'qxl' if $qxlnum;
3654
3655     if (!$vga->{type}) {
3656         if ($arch eq 'aarch64') {
3657             $vga->{type} = 'virtio';
3658         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3659             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3660         } else {
3661             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3662         }
3663     }
3664
3665     # enable absolute mouse coordinates (needed by vnc)
3666     my $tablet;
3667     if (defined($conf->{tablet})) {
3668         $tablet = $conf->{tablet};
3669     } else {
3670         $tablet = $defaults->{tablet};
3671         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3672         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3673     }
3674
3675     if ($tablet) {
3676         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3677         my $kbd = print_keyboarddevice_full($conf, $arch);
3678         push @$devices, '-device', $kbd if defined($kbd);
3679     }
3680
3681     my $kvm_off = 0;
3682     my $gpu_passthrough;
3683
3684     # host pci devices
3685     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3686         my $d = parse_hostpci($conf->{"hostpci$i"});
3687         next if !$d;
3688
3689         my $pcie = $d->{pcie};
3690         if ($pcie) {
3691             die "q35 machine model is not enabled" if !$q35;
3692             # win7 wants to have the pcie devices directly on the pcie bus
3693             # instead of in the root port
3694             if ($winversion == 7) {
3695                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3696             } else {
3697                 $pciaddr = print_pcie_addr("hostpci$i");
3698             }
3699         } else {
3700             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3701         }
3702
3703         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3704         my $romfile = $d->{romfile};
3705
3706         my $xvga = '';
3707         if ($d->{'x-vga'}) {
3708             $xvga = ',x-vga=on';
3709             $kvm_off = 1;
3710             $vga->{type} = 'none' if !defined($conf->{vga});
3711             $gpu_passthrough = 1;
3712
3713             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3714                 $xvga = "";
3715             }
3716         }
3717         my $pcidevices = $d->{pciid};
3718         my $multifunction = 1 if @$pcidevices > 1;
3719         my $sysfspath;
3720         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3721             my $id = $pcidevices->[0]->{id};
3722             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3723             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3724         } elsif ($d->{mdev}) {
3725             warn "ignoring mediated device with multifunction device\n";
3726         }
3727
3728         my $j=0;
3729         foreach my $pcidevice (@$pcidevices) {
3730
3731             my $id = "hostpci$i";
3732             $id .= ".$j" if $multifunction;
3733             my $addr = $pciaddr;
3734             $addr .= ".$j" if $multifunction;
3735             my $devicestr = "vfio-pci";
3736             if ($sysfspath) {
3737                 $devicestr .= ",sysfsdev=$sysfspath";
3738             } else {
3739                 $devicestr .= ",host=$pcidevice->{id}";
3740             }
3741             $devicestr .= ",id=$id$addr";
3742
3743             if($j == 0){
3744                 $devicestr .= "$rombar$xvga";
3745                 $devicestr .= ",multifunction=on" if $multifunction;
3746                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3747             }
3748
3749             push @$devices, '-device', $devicestr;
3750             $j++;
3751         }
3752     }
3753
3754     # usb devices
3755     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3756     push @$devices, @usbdevices if @usbdevices;
3757     # serial devices
3758     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3759         if (my $path = $conf->{"serial$i"}) {
3760             if ($path eq 'socket') {
3761                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3762                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3763                 # On aarch64, serial0 is the UART device. Qemu only allows
3764                 # connecting UART devices via the '-serial' command line, as
3765                 # the device has a fixed slot on the hardware...
3766                 if ($arch eq 'aarch64' && $i == 0) {
3767                     push @$devices, '-serial', "chardev:serial$i";
3768                 } else {
3769                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3770                 }
3771             } else {
3772                 die "no such serial device\n" if ! -c $path;
3773                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3775             }
3776         }
3777     }
3778
3779     # parallel devices
3780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3781         if (my $path = $conf->{"parallel$i"}) {
3782             die "no such parallel device\n" if ! -c $path;
3783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3786         }
3787     }
3788
3789     if (my $audiodevice = $conf->{audio0}) {
3790         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3791
3792         if ($audiodevice eq 'AC97') {
3793             push @$devices, '-device', "AC97,id=sound0${audiopciaddr}";
3794         } elsif ($audiodevice =~ /intel\-hda$/) {
3795             push @$devices, '-device', "${audiodevice},id=sound5${audiopciaddr}";
3796             push @$devices, '-device', "hda-micro,id=sound5-codec0,bus=sound5.0,cad=0";
3797             push @$devices, '-device', "hda-duplex,id=sound5-codec1,bus=sound5.0,cad=1";
3798         } else {
3799             die "unkown audio device '$audiodevice', implement me!";
3800         }
3801     }
3802
3803     my $sockets = 1;
3804     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3805     $sockets = $conf->{sockets} if  $conf->{sockets};
3806
3807     my $cores = $conf->{cores} || 1;
3808
3809     my $maxcpus = $sockets * $cores;
3810
3811     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3812
3813     my $allowed_vcpus = $cpuinfo->{cpus};
3814
3815     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3816         if ($allowed_vcpus < $maxcpus);
3817
3818     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3819
3820         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3821         for (my $i = 2; $i <= $vcpus; $i++)  {
3822             my $cpustr = print_cpu_device($conf,$i);
3823             push @$cmd, '-device', $cpustr;
3824         }
3825
3826     } else {
3827
3828         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3829     }
3830     push @$cmd, '-nodefaults';
3831
3832     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3833
3834     my $bootindex_hash = {};
3835     my $i = 1;
3836     foreach my $o (split(//, $bootorder)) {
3837         $bootindex_hash->{$o} = $i*100;
3838         $i++;
3839     }
3840
3841     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3842
3843     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3844
3845     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3846
3847     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3848         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3849         my $socket = vnc_socket($vmid);
3850         push @$cmd,  '-vnc', "unix:$socket,password";
3851     } else {
3852         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3853         push @$cmd, '-nographic';
3854     }
3855
3856     # time drift fix
3857     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3858
3859     my $useLocaltime = $conf->{localtime};
3860
3861     if ($winversion >= 5) { # windows
3862         $useLocaltime = 1 if !defined($conf->{localtime});
3863
3864         # use time drift fix when acpi is enabled
3865         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3866             $tdf = 1 if !defined($conf->{tdf});
3867         }
3868     }
3869
3870     if ($winversion >= 6) {
3871         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3872         push @$cmd, '-no-hpet';
3873     }
3874
3875     push @$rtcFlags, 'driftfix=slew' if $tdf;
3876
3877     if (!$kvm) {
3878         push @$machineFlags, 'accel=tcg';
3879     }
3880
3881     if ($machine_type) {
3882         push @$machineFlags, "type=${machine_type}";
3883     }
3884
3885     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3886         push @$rtcFlags, "base=$conf->{startdate}";
3887     } elsif ($useLocaltime) {
3888         push @$rtcFlags, 'base=localtime';
3889     }
3890
3891     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3892
3893     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3894
3895     push @$cmd, '-S' if $conf->{freeze};
3896
3897     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3898
3899     if (parse_guest_agent($conf)->{enabled}) {
3900         my $qgasocket = qmp_socket($vmid, 1);
3901         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3902         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3903         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3904         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3905     }
3906
3907     my $spice_port;
3908
3909     if ($qxlnum) {
3910         if ($qxlnum > 1) {
3911             if ($winversion){
3912                 for(my $i = 1; $i < $qxlnum; $i++){
3913                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3914                 }
3915             } else {
3916                 # assume other OS works like Linux
3917                 my ($ram, $vram) = ("134217728", "67108864");
3918                 if ($vga->{memory}) {
3919                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3920                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3921                 }
3922                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3923                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3924             }
3925         }
3926
3927         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3928
3929         my $nodename = PVE::INotify::nodename();
3930         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3931         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3932         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3933         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3934         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3935
3936         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3937
3938         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3939         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3940         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3941     }
3942
3943     # enable balloon by default, unless explicitly disabled
3944     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3945         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3946         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3947     }
3948
3949     if ($conf->{watchdog}) {
3950         my $wdopts = parse_watchdog($conf->{watchdog});
3951         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3952         my $watchdog = $wdopts->{model} || 'i6300esb';
3953         push @$devices, '-device', "$watchdog$pciaddr";
3954         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3955     }
3956
3957     my $vollist = [];
3958     my $scsicontroller = {};
3959     my $ahcicontroller = {};
3960     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3961
3962     # Add iscsi initiator name if available
3963     if (my $initiator = get_initiator_name()) {
3964         push @$devices, '-iscsi', "initiator-name=$initiator";
3965     }
3966
3967     foreach_drive($conf, sub {
3968         my ($ds, $drive) = @_;
3969
3970         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3971             push @$vollist, $drive->{file};
3972         }
3973
3974         # ignore efidisk here, already added in bios/fw handling code above
3975         return if $drive->{interface} eq 'efidisk';
3976
3977         $use_virtio = 1 if $ds =~ m/^virtio/;
3978
3979         if (drive_is_cdrom ($drive)) {
3980             if ($bootindex_hash->{d}) {
3981                 $drive->{bootindex} = $bootindex_hash->{d};
3982                 $bootindex_hash->{d} += 1;
3983             }
3984         } else {
3985             if ($bootindex_hash->{c}) {
3986                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3987                 $bootindex_hash->{c} += 1;
3988             }
3989         }
3990
3991         if($drive->{interface} eq 'virtio'){
3992            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3993         }
3994
3995         if ($drive->{interface} eq 'scsi') {
3996
3997             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3998
3999             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4000             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4001
4002             my $iothread = '';
4003             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4004                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4005                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4006             } elsif ($drive->{iothread}) {
4007                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4008             }
4009
4010             my $queues = '';
4011             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4012                 $queues = ",num_queues=$drive->{queues}";
4013             }
4014
4015             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4016             $scsicontroller->{$controller}=1;
4017         }
4018
4019         if ($drive->{interface} eq 'sata') {
4020            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4021            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4022            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4023            $ahcicontroller->{$controller}=1;
4024         }
4025
4026         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4027         push @$devices, '-drive',$drive_cmd;
4028         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4029     });
4030
4031     for (my $i = 0; $i < $MAX_NETS; $i++) {
4032          next if !$conf->{"net$i"};
4033          my $d = parse_net($conf->{"net$i"});
4034          next if !$d;
4035
4036          $use_virtio = 1 if $d->{model} eq 'virtio';
4037
4038          if ($bootindex_hash->{n}) {
4039             $d->{bootindex} = $bootindex_hash->{n};
4040             $bootindex_hash->{n} += 1;
4041          }
4042
4043          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4044          push @$devices, '-netdev', $netdevfull;
4045
4046          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4047          push @$devices, '-device', $netdevicefull;
4048     }
4049
4050     if ($conf->{ivshmem}) {
4051         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4052
4053         my $bus;
4054         if ($q35) {
4055             $bus = print_pcie_addr("ivshmem");
4056         } else {
4057             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4058         }
4059
4060         my $ivshmem_name = $ivshmem->{name} // $vmid;
4061         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4062
4063         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4064         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4065     }
4066
4067     if (!$q35) {
4068         # add pci bridges
4069         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4070            $bridges->{1} = 1;
4071            $bridges->{2} = 1;
4072         }
4073
4074         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4075
4076         while (my ($k, $v) = each %$bridges) {
4077             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4078             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4079         }
4080     }
4081
4082     push @$cmd, @$devices;
4083     push @$cmd, '-rtc', join(',', @$rtcFlags)
4084         if scalar(@$rtcFlags);
4085     push @$cmd, '-machine', join(',', @$machineFlags)
4086         if scalar(@$machineFlags);
4087     push @$cmd, '-global', join(',', @$globalFlags)
4088         if scalar(@$globalFlags);
4089
4090     if (my $vmstate = $conf->{vmstate}) {
4091         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4092         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4093         push @$cmd, '-loadstate', $statepath;
4094     }
4095
4096     # add custom args
4097     if ($conf->{args}) {
4098         my $aa = PVE::Tools::split_args($conf->{args});
4099         push @$cmd, @$aa;
4100     }
4101
4102     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4103 }
4104
4105 sub vnc_socket {
4106     my ($vmid) = @_;
4107     return "${var_run_tmpdir}/$vmid.vnc";
4108 }
4109
4110 sub spice_port {
4111     my ($vmid) = @_;
4112
4113     my $res = vm_mon_cmd($vmid, 'query-spice');
4114
4115     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4116 }
4117
4118 sub qmp_socket {
4119     my ($vmid, $qga, $name) = @_;
4120     my $sockettype = $qga ? 'qga' : 'qmp';
4121     my $ext = $name ? '-'.$name : '';
4122     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4123 }
4124
4125 sub pidfile_name {
4126     my ($vmid) = @_;
4127     return "${var_run_tmpdir}/$vmid.pid";
4128 }
4129
4130 sub vm_devices_list {
4131     my ($vmid) = @_;
4132
4133     my $res = vm_mon_cmd($vmid, 'query-pci');
4134     my $devices_to_check = [];
4135     my $devices = {};
4136     foreach my $pcibus (@$res) {
4137         push @$devices_to_check, @{$pcibus->{devices}},
4138     }
4139
4140     while (@$devices_to_check) {
4141         my $to_check = [];
4142         for my $d (@$devices_to_check) {
4143             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4144             next if !$d->{'pci_bridge'};
4145
4146             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4147             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4148         }
4149         $devices_to_check = $to_check;
4150     }
4151
4152     my $resblock = vm_mon_cmd($vmid, 'query-block');
4153     foreach my $block (@$resblock) {
4154         if($block->{device} =~ m/^drive-(\S+)/){
4155                 $devices->{$1} = 1;
4156         }
4157     }
4158
4159     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4160     foreach my $mice (@$resmice) {
4161         if ($mice->{name} eq 'QEMU HID Tablet') {
4162             $devices->{tablet} = 1;
4163             last;
4164         }
4165     }
4166
4167     # for usb devices there is no query-usb
4168     # but we can iterate over the entries in
4169     # qom-list path=/machine/peripheral
4170     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4171     foreach my $per (@$resperipheral) {
4172         if ($per->{name} =~ m/^usb\d+$/) {
4173             $devices->{$per->{name}} = 1;
4174         }
4175     }
4176
4177     return $devices;
4178 }
4179
4180 sub vm_deviceplug {
4181     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4182
4183     my $q35 = machine_type_is_q35($conf);
4184
4185     my $devices_list = vm_devices_list($vmid);
4186     return 1 if defined($devices_list->{$deviceid});
4187
4188     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4189
4190     if ($deviceid eq 'tablet') {
4191
4192         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4193
4194     } elsif ($deviceid eq 'keyboard') {
4195
4196         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4197
4198     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4199
4200         die "usb hotplug currently not reliable\n";
4201         # since we can't reliably hot unplug all added usb devices
4202         # and usb passthrough disables live migration
4203         # we disable usb hotplugging for now
4204         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4205
4206     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4207
4208         qemu_iothread_add($vmid, $deviceid, $device);
4209
4210         qemu_driveadd($storecfg, $vmid, $device);
4211         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4212
4213         qemu_deviceadd($vmid, $devicefull);
4214         eval { qemu_deviceaddverify($vmid, $deviceid); };
4215         if (my $err = $@) {
4216             eval { qemu_drivedel($vmid, $deviceid); };
4217             warn $@ if $@;
4218             die $err;
4219         }
4220
4221     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4222
4223
4224         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4225         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4226         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4227
4228         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4229
4230         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4231             qemu_iothread_add($vmid, $deviceid, $device);
4232             $devicefull .= ",iothread=iothread-$deviceid";
4233         }
4234
4235         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4236             $devicefull .= ",num_queues=$device->{queues}";
4237         }
4238
4239         qemu_deviceadd($vmid, $devicefull);
4240         qemu_deviceaddverify($vmid, $deviceid);
4241
4242     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4243
4244         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4245         qemu_driveadd($storecfg, $vmid, $device);
4246
4247         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4248         eval { qemu_deviceadd($vmid, $devicefull); };
4249         if (my $err = $@) {
4250             eval { qemu_drivedel($vmid, $deviceid); };
4251             warn $@ if $@;
4252             die $err;
4253         }
4254
4255     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4256
4257         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4258
4259         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4260         my $use_old_bios_files = undef;
4261         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4262
4263         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4264         qemu_deviceadd($vmid, $netdevicefull);
4265         eval {
4266             qemu_deviceaddverify($vmid, $deviceid);
4267             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4268         };
4269         if (my $err = $@) {
4270             eval { qemu_netdevdel($vmid, $deviceid); };
4271             warn $@ if $@;
4272             die $err;
4273         }
4274
4275     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4276
4277         my $bridgeid = $2;
4278         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4279         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4280
4281         qemu_deviceadd($vmid, $devicefull);
4282         qemu_deviceaddverify($vmid, $deviceid);
4283
4284     } else {
4285         die "can't hotplug device '$deviceid'\n";
4286     }
4287
4288     return 1;
4289 }
4290
4291 # fixme: this should raise exceptions on error!
4292 sub vm_deviceunplug {
4293     my ($vmid, $conf, $deviceid) = @_;
4294
4295     my $devices_list = vm_devices_list($vmid);
4296     return 1 if !defined($devices_list->{$deviceid});
4297
4298     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4299
4300     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4301
4302         qemu_devicedel($vmid, $deviceid);
4303
4304     } elsif ($deviceid =~ m/^usb\d+$/) {
4305
4306         die "usb hotplug currently not reliable\n";
4307         # when unplugging usb devices this way,
4308         # there may be remaining usb controllers/hubs
4309         # so we disable it for now
4310         qemu_devicedel($vmid, $deviceid);
4311         qemu_devicedelverify($vmid, $deviceid);
4312
4313     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4314
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_drivedel($vmid, $deviceid);
4318         qemu_iothread_del($conf, $vmid, $deviceid);
4319
4320     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4321
4322         qemu_devicedel($vmid, $deviceid);
4323         qemu_devicedelverify($vmid, $deviceid);
4324         qemu_iothread_del($conf, $vmid, $deviceid);
4325
4326     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4327
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_drivedel($vmid, $deviceid);
4330         qemu_deletescsihw($conf, $vmid, $deviceid);
4331
4332     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4333
4334         qemu_devicedel($vmid, $deviceid);
4335         qemu_devicedelverify($vmid, $deviceid);
4336         qemu_netdevdel($vmid, $deviceid);
4337
4338     } else {
4339         die "can't unplug device '$deviceid'\n";
4340     }
4341
4342     return 1;
4343 }
4344
4345 sub qemu_deviceadd {
4346     my ($vmid, $devicefull) = @_;
4347
4348     $devicefull = "driver=".$devicefull;
4349     my %options =  split(/[=,]/, $devicefull);
4350
4351     vm_mon_cmd($vmid, "device_add" , %options);
4352 }
4353
4354 sub qemu_devicedel {
4355     my ($vmid, $deviceid) = @_;
4356
4357     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4358 }
4359
4360 sub qemu_iothread_add {
4361     my($vmid, $deviceid, $device) = @_;
4362
4363     if ($device->{iothread}) {
4364         my $iothreads = vm_iothreads_list($vmid);
4365         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4366     }
4367 }
4368
4369 sub qemu_iothread_del {
4370     my($conf, $vmid, $deviceid) = @_;
4371
4372     my $confid = $deviceid;
4373     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4374         $confid = 'scsi' . $1;
4375     }
4376     my $device = parse_drive($confid, $conf->{$confid});
4377     if ($device->{iothread}) {
4378         my $iothreads = vm_iothreads_list($vmid);
4379         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4380     }
4381 }
4382
4383 sub qemu_objectadd {
4384     my($vmid, $objectid, $qomtype) = @_;
4385
4386     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4387
4388     return 1;
4389 }
4390
4391 sub qemu_objectdel {
4392     my($vmid, $objectid) = @_;
4393
4394     vm_mon_cmd($vmid, "object-del", id => $objectid);
4395
4396     return 1;
4397 }
4398
4399 sub qemu_driveadd {
4400     my ($storecfg, $vmid, $device) = @_;
4401
4402     my $drive = print_drive_full($storecfg, $vmid, $device);
4403     $drive =~ s/\\/\\\\/g;
4404     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4405
4406     # If the command succeeds qemu prints: "OK"
4407     return 1 if $ret =~ m/OK/s;
4408
4409     die "adding drive failed: $ret\n";
4410 }
4411
4412 sub qemu_drivedel {
4413     my($vmid, $deviceid) = @_;
4414
4415     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4416     $ret =~ s/^\s+//;
4417
4418     return 1 if $ret eq "";
4419
4420     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4421     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4422
4423     die "deleting drive $deviceid failed : $ret\n";
4424 }
4425
4426 sub qemu_deviceaddverify {
4427     my ($vmid, $deviceid) = @_;
4428
4429     for (my $i = 0; $i <= 5; $i++) {
4430          my $devices_list = vm_devices_list($vmid);
4431          return 1 if defined($devices_list->{$deviceid});
4432          sleep 1;
4433     }
4434
4435     die "error on hotplug device '$deviceid'\n";
4436 }
4437
4438
4439 sub qemu_devicedelverify {
4440     my ($vmid, $deviceid) = @_;
4441
4442     # need to verify that the device is correctly removed as device_del
4443     # is async and empty return is not reliable
4444
4445     for (my $i = 0; $i <= 5; $i++) {
4446          my $devices_list = vm_devices_list($vmid);
4447          return 1 if !defined($devices_list->{$deviceid});
4448          sleep 1;
4449     }
4450
4451     die "error on hot-unplugging device '$deviceid'\n";
4452 }
4453
4454 sub qemu_findorcreatescsihw {
4455     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4456
4457     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4458
4459     my $scsihwid="$controller_prefix$controller";
4460     my $devices_list = vm_devices_list($vmid);
4461
4462     if(!defined($devices_list->{$scsihwid})) {
4463         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4464     }
4465
4466     return 1;
4467 }
4468
4469 sub qemu_deletescsihw {
4470     my ($conf, $vmid, $opt) = @_;
4471
4472     my $device = parse_drive($opt, $conf->{$opt});
4473
4474     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4475         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4476         return 1;
4477     }
4478
4479     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4480
4481     my $devices_list = vm_devices_list($vmid);
4482     foreach my $opt (keys %{$devices_list}) {
4483         if (PVE::QemuServer::is_valid_drivename($opt)) {
4484             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4485             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4486                 return 1;
4487             }
4488         }
4489     }
4490
4491     my $scsihwid="scsihw$controller";
4492
4493     vm_deviceunplug($vmid, $conf, $scsihwid);
4494
4495     return 1;
4496 }
4497
4498 sub qemu_add_pci_bridge {
4499     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4500
4501     my $bridges = {};
4502
4503     my $bridgeid;
4504
4505     print_pci_addr($device, $bridges, $arch, $machine_type);
4506
4507     while (my ($k, $v) = each %$bridges) {
4508         $bridgeid = $k;
4509     }
4510     return 1 if !defined($bridgeid) || $bridgeid < 1;
4511
4512     my $bridge = "pci.$bridgeid";
4513     my $devices_list = vm_devices_list($vmid);
4514
4515     if (!defined($devices_list->{$bridge})) {
4516         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4517     }
4518
4519     return 1;
4520 }
4521
4522 sub qemu_set_link_status {
4523     my ($vmid, $device, $up) = @_;
4524
4525     vm_mon_cmd($vmid, "set_link", name => $device,
4526                up => $up ? JSON::true : JSON::false);
4527 }
4528
4529 sub qemu_netdevadd {
4530     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4531
4532     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4533     my %options =  split(/[=,]/, $netdev);
4534
4535     vm_mon_cmd($vmid, "netdev_add",  %options);
4536     return 1;
4537 }
4538
4539 sub qemu_netdevdel {
4540     my ($vmid, $deviceid) = @_;
4541
4542     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4543 }
4544
4545 sub qemu_usb_hotplug {
4546     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4547
4548     return if !$device;
4549
4550     # remove the old one first
4551     vm_deviceunplug($vmid, $conf, $deviceid);
4552
4553     # check if xhci controller is necessary and available
4554     if ($device->{usb3}) {
4555
4556         my $devicelist = vm_devices_list($vmid);
4557
4558         if (!$devicelist->{xhci}) {
4559             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4560             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4561         }
4562     }
4563     my $d = parse_usb_device($device->{host});
4564     $d->{usb3} = $device->{usb3};
4565
4566     # add the new one
4567     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4568 }
4569
4570 sub qemu_cpu_hotplug {
4571     my ($vmid, $conf, $vcpus) = @_;
4572
4573     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4574
4575     my $sockets = 1;
4576     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4577     $sockets = $conf->{sockets} if  $conf->{sockets};
4578     my $cores = $conf->{cores} || 1;
4579     my $maxcpus = $sockets * $cores;
4580
4581     $vcpus = $maxcpus if !$vcpus;
4582
4583     die "you can't add more vcpus than maxcpus\n"
4584         if $vcpus > $maxcpus;
4585
4586     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4587
4588     if ($vcpus < $currentvcpus) {
4589
4590         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4591
4592             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4593                 qemu_devicedel($vmid, "cpu$i");
4594                 my $retry = 0;
4595                 my $currentrunningvcpus = undef;
4596                 while (1) {
4597                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4598                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4599                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4600                     $retry++;
4601                     sleep 1;
4602                 }
4603                 #update conf after each succesfull cpu unplug
4604                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4605                 PVE::QemuConfig->write_config($vmid, $conf);
4606             }
4607         } else {
4608             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4609         }
4610
4611         return;
4612     }
4613
4614     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4615     die "vcpus in running vm does not match its configuration\n"
4616         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4617
4618     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4619
4620         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4621             my $cpustr = print_cpu_device($conf, $i);
4622             qemu_deviceadd($vmid, $cpustr);
4623
4624             my $retry = 0;
4625             my $currentrunningvcpus = undef;
4626             while (1) {
4627                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4628                 last if scalar(@{$currentrunningvcpus}) == $i;
4629                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4630                 sleep 1;
4631                 $retry++;
4632             }
4633             #update conf after each succesfull cpu hotplug
4634             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4635             PVE::QemuConfig->write_config($vmid, $conf);
4636         }
4637     } else {
4638
4639         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4640             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4641         }
4642     }
4643 }
4644
4645 sub qemu_block_set_io_throttle {
4646     my ($vmid, $deviceid,
4647         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4648         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4649         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4650         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4651
4652     return if !check_running($vmid) ;
4653
4654     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4655         bps => int($bps),
4656         bps_rd => int($bps_rd),
4657         bps_wr => int($bps_wr),
4658         iops => int($iops),
4659         iops_rd => int($iops_rd),
4660         iops_wr => int($iops_wr),
4661         bps_max => int($bps_max),
4662         bps_rd_max => int($bps_rd_max),
4663         bps_wr_max => int($bps_wr_max),
4664         iops_max => int($iops_max),
4665         iops_rd_max => int($iops_rd_max),
4666         iops_wr_max => int($iops_wr_max),
4667         bps_max_length => int($bps_max_length),
4668         bps_rd_max_length => int($bps_rd_max_length),
4669         bps_wr_max_length => int($bps_wr_max_length),
4670         iops_max_length => int($iops_max_length),
4671         iops_rd_max_length => int($iops_rd_max_length),
4672         iops_wr_max_length => int($iops_wr_max_length),
4673     );
4674
4675 }
4676
4677 # old code, only used to shutdown old VM after update
4678 sub __read_avail {
4679     my ($fh, $timeout) = @_;
4680
4681     my $sel = new IO::Select;
4682     $sel->add($fh);
4683
4684     my $res = '';
4685     my $buf;
4686
4687     my @ready;
4688     while (scalar (@ready = $sel->can_read($timeout))) {
4689         my $count;
4690         if ($count = $fh->sysread($buf, 8192)) {
4691             if ($buf =~ /^(.*)\(qemu\) $/s) {
4692                 $res .= $1;
4693                 last;
4694             } else {
4695                 $res .= $buf;
4696             }
4697         } else {
4698             if (!defined($count)) {
4699                 die "$!\n";
4700             }
4701             last;
4702         }
4703     }
4704
4705     die "monitor read timeout\n" if !scalar(@ready);
4706
4707     return $res;
4708 }
4709
4710 sub qemu_block_resize {
4711     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4712
4713     my $running = check_running($vmid);
4714
4715     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4716
4717     return if !$running;
4718
4719     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4720
4721 }
4722
4723 sub qemu_volume_snapshot {
4724     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4725
4726     my $running = check_running($vmid);
4727
4728     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4729         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4730     } else {
4731         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4732     }
4733 }
4734
4735 sub qemu_volume_snapshot_delete {
4736     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4737
4738     my $running = check_running($vmid);
4739
4740     if($running) {
4741
4742         $running = undef;
4743         my $conf = PVE::QemuConfig->load_config($vmid);
4744         foreach_drive($conf, sub {
4745             my ($ds, $drive) = @_;
4746             $running = 1 if $drive->{file} eq $volid;
4747         });
4748     }
4749
4750     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4751         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4752     } else {
4753         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4754     }
4755 }
4756
4757 sub set_migration_caps {
4758     my ($vmid) = @_;
4759
4760     my $cap_ref = [];
4761
4762     my $enabled_cap = {
4763         "auto-converge" => 1,
4764         "xbzrle" => 1,
4765         "x-rdma-pin-all" => 0,
4766         "zero-blocks" => 0,
4767         "compress" => 0
4768     };
4769
4770     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4771
4772     for my $supported_capability (@$supported_capabilities) {
4773         push @$cap_ref, {
4774             capability => $supported_capability->{capability},
4775             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4776         };
4777     }
4778
4779     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4780 }
4781
4782 my $fast_plug_option = {
4783     'lock' => 1,
4784     'name' => 1,
4785     'onboot' => 1,
4786     'shares' => 1,
4787     'startup' => 1,
4788     'description' => 1,
4789     'protection' => 1,
4790     'vmstatestorage' => 1,
4791     'hookscript' => 1,
4792 };
4793
4794 # hotplug changes in [PENDING]
4795 # $selection hash can be used to only apply specified options, for
4796 # example: { cores => 1 } (only apply changed 'cores')
4797 # $errors ref is used to return error messages
4798 sub vmconfig_hotplug_pending {
4799     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4800
4801     my $defaults = load_defaults();
4802     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4803
4804     # commit values which do not have any impact on running VM first
4805     # Note: those option cannot raise errors, we we do not care about
4806     # $selection and always apply them.
4807
4808     my $add_error = sub {
4809         my ($opt, $msg) = @_;
4810         $errors->{$opt} = "hotplug problem - $msg";
4811     };
4812
4813     my $changes = 0;
4814     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4815         if ($fast_plug_option->{$opt}) {
4816             $conf->{$opt} = $conf->{pending}->{$opt};
4817             delete $conf->{pending}->{$opt};
4818             $changes = 1;
4819         }
4820     }
4821
4822     if ($changes) {
4823         PVE::QemuConfig->write_config($vmid, $conf);
4824         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825     }
4826
4827     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4828
4829     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4830     while (my ($opt, $force) = each %$pending_delete_hash) {
4831         next if $selection && !$selection->{$opt};
4832         eval {
4833             if ($opt eq 'hotplug') {
4834                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4835             } elsif ($opt eq 'tablet') {
4836                 die "skip\n" if !$hotplug_features->{usb};
4837                 if ($defaults->{tablet}) {
4838                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4839                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4840                         if $arch eq 'aarch64';
4841                 } else {
4842                     vm_deviceunplug($vmid, $conf, 'tablet');
4843                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4844                 }
4845             } elsif ($opt =~ m/^usb\d+/) {
4846                 die "skip\n";
4847                 # since we cannot reliably hot unplug usb devices
4848                 # we are disabling it
4849                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4850                 vm_deviceunplug($vmid, $conf, $opt);
4851             } elsif ($opt eq 'vcpus') {
4852                 die "skip\n" if !$hotplug_features->{cpu};
4853                 qemu_cpu_hotplug($vmid, $conf, undef);
4854             } elsif ($opt eq 'balloon') {
4855                 # enable balloon device is not hotpluggable
4856                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4857                 # here we reset the ballooning value to memory
4858                 my $balloon = $conf->{memory} || $defaults->{memory};
4859                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4860             } elsif ($fast_plug_option->{$opt}) {
4861                 # do nothing
4862             } elsif ($opt =~ m/^net(\d+)$/) {
4863                 die "skip\n" if !$hotplug_features->{network};
4864                 vm_deviceunplug($vmid, $conf, $opt);
4865             } elsif (is_valid_drivename($opt)) {
4866                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4867                 vm_deviceunplug($vmid, $conf, $opt);
4868                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4869             } elsif ($opt =~ m/^memory$/) {
4870                 die "skip\n" if !$hotplug_features->{memory};
4871                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4872             } elsif ($opt eq 'cpuunits') {
4873                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4874             } elsif ($opt eq 'cpulimit') {
4875                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4876             } else {
4877                 die "skip\n";
4878             }
4879         };
4880         if (my $err = $@) {
4881             &$add_error($opt, $err) if $err ne "skip\n";
4882         } else {
4883             # save new config if hotplug was successful
4884             delete $conf->{$opt};
4885             vmconfig_undelete_pending_option($conf, $opt);
4886             PVE::QemuConfig->write_config($vmid, $conf);
4887             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4888         }
4889     }
4890
4891     my $apply_pending_cloudinit;
4892     $apply_pending_cloudinit = sub {
4893         my ($key, $value) = @_;
4894         $apply_pending_cloudinit = sub {}; # once is enough
4895
4896         my @cloudinit_opts = keys %$confdesc_cloudinit;
4897         foreach my $opt (keys %{$conf->{pending}}) {
4898             next if !grep { $_ eq $opt } @cloudinit_opts;
4899             $conf->{$opt} = delete $conf->{pending}->{$opt};
4900         }
4901
4902         my $new_conf = { %$conf };
4903         $new_conf->{$key} = $value;
4904         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4905     };
4906
4907     foreach my $opt (keys %{$conf->{pending}}) {
4908         next if $selection && !$selection->{$opt};
4909         my $value = $conf->{pending}->{$opt};
4910         eval {
4911             if ($opt eq 'hotplug') {
4912                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4913             } elsif ($opt eq 'tablet') {
4914                 die "skip\n" if !$hotplug_features->{usb};
4915                 if ($value == 1) {
4916                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4917                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4918                         if $arch eq 'aarch64';
4919                 } elsif ($value == 0) {
4920                     vm_deviceunplug($vmid, $conf, 'tablet');
4921                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4922                 }
4923             } elsif ($opt =~ m/^usb\d+$/) {
4924                 die "skip\n";
4925                 # since we cannot reliably hot unplug usb devices
4926                 # we are disabling it
4927                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4928                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4929                 die "skip\n" if !$d;
4930                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4931             } elsif ($opt eq 'vcpus') {
4932                 die "skip\n" if !$hotplug_features->{cpu};
4933                 qemu_cpu_hotplug($vmid, $conf, $value);
4934             } elsif ($opt eq 'balloon') {
4935                 # enable/disable balloning device is not hotpluggable
4936                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4937                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4938                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4939
4940                 # allow manual ballooning if shares is set to zero
4941                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4942                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4943                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4944                 }
4945             } elsif ($opt =~ m/^net(\d+)$/) {
4946                 # some changes can be done without hotplug
4947                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4948                                     $vmid, $opt, $value, $arch, $machine_type);
4949             } elsif (is_valid_drivename($opt)) {
4950                 # some changes can be done without hotplug
4951                 my $drive = parse_drive($opt, $value);
4952                 if (drive_is_cloudinit($drive)) {
4953                     &$apply_pending_cloudinit($opt, $value);
4954                 }
4955                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4956                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4957             } elsif ($opt =~ m/^memory$/) { #dimms
4958                 die "skip\n" if !$hotplug_features->{memory};
4959                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4960             } elsif ($opt eq 'cpuunits') {
4961                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4962             } elsif ($opt eq 'cpulimit') {
4963                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4964                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4965             } else {
4966                 die "skip\n";  # skip non-hot-pluggable options
4967             }
4968         };
4969         if (my $err = $@) {
4970             &$add_error($opt, $err) if $err ne "skip\n";
4971         } else {
4972             # save new config if hotplug was successful
4973             $conf->{$opt} = $value;
4974             delete $conf->{pending}->{$opt};
4975             PVE::QemuConfig->write_config($vmid, $conf);
4976             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4977         }
4978     }
4979 }
4980
4981 sub try_deallocate_drive {
4982     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4983
4984     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4985         my $volid = $drive->{file};
4986         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4987             my $sid = PVE::Storage::parse_volume_id($volid);
4988             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4989
4990             # check if the disk is really unused
4991             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4992                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4993             PVE::Storage::vdisk_free($storecfg, $volid);
4994             return 1;
4995         } else {
4996             # If vm is not owner of this disk remove from config
4997             return 1;
4998         }
4999     }
5000
5001     return undef;
5002 }
5003
5004 sub vmconfig_delete_or_detach_drive {
5005     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5006
5007     my $drive = parse_drive($opt, $conf->{$opt});
5008
5009     my $rpcenv = PVE::RPCEnvironment::get();
5010     my $authuser = $rpcenv->get_user();
5011
5012     if ($force) {
5013         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5014         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5015     } else {
5016         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5017     }
5018 }
5019
5020 sub vmconfig_apply_pending {
5021     my ($vmid, $conf, $storecfg) = @_;
5022
5023     # cold plug
5024
5025     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5026     while (my ($opt, $force) = each %$pending_delete_hash) {
5027         die "internal error" if $opt =~ m/^unused/;
5028         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5029         if (!defined($conf->{$opt})) {
5030             vmconfig_undelete_pending_option($conf, $opt);
5031             PVE::QemuConfig->write_config($vmid, $conf);
5032         } elsif (is_valid_drivename($opt)) {
5033             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5034             vmconfig_undelete_pending_option($conf, $opt);
5035             delete $conf->{$opt};
5036             PVE::QemuConfig->write_config($vmid, $conf);
5037         } else {
5038             vmconfig_undelete_pending_option($conf, $opt);
5039             delete $conf->{$opt};
5040             PVE::QemuConfig->write_config($vmid, $conf);
5041         }
5042     }
5043
5044     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5045
5046     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5047         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5048
5049         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5050             # skip if nothing changed
5051         } elsif (is_valid_drivename($opt)) {
5052             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5053                 if defined($conf->{$opt});
5054             $conf->{$opt} = $conf->{pending}->{$opt};
5055         } else {
5056             $conf->{$opt} = $conf->{pending}->{$opt};
5057         }
5058
5059         delete $conf->{pending}->{$opt};
5060         PVE::QemuConfig->write_config($vmid, $conf);
5061     }
5062 }
5063
5064 my $safe_num_ne = sub {
5065     my ($a, $b) = @_;
5066
5067     return 0 if !defined($a) && !defined($b);
5068     return 1 if !defined($a);
5069     return 1 if !defined($b);
5070
5071     return $a != $b;
5072 };
5073
5074 my $safe_string_ne = sub {
5075     my ($a, $b) = @_;
5076
5077     return 0 if !defined($a) && !defined($b);
5078     return 1 if !defined($a);
5079     return 1 if !defined($b);
5080
5081     return $a ne $b;
5082 };
5083
5084 sub vmconfig_update_net {
5085     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5086
5087     my $newnet = parse_net($value);
5088
5089     if ($conf->{$opt}) {
5090         my $oldnet = parse_net($conf->{$opt});
5091
5092         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5093             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5094             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5095             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5096
5097             # for non online change, we try to hot-unplug
5098             die "skip\n" if !$hotplug;
5099             vm_deviceunplug($vmid, $conf, $opt);
5100         } else {
5101
5102             die "internal error" if $opt !~ m/net(\d+)/;
5103             my $iface = "tap${vmid}i$1";
5104
5105             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5106                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5107                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5108                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5109                 PVE::Network::tap_unplug($iface);
5110                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5111             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5112                 # Rate can be applied on its own but any change above needs to
5113                 # include the rate in tap_plug since OVS resets everything.
5114                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5115             }
5116
5117             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5118                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5119             }
5120
5121             return 1;
5122         }
5123     }
5124
5125     if ($hotplug) {
5126         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5127     } else {
5128         die "skip\n";
5129     }
5130 }
5131
5132 sub vmconfig_update_disk {
5133     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5134
5135     # fixme: do we need force?
5136
5137     my $drive = parse_drive($opt, $value);
5138
5139     if ($conf->{$opt}) {
5140
5141         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5142
5143             my $media = $drive->{media} || 'disk';
5144             my $oldmedia = $old_drive->{media} || 'disk';
5145             die "unable to change media type\n" if $media ne $oldmedia;
5146
5147             if (!drive_is_cdrom($old_drive)) {
5148
5149                 if ($drive->{file} ne $old_drive->{file}) {
5150
5151                     die "skip\n" if !$hotplug;
5152
5153                     # unplug and register as unused
5154                     vm_deviceunplug($vmid, $conf, $opt);
5155                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5156
5157                 } else {
5158                     # update existing disk
5159
5160                     # skip non hotpluggable value
5161                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5162                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5163                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5164                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5165                         die "skip\n";
5166                     }
5167
5168                     # apply throttle
5169                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5170                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5171                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5172                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5173                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5174                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5175                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5176                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5177                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5178                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5179                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5180                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5181                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5182                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5183                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5184                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5185                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5186                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5187
5188                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5189                                                    ($drive->{mbps} || 0)*1024*1024,
5190                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5191                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5192                                                    $drive->{iops} || 0,
5193                                                    $drive->{iops_rd} || 0,
5194                                                    $drive->{iops_wr} || 0,
5195                                                    ($drive->{mbps_max} || 0)*1024*1024,
5196                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5197                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5198                                                    $drive->{iops_max} || 0,
5199                                                    $drive->{iops_rd_max} || 0,
5200                                                    $drive->{iops_wr_max} || 0,
5201                                                    $drive->{bps_max_length} || 1,
5202                                                    $drive->{bps_rd_max_length} || 1,
5203                                                    $drive->{bps_wr_max_length} || 1,
5204                                                    $drive->{iops_max_length} || 1,
5205                                                    $drive->{iops_rd_max_length} || 1,
5206                                                    $drive->{iops_wr_max_length} || 1);
5207
5208                     }
5209
5210                     return 1;
5211                 }
5212
5213             } else { # cdrom
5214
5215                 if ($drive->{file} eq 'none') {
5216                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5217                     if (drive_is_cloudinit($old_drive)) {
5218                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5219                     }
5220                 } else {
5221                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5222                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5223                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5224                 }
5225
5226                 return 1;
5227             }
5228         }
5229     }
5230
5231     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5232     # hotplug new disks
5233     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5234     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5235 }
5236
5237 sub vm_start {
5238     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5239         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5240
5241     PVE::QemuConfig->lock_config($vmid, sub {
5242         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5243
5244         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5245
5246         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5247
5248         PVE::QemuConfig->check_lock($conf)
5249             if !($skiplock || $is_suspended);
5250
5251         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5252
5253         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5254             vmconfig_apply_pending($vmid, $conf, $storecfg);
5255             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5256         }
5257
5258         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5259
5260         my $defaults = load_defaults();
5261
5262         # set environment variable useful inside network script
5263         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5264
5265         my $local_volumes = {};
5266
5267         if ($targetstorage) {
5268             foreach_drive($conf, sub {
5269                 my ($ds, $drive) = @_;
5270
5271                 return if drive_is_cdrom($drive);
5272
5273                 my $volid = $drive->{file};
5274
5275                 return if !$volid;
5276
5277                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5278
5279                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5280                 return if $scfg->{shared};
5281                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5282             });
5283
5284             my $format = undef;
5285
5286             foreach my $opt (sort keys %$local_volumes) {
5287
5288                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5289                 my $drive = parse_drive($opt, $conf->{$opt});
5290
5291                 #if remote storage is specified, use default format
5292                 if ($targetstorage && $targetstorage ne "1") {
5293                     $storeid = $targetstorage;
5294                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5295                     $format = $defFormat;
5296                 } else {
5297                     #else we use same format than original
5298                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5299                     $format = qemu_img_format($scfg, $volid);
5300                 }
5301
5302                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5303                 my $newdrive = $drive;
5304                 $newdrive->{format} = $format;
5305                 $newdrive->{file} = $newvolid;
5306                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5307                 $local_volumes->{$opt} = $drivestr;
5308                 #pass drive to conf for command line
5309                 $conf->{$opt} = $drivestr;
5310             }
5311         }
5312
5313         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5314
5315         if ($is_suspended) {
5316             # enforce machine type on suspended vm to ensure HW compatibility
5317             $forcemachine = $conf->{runningmachine};
5318             print "Resuming suspended VM\n";
5319         }
5320
5321         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5322
5323         my $migrate_port = 0;
5324         my $migrate_uri;
5325         if ($statefile) {
5326             if ($statefile eq 'tcp') {
5327                 my $localip = "localhost";
5328                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5329                 my $nodename = PVE::INotify::nodename();
5330
5331                 if (!defined($migration_type)) {
5332                     if (defined($datacenterconf->{migration}->{type})) {
5333                         $migration_type = $datacenterconf->{migration}->{type};
5334                     } else {
5335                         $migration_type = 'secure';
5336                     }
5337                 }
5338
5339                 if ($migration_type eq 'insecure') {
5340                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5341                     if ($migrate_network_addr) {
5342                         $localip = $migrate_network_addr;
5343                     } else {
5344                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5345                     }
5346
5347                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5348                 }
5349
5350                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5351                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5352                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5353                 push @$cmd, '-incoming', $migrate_uri;
5354                 push @$cmd, '-S';
5355
5356             } elsif ($statefile eq 'unix') {
5357                 # should be default for secure migrations as a ssh TCP forward
5358                 # tunnel is not deterministic reliable ready and fails regurarly
5359                 # to set up in time, so use UNIX socket forwards
5360                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5361                 unlink $socket_addr;
5362
5363                 $migrate_uri = "unix:$socket_addr";
5364
5365                 push @$cmd, '-incoming', $migrate_uri;
5366                 push @$cmd, '-S';
5367
5368             } else {
5369                 push @$cmd, '-loadstate', $statefile;
5370             }
5371         } elsif ($paused) {
5372             push @$cmd, '-S';
5373         }
5374
5375         # host pci devices
5376         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5377           my $d = parse_hostpci($conf->{"hostpci$i"});
5378           next if !$d;
5379           my $pcidevices = $d->{pciid};
5380           foreach my $pcidevice (@$pcidevices) {
5381                 my $pciid = $pcidevice->{id};
5382
5383                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5384                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5385                 die "no pci device info for device '$pciid'\n" if !$info;
5386
5387                 if ($d->{mdev}) {
5388                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5389                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5390                 } else {
5391                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5392                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5393                     die "can't reset pci device '$pciid'\n"
5394                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5395                 }
5396           }
5397         }
5398
5399         PVE::Storage::activate_volumes($storecfg, $vollist);
5400
5401         eval {
5402             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5403                 outfunc => sub {}, errfunc => sub {});
5404         };
5405         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5406         # timeout should be more than enough here...
5407         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5408
5409         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5410                                                   : $defaults->{cpuunits};
5411
5412         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5413         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5414
5415         my %properties = (
5416             Slice => 'qemu.slice',
5417             KillMode => 'none',
5418             CPUShares => $cpuunits
5419         );
5420
5421         if (my $cpulimit = $conf->{cpulimit}) {
5422             $properties{CPUQuota} = int($cpulimit * 100);
5423         }
5424         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5425
5426         my $run_qemu = sub {
5427             PVE::Tools::run_fork sub {
5428                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5429                 run_command($cmd, %run_params);
5430             };
5431         };
5432
5433         if ($conf->{hugepages}) {
5434
5435             my $code = sub {
5436                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5437                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5438
5439                 PVE::QemuServer::Memory::hugepages_mount();
5440                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5441
5442                 eval { $run_qemu->() };
5443                 if (my $err = $@) {
5444                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5445                     die $err;
5446                 }
5447
5448                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5449             };
5450             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5451
5452         } else {
5453             eval { $run_qemu->() };
5454         }
5455
5456         if (my $err = $@) {
5457             # deactivate volumes if start fails
5458             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5459             die "start failed: $err";
5460         }
5461
5462         print "migration listens on $migrate_uri\n" if $migrate_uri;
5463
5464         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5465             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5466             warn $@ if $@;
5467         }
5468
5469         #start nbd server for storage migration
5470         if ($targetstorage) {
5471             my $nodename = PVE::INotify::nodename();
5472             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5473             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5474             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5475             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5476
5477             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5478
5479             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5480
5481             foreach my $opt (sort keys %$local_volumes) {
5482                 my $volid = $local_volumes->{$opt};
5483                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5484                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5485                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5486             }
5487         }
5488
5489         if ($migratedfrom) {
5490             eval {
5491                 set_migration_caps($vmid);
5492             };
5493             warn $@ if $@;
5494
5495             if ($spice_port) {
5496                 print "spice listens on port $spice_port\n";
5497                 if ($spice_ticket) {
5498                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5499                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5500                 }
5501             }
5502
5503         } else {
5504             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5505                 if !$statefile && $conf->{balloon};
5506
5507             foreach my $opt (keys %$conf) {
5508                 next if $opt !~  m/^net\d+$/;
5509                 my $nicconf = parse_net($conf->{$opt});
5510                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5511             }
5512         }
5513
5514         vm_mon_cmd_nocheck($vmid, 'qom-set',
5515                     path => "machine/peripheral/balloon0",
5516                     property => "guest-stats-polling-interval",
5517                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5518
5519         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5520             print "Resumed VM, removing state\n";
5521             delete $conf->@{qw(lock vmstate runningmachine)};
5522             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5523             PVE::Storage::vdisk_free($storecfg, $vmstate);
5524             PVE::QemuConfig->write_config($vmid, $conf);
5525         }
5526
5527         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5528     });
5529 }
5530
5531 sub vm_mon_cmd {
5532     my ($vmid, $execute, %params) = @_;
5533
5534     my $cmd = { execute => $execute, arguments => \%params };
5535     vm_qmp_command($vmid, $cmd);
5536 }
5537
5538 sub vm_mon_cmd_nocheck {
5539     my ($vmid, $execute, %params) = @_;
5540
5541     my $cmd = { execute => $execute, arguments => \%params };
5542     vm_qmp_command($vmid, $cmd, 1);
5543 }
5544
5545 sub vm_qmp_command {
5546     my ($vmid, $cmd, $nocheck) = @_;
5547
5548     my $res;
5549
5550     my $timeout;
5551     if ($cmd->{arguments}) {
5552         $timeout = delete $cmd->{arguments}->{timeout};
5553     }
5554
5555     eval {
5556         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5557         my $sname = qmp_socket($vmid);
5558         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5559             my $qmpclient = PVE::QMPClient->new();
5560
5561             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5562         } else {
5563             die "unable to open monitor socket\n";
5564         }
5565     };
5566     if (my $err = $@) {
5567         syslog("err", "VM $vmid qmp command failed - $err");
5568         die $err;
5569     }
5570
5571     return $res;
5572 }
5573
5574 sub vm_human_monitor_command {
5575     my ($vmid, $cmdline) = @_;
5576
5577     my $res;
5578
5579     my $cmd = {
5580         execute => 'human-monitor-command',
5581         arguments => { 'command-line' => $cmdline},
5582     };
5583
5584     return vm_qmp_command($vmid, $cmd);
5585 }
5586
5587 sub vm_commandline {
5588     my ($storecfg, $vmid, $snapname) = @_;
5589
5590     my $conf = PVE::QemuConfig->load_config($vmid);
5591
5592     if ($snapname) {
5593         my $snapshot = $conf->{snapshots}->{$snapname};
5594         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5595
5596         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5597
5598         $conf = $snapshot;
5599     }
5600
5601     my $defaults = load_defaults();
5602
5603     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5604
5605     return PVE::Tools::cmd2string($cmd);
5606 }
5607
5608 sub vm_reset {
5609     my ($vmid, $skiplock) = @_;
5610
5611     PVE::QemuConfig->lock_config($vmid, sub {
5612
5613         my $conf = PVE::QemuConfig->load_config($vmid);
5614
5615         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5616
5617         vm_mon_cmd($vmid, "system_reset");
5618     });
5619 }
5620
5621 sub get_vm_volumes {
5622     my ($conf) = @_;
5623
5624     my $vollist = [];
5625     foreach_volid($conf, sub {
5626         my ($volid, $attr) = @_;
5627
5628         return if $volid =~ m|^/|;
5629
5630         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5631         return if !$sid;
5632
5633         push @$vollist, $volid;
5634     });
5635
5636     return $vollist;
5637 }
5638
5639 sub vm_stop_cleanup {
5640     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5641
5642     eval {
5643
5644         if (!$keepActive) {
5645             my $vollist = get_vm_volumes($conf);
5646             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5647         }
5648
5649         foreach my $ext (qw(mon qmp pid vnc qga)) {
5650             unlink "/var/run/qemu-server/${vmid}.$ext";
5651         }
5652
5653         if ($conf->{ivshmem}) {
5654             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5655             # just delete it for now, VMs which have this already open do not
5656             # are affected, but new VMs will get a separated one. If this
5657             # becomes an issue we either add some sort of ref-counting or just
5658             # add a "don't delete on stop" flag to the ivshmem format.
5659             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5660         }
5661
5662         foreach my $key (keys %$conf) {
5663             next if $key !~ m/^hostpci(\d+)$/;
5664             my $hostpciindex = $1;
5665             my $d = parse_hostpci($conf->{$key});
5666             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5667
5668             foreach my $pci (@{$d->{pciid}}) {
5669                 my $pciid = $pci->{id};
5670                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5671             }
5672         }
5673
5674         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5675     };
5676     warn $@ if $@; # avoid errors - just warn
5677 }
5678
5679 # Note: use $nockeck to skip tests if VM configuration file exists.
5680 # We need that when migration VMs to other nodes (files already moved)
5681 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5682 sub vm_stop {
5683     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5684
5685     $force = 1 if !defined($force) && !$shutdown;
5686
5687     if ($migratedfrom){
5688         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5689         kill 15, $pid if $pid;
5690         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5691         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5692         return;
5693     }
5694
5695     PVE::QemuConfig->lock_config($vmid, sub {
5696
5697         my $pid = check_running($vmid, $nocheck);
5698         return if !$pid;
5699
5700         my $conf;
5701         if (!$nocheck) {
5702             $conf = PVE::QemuConfig->load_config($vmid);
5703             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5704             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5705                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5706                 $timeout = $opts->{down} if $opts->{down};
5707             }
5708             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5709         }
5710
5711         eval {
5712             if ($shutdown) {
5713                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5714                     vm_qmp_command($vmid, {
5715                         execute => "guest-shutdown",
5716                         arguments => { timeout => $timeout }
5717                     }, $nocheck);
5718                 } else {
5719                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5720                 }
5721             } else {
5722                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5723             }
5724         };
5725         my $err = $@;
5726
5727         if (!$err) {
5728             $timeout = 60 if !defined($timeout);
5729
5730             my $count = 0;
5731             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5732                 $count++;
5733                 sleep 1;
5734             }
5735
5736             if ($count >= $timeout) {
5737                 if ($force) {
5738                     warn "VM still running - terminating now with SIGTERM\n";
5739                     kill 15, $pid;
5740                 } else {
5741                     die "VM quit/powerdown failed - got timeout\n";
5742                 }
5743             } else {
5744                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5745                 return;
5746             }
5747         } else {
5748             if ($force) {
5749                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5750                 kill 15, $pid;
5751             } else {
5752                 die "VM quit/powerdown failed\n";
5753             }
5754         }
5755
5756         # wait again
5757         $timeout = 10;
5758
5759         my $count = 0;
5760         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5761             $count++;
5762             sleep 1;
5763         }
5764
5765         if ($count >= $timeout) {
5766             warn "VM still running - terminating now with SIGKILL\n";
5767             kill 9, $pid;
5768             sleep 1;
5769         }
5770
5771         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5772    });
5773 }
5774
5775 sub vm_suspend {
5776     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5777
5778     my $conf;
5779     my $path;
5780     my $storecfg;
5781     my $vmstate;
5782
5783     PVE::QemuConfig->lock_config($vmid, sub {
5784
5785         $conf = PVE::QemuConfig->load_config($vmid);
5786
5787         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5788         PVE::QemuConfig->check_lock($conf)
5789             if !($skiplock || $is_backing_up);
5790
5791         die "cannot suspend to disk during backup\n"
5792             if $is_backing_up && $includestate;
5793
5794         if ($includestate) {
5795             $conf->{lock} = 'suspending';
5796             my $date = strftime("%Y-%m-%d", localtime(time()));
5797             $storecfg = PVE::Storage::config();
5798             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5799             $path = PVE::Storage::path($storecfg, $vmstate);
5800             PVE::QemuConfig->write_config($vmid, $conf);
5801         } else {
5802             vm_mon_cmd($vmid, "stop");
5803         }
5804     });
5805
5806     if ($includestate) {
5807         # save vm state
5808         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5809
5810         eval {
5811             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5812             for(;;) {
5813                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5814                 if (!$state->{status}) {
5815                     die "savevm not active\n";
5816                 } elsif ($state->{status} eq 'active') {
5817                     sleep(1);
5818                     next;
5819                 } elsif ($state->{status} eq 'completed') {
5820                     print "State saved, quitting\n";
5821                     last;
5822                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5823                     die "query-savevm failed with error '$state->{error}'\n"
5824                 } else {
5825                     die "query-savevm returned status '$state->{status}'\n";
5826                 }
5827             }
5828         };
5829         my $err = $@;
5830
5831         PVE::QemuConfig->lock_config($vmid, sub {
5832             $conf = PVE::QemuConfig->load_config($vmid);
5833             if ($err) {
5834                 # cleanup, but leave suspending lock, to indicate something went wrong
5835                 eval {
5836                     vm_mon_cmd($vmid, "savevm-end");
5837                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5838                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5839                     delete $conf->@{qw(vmstate runningmachine)};
5840                     PVE::QemuConfig->write_config($vmid, $conf);
5841                 };
5842                 warn $@ if $@;
5843                 die $err;
5844             }
5845
5846             die "lock changed unexpectedly\n"
5847                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5848
5849             vm_qmp_command($vmid, { execute => "quit" });
5850             $conf->{lock} = 'suspended';
5851             PVE::QemuConfig->write_config($vmid, $conf);
5852         });
5853     }
5854 }
5855
5856 sub vm_resume {
5857     my ($vmid, $skiplock, $nocheck) = @_;
5858
5859     PVE::QemuConfig->lock_config($vmid, sub {
5860         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5861         my $res = $vm_mon_cmd->($vmid, 'query-status');
5862         my $resume_cmd = 'cont';
5863
5864         if ($res->{status} && $res->{status} eq 'suspended') {
5865             $resume_cmd = 'system_wakeup';
5866         }
5867
5868         if (!$nocheck) {
5869
5870             my $conf = PVE::QemuConfig->load_config($vmid);
5871
5872             PVE::QemuConfig->check_lock($conf)
5873                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5874         }
5875
5876         $vm_mon_cmd->($vmid, $resume_cmd);
5877     });
5878 }
5879
5880 sub vm_sendkey {
5881     my ($vmid, $skiplock, $key) = @_;
5882
5883     PVE::QemuConfig->lock_config($vmid, sub {
5884
5885         my $conf = PVE::QemuConfig->load_config($vmid);
5886
5887         # there is no qmp command, so we use the human monitor command
5888         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5889         die $res if $res ne '';
5890     });
5891 }
5892
5893 sub vm_destroy {
5894     my ($storecfg, $vmid, $skiplock) = @_;
5895
5896     PVE::QemuConfig->lock_config($vmid, sub {
5897
5898         my $conf = PVE::QemuConfig->load_config($vmid);
5899
5900         if (!check_running($vmid)) {
5901             destroy_vm($storecfg, $vmid, undef, $skiplock);
5902         } else {
5903             die "VM $vmid is running - destroy failed\n";
5904         }
5905     });
5906 }
5907
5908 # vzdump restore implementaion
5909
5910 sub tar_archive_read_firstfile {
5911     my $archive = shift;
5912
5913     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5914
5915     # try to detect archive type first
5916     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5917         die "unable to open file '$archive'\n";
5918     my $firstfile = <$fh>;
5919     kill 15, $pid;
5920     close $fh;
5921
5922     die "ERROR: archive contaions no data\n" if !$firstfile;
5923     chomp $firstfile;
5924
5925     return $firstfile;
5926 }
5927
5928 sub tar_restore_cleanup {
5929     my ($storecfg, $statfile) = @_;
5930
5931     print STDERR "starting cleanup\n";
5932
5933     if (my $fd = IO::File->new($statfile, "r")) {
5934         while (defined(my $line = <$fd>)) {
5935             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5936                 my $volid = $2;
5937                 eval {
5938                     if ($volid =~ m|^/|) {
5939                         unlink $volid || die 'unlink failed\n';
5940                     } else {
5941                         PVE::Storage::vdisk_free($storecfg, $volid);
5942                     }
5943                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5944                 };
5945                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5946             } else {
5947                 print STDERR "unable to parse line in statfile - $line";
5948             }
5949         }
5950         $fd->close();
5951     }
5952 }
5953
5954 sub restore_archive {
5955     my ($archive, $vmid, $user, $opts) = @_;
5956
5957     my $format = $opts->{format};
5958     my $comp;
5959
5960     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5961         $format = 'tar' if !$format;
5962         $comp = 'gzip';
5963     } elsif ($archive =~ m/\.tar$/) {
5964         $format = 'tar' if !$format;
5965     } elsif ($archive =~ m/.tar.lzo$/) {
5966         $format = 'tar' if !$format;
5967         $comp = 'lzop';
5968     } elsif ($archive =~ m/\.vma$/) {
5969         $format = 'vma' if !$format;
5970     } elsif ($archive =~ m/\.vma\.gz$/) {
5971         $format = 'vma' if !$format;
5972         $comp = 'gzip';
5973     } elsif ($archive =~ m/\.vma\.lzo$/) {
5974         $format = 'vma' if !$format;
5975         $comp = 'lzop';
5976     } else {
5977         $format = 'vma' if !$format; # default
5978     }
5979
5980     # try to detect archive format
5981     if ($format eq 'tar') {
5982         return restore_tar_archive($archive, $vmid, $user, $opts);
5983     } else {
5984         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5985     }
5986 }
5987
5988 sub restore_update_config_line {
5989     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5990
5991     return if $line =~ m/^\#qmdump\#/;
5992     return if $line =~ m/^\#vzdump\#/;
5993     return if $line =~ m/^lock:/;
5994     return if $line =~ m/^unused\d+:/;
5995     return if $line =~ m/^parent:/;
5996
5997     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5998     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5999         # try to convert old 1.X settings
6000         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6001         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6002             my ($model, $macaddr) = split(/\=/, $devconfig);
6003             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6004             my $net = {
6005                 model => $model,
6006                 bridge => "vmbr$ind",
6007                 macaddr => $macaddr,
6008             };
6009             my $netstr = print_net($net);
6010
6011             print $outfd "net$cookie->{netcount}: $netstr\n";
6012             $cookie->{netcount}++;
6013         }
6014     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6015         my ($id, $netstr) = ($1, $2);
6016         my $net = parse_net($netstr);
6017         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6018         $netstr = print_net($net);
6019         print $outfd "$id: $netstr\n";
6020     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6021         my $virtdev = $1;
6022         my $value = $3;
6023         my $di = parse_drive($virtdev, $value);
6024         if (defined($di->{backup}) && !$di->{backup}) {
6025             print $outfd "#$line";
6026         } elsif ($map->{$virtdev}) {
6027             delete $di->{format}; # format can change on restore
6028             $di->{file} = $map->{$virtdev};
6029             $value = print_drive($vmid, $di);
6030             print $outfd "$virtdev: $value\n";
6031         } else {
6032             print $outfd $line;
6033         }
6034     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6035         my $vmgenid = $1;
6036         if ($vmgenid ne '0') {
6037             # always generate a new vmgenid if there was a valid one setup
6038             $vmgenid = generate_uuid();
6039         }
6040         print $outfd "vmgenid: $vmgenid\n";
6041     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6042         my ($uuid, $uuid_str);
6043         UUID::generate($uuid);
6044         UUID::unparse($uuid, $uuid_str);
6045         my $smbios1 = parse_smbios1($2);
6046         $smbios1->{uuid} = $uuid_str;
6047         print $outfd $1.print_smbios1($smbios1)."\n";
6048     } else {
6049         print $outfd $line;
6050     }
6051 }
6052
6053 sub scan_volids {
6054     my ($cfg, $vmid) = @_;
6055
6056     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6057
6058     my $volid_hash = {};
6059     foreach my $storeid (keys %$info) {
6060         foreach my $item (@{$info->{$storeid}}) {
6061             next if !($item->{volid} && $item->{size});
6062             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6063             $volid_hash->{$item->{volid}} = $item;
6064         }
6065     }
6066
6067     return $volid_hash;
6068 }
6069
6070 sub is_volume_in_use {
6071     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6072
6073     my $path = PVE::Storage::path($storecfg, $volid);
6074
6075     my $scan_config = sub {
6076         my ($cref, $snapname) = @_;
6077
6078         foreach my $key (keys %$cref) {
6079             my $value = $cref->{$key};
6080             if (is_valid_drivename($key)) {
6081                 next if $skip_drive && $key eq $skip_drive;
6082                 my $drive = parse_drive($key, $value);
6083                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6084                 return 1 if $volid eq $drive->{file};
6085                 if ($drive->{file} =~ m!^/!) {
6086                     return 1 if $drive->{file} eq $path;
6087                 } else {
6088                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6089                     next if !$storeid;
6090                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6091                     next if !$scfg;
6092                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6093                 }
6094             }
6095         }
6096
6097         return 0;
6098     };
6099
6100     return 1 if &$scan_config($conf);
6101
6102     undef $skip_drive;
6103
6104     foreach my $snapname (keys %{$conf->{snapshots}}) {
6105         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6106     }
6107
6108     return 0;
6109 }
6110
6111 sub update_disksize {
6112     my ($vmid, $conf, $volid_hash) = @_;
6113
6114     my $changes;
6115     my $prefix = "VM $vmid:";
6116
6117     # used and unused disks
6118     my $referenced = {};
6119
6120     # Note: it is allowed to define multiple storages with same path (alias), so
6121     # we need to check both 'volid' and real 'path' (two different volid can point
6122     # to the same path).
6123
6124     my $referencedpath = {};
6125
6126     # update size info
6127     foreach my $opt (keys %$conf) {
6128         if (is_valid_drivename($opt)) {
6129             my $drive = parse_drive($opt, $conf->{$opt});
6130             my $volid = $drive->{file};
6131             next if !$volid;
6132
6133             $referenced->{$volid} = 1;
6134             if ($volid_hash->{$volid} &&
6135                 (my $path = $volid_hash->{$volid}->{path})) {
6136                 $referencedpath->{$path} = 1;
6137             }
6138
6139             next if drive_is_cdrom($drive);
6140             next if !$volid_hash->{$volid};
6141
6142             $drive->{size} = $volid_hash->{$volid}->{size};
6143             my $new = print_drive($vmid, $drive);
6144             if ($new ne $conf->{$opt}) {
6145                 $changes = 1;
6146                 $conf->{$opt} = $new;
6147                 print "$prefix update disk '$opt' information.\n";
6148             }
6149         }
6150     }
6151
6152     # remove 'unusedX' entry if volume is used
6153     foreach my $opt (keys %$conf) {
6154         next if $opt !~ m/^unused\d+$/;
6155         my $volid = $conf->{$opt};
6156         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6157         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6158             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6159             $changes = 1;
6160             delete $conf->{$opt};
6161         }
6162
6163         $referenced->{$volid} = 1;
6164         $referencedpath->{$path} = 1 if $path;
6165     }
6166
6167     foreach my $volid (sort keys %$volid_hash) {
6168         next if $volid =~ m/vm-$vmid-state-/;
6169         next if $referenced->{$volid};
6170         my $path = $volid_hash->{$volid}->{path};
6171         next if !$path; # just to be sure
6172         next if $referencedpath->{$path};
6173         $changes = 1;
6174         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6175         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6176         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6177     }
6178
6179     return $changes;
6180 }
6181
6182 sub rescan {
6183     my ($vmid, $nolock, $dryrun) = @_;
6184
6185     my $cfg = PVE::Storage::config();
6186
6187     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6188     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6189     foreach my $stor (keys %{$cfg->{ids}}) {
6190         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6191     }
6192
6193     print "rescan volumes...\n";
6194     my $volid_hash = scan_volids($cfg, $vmid);
6195
6196     my $updatefn =  sub {
6197         my ($vmid) = @_;
6198
6199         my $conf = PVE::QemuConfig->load_config($vmid);
6200
6201         PVE::QemuConfig->check_lock($conf);
6202
6203         my $vm_volids = {};
6204         foreach my $volid (keys %$volid_hash) {
6205             my $info = $volid_hash->{$volid};
6206             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6207         }
6208
6209         my $changes = update_disksize($vmid, $conf, $vm_volids);
6210
6211         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6212     };
6213
6214     if (defined($vmid)) {
6215         if ($nolock) {
6216             &$updatefn($vmid);
6217         } else {
6218             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6219         }
6220     } else {
6221         my $vmlist = config_list();
6222         foreach my $vmid (keys %$vmlist) {
6223             if ($nolock) {
6224                 &$updatefn($vmid);
6225             } else {
6226                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6227             }
6228         }
6229     }
6230 }
6231
6232 sub restore_vma_archive {
6233     my ($archive, $vmid, $user, $opts, $comp) = @_;
6234
6235     my $readfrom = $archive;
6236
6237     my $cfg = PVE::Storage::config();
6238     my $commands = [];
6239     my $bwlimit = $opts->{bwlimit};
6240
6241     my $dbg_cmdstring = '';
6242     my $add_pipe = sub {
6243         my ($cmd) = @_;
6244         push @$commands, $cmd;
6245         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6246         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6247         $readfrom = '-';
6248     };
6249
6250     my $input = undef;
6251     if ($archive eq '-') {
6252         $input = '<&STDIN';
6253     } else {
6254         # If we use a backup from a PVE defined storage we also consider that
6255         # storage's rate limit:
6256         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6257         if (defined($volid)) {
6258             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6259             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6260             if ($readlimit) {
6261                 print STDERR "applying read rate limit: $readlimit\n";
6262                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6263                 $add_pipe->($cstream);
6264             }
6265         }
6266     }
6267
6268     if ($comp) {
6269         my $cmd;
6270         if ($comp eq 'gzip') {
6271             $cmd = ['zcat', $readfrom];
6272         } elsif ($comp eq 'lzop') {
6273             $cmd = ['lzop', '-d', '-c', $readfrom];
6274         } else {
6275             die "unknown compression method '$comp'\n";
6276         }
6277         $add_pipe->($cmd);
6278     }
6279
6280     my $tmpdir = "/var/tmp/vzdumptmp$$";
6281     rmtree $tmpdir;
6282
6283     # disable interrupts (always do cleanups)
6284     local $SIG{INT} =
6285         local $SIG{TERM} =
6286         local $SIG{QUIT} =
6287         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6288
6289     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6290     POSIX::mkfifo($mapfifo, 0600);
6291     my $fifofh;
6292
6293     my $openfifo = sub {
6294         open($fifofh, '>', $mapfifo) || die $!;
6295     };
6296
6297     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6298
6299     my $oldtimeout;
6300     my $timeout = 5;
6301
6302     my $devinfo = {};
6303
6304     my $rpcenv = PVE::RPCEnvironment::get();
6305
6306     my $conffile = PVE::QemuConfig->config_file($vmid);
6307     my $tmpfn = "$conffile.$$.tmp";
6308
6309     # Note: $oldconf is undef if VM does not exists
6310     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6311     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6312
6313     my %storage_limits;
6314
6315     my $print_devmap = sub {
6316         my $virtdev_hash = {};
6317
6318         my $cfgfn = "$tmpdir/qemu-server.conf";
6319
6320         # we can read the config - that is already extracted
6321         my $fh = IO::File->new($cfgfn, "r") ||
6322             "unable to read qemu-server.conf - $!\n";
6323
6324         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6325         if (-f $fwcfgfn) {
6326             my $pve_firewall_dir = '/etc/pve/firewall';
6327             mkdir $pve_firewall_dir; # make sure the dir exists
6328             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6329         }
6330
6331         while (defined(my $line = <$fh>)) {
6332             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6333                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6334                 die "archive does not contain data for drive '$virtdev'\n"
6335                     if !$devinfo->{$devname};
6336                 if (defined($opts->{storage})) {
6337                     $storeid = $opts->{storage} || 'local';
6338                 } elsif (!$storeid) {
6339                     $storeid = 'local';
6340                 }
6341                 $format = 'raw' if !$format;
6342                 $devinfo->{$devname}->{devname} = $devname;
6343                 $devinfo->{$devname}->{virtdev} = $virtdev;
6344                 $devinfo->{$devname}->{format} = $format;
6345                 $devinfo->{$devname}->{storeid} = $storeid;
6346
6347                 # check permission on storage
6348                 my $pool = $opts->{pool}; # todo: do we need that?
6349                 if ($user ne 'root@pam') {
6350                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6351                 }
6352
6353                 $storage_limits{$storeid} = $bwlimit;
6354
6355                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6356             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6357                 my $virtdev = $1;
6358                 my $drive = parse_drive($virtdev, $2);
6359                 if (drive_is_cloudinit($drive)) {
6360                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6361                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6362                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6363
6364                     my $d = {
6365                         format => $format,
6366                         storeid => $opts->{storage} // $storeid,
6367                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6368                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6369                         name => "vm-$vmid-cloudinit",
6370                         is_cloudinit => 1,
6371                     };
6372                     $virtdev_hash->{$virtdev} = $d;
6373                 }
6374             }
6375         }
6376
6377         foreach my $key (keys %storage_limits) {
6378             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6379             next if !$limit;
6380             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6381             $storage_limits{$key} = $limit * 1024;
6382         }
6383
6384         foreach my $devname (keys %$devinfo) {
6385             die "found no device mapping information for device '$devname'\n"
6386                 if !$devinfo->{$devname}->{virtdev};
6387         }
6388
6389         # create empty/temp config
6390         if ($oldconf) {
6391             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6392             foreach_drive($oldconf, sub {
6393                 my ($ds, $drive) = @_;
6394
6395                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6396
6397                 my $volid = $drive->{file};
6398                 return if !$volid || $volid =~ m|^/|;
6399
6400                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6401                 return if !$path || !$owner || ($owner != $vmid);
6402
6403                 # Note: only delete disk we want to restore
6404                 # other volumes will become unused
6405                 if ($virtdev_hash->{$ds}) {
6406                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6407                     if (my $err = $@) {
6408                         warn $err;
6409                     }
6410                 }
6411             });
6412
6413             # delete vmstate files, after the restore we have no snapshots anymore
6414             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6415                 my $snap = $oldconf->{snapshots}->{$snapname};
6416                 if ($snap->{vmstate}) {
6417                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6418                     if (my $err = $@) {
6419                         warn $err;
6420                     }
6421                 }
6422             }
6423         }
6424
6425         my $map = {};
6426         foreach my $virtdev (sort keys %$virtdev_hash) {
6427             my $d = $virtdev_hash->{$virtdev};
6428             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6429             my $storeid = $d->{storeid};
6430             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6431
6432             my $map_opts = '';
6433             if (my $limit = $storage_limits{$storeid}) {
6434                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6435             }
6436
6437             # test if requested format is supported
6438             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6439             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6440             $d->{format} = $defFormat if !$supported;
6441
6442             my $name;
6443             if ($d->{is_cloudinit}) {
6444                 $name = $d->{name};
6445                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6446             }
6447
6448             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6449             print STDERR "new volume ID is '$volid'\n";
6450             $d->{volid} = $volid;
6451
6452             PVE::Storage::activate_volumes($cfg, [$volid]);
6453
6454             my $write_zeros = 1;
6455             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6456                 $write_zeros = 0;
6457             }
6458
6459             if (!$d->{is_cloudinit}) {
6460                 my $path = PVE::Storage::path($cfg, $volid);
6461
6462                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6463
6464                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6465             }
6466             $map->{$virtdev} = $volid;
6467         }
6468
6469         $fh->seek(0, 0) || die "seek failed - $!\n";
6470
6471         my $outfd = new IO::File ($tmpfn, "w") ||
6472             die "unable to write config for VM $vmid\n";
6473
6474         my $cookie = { netcount => 0 };
6475         while (defined(my $line = <$fh>)) {
6476             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6477         }
6478
6479         $fh->close();
6480         $outfd->close();
6481     };
6482
6483     eval {
6484         # enable interrupts
6485         local $SIG{INT} =
6486             local $SIG{TERM} =
6487             local $SIG{QUIT} =
6488             local $SIG{HUP} =
6489             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6490         local $SIG{ALRM} = sub { die "got timeout\n"; };
6491
6492         $oldtimeout = alarm($timeout);
6493
6494         my $parser = sub {
6495             my $line = shift;
6496
6497             print "$line\n";
6498
6499             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6500                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6501                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6502             } elsif ($line =~ m/^CTIME: /) {
6503                 # we correctly received the vma config, so we can disable
6504                 # the timeout now for disk allocation (set to 10 minutes, so
6505                 # that we always timeout if something goes wrong)
6506                 alarm(600);
6507                 &$print_devmap();
6508                 print $fifofh "done\n";
6509                 my $tmp = $oldtimeout || 0;
6510                 $oldtimeout = undef;
6511                 alarm($tmp);
6512                 close($fifofh);
6513             }
6514         };
6515
6516         print "restore vma archive: $dbg_cmdstring\n";
6517         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6518     };
6519     my $err = $@;
6520
6521     alarm($oldtimeout) if $oldtimeout;
6522
6523     my $vollist = [];
6524     foreach my $devname (keys %$devinfo) {
6525         my $volid = $devinfo->{$devname}->{volid};
6526         push @$vollist, $volid if $volid;
6527     }
6528
6529     PVE::Storage::deactivate_volumes($cfg, $vollist);
6530
6531     unlink $mapfifo;
6532
6533     if ($err) {
6534         rmtree $tmpdir;
6535         unlink $tmpfn;
6536
6537         foreach my $devname (keys %$devinfo) {
6538             my $volid = $devinfo->{$devname}->{volid};
6539             next if !$volid;
6540             eval {
6541                 if ($volid =~ m|^/|) {
6542                     unlink $volid || die 'unlink failed\n';
6543                 } else {
6544                     PVE::Storage::vdisk_free($cfg, $volid);
6545                 }
6546                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6547             };
6548             print STDERR "unable to cleanup '$volid' - $@" if $@;
6549         }
6550         die $err;
6551     }
6552
6553     rmtree $tmpdir;
6554
6555     rename($tmpfn, $conffile) ||
6556         die "unable to commit configuration file '$conffile'\n";
6557
6558     PVE::Cluster::cfs_update(); # make sure we read new file
6559
6560     eval { rescan($vmid, 1); };
6561     warn $@ if $@;
6562 }
6563
6564 sub restore_tar_archive {
6565     my ($archive, $vmid, $user, $opts) = @_;
6566
6567     if ($archive ne '-') {
6568         my $firstfile = tar_archive_read_firstfile($archive);
6569         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6570             if $firstfile ne 'qemu-server.conf';
6571     }
6572
6573     my $storecfg = PVE::Storage::config();
6574
6575     # destroy existing data - keep empty config
6576     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6577     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6578
6579     my $tocmd = "/usr/lib/qemu-server/qmextract";
6580
6581     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6582     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6583     $tocmd .= ' --prealloc' if $opts->{prealloc};
6584     $tocmd .= ' --info' if $opts->{info};
6585
6586     # tar option "xf" does not autodetect compression when read from STDIN,
6587     # so we pipe to zcat
6588     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6589         PVE::Tools::shellquote("--to-command=$tocmd");
6590
6591     my $tmpdir = "/var/tmp/vzdumptmp$$";
6592     mkpath $tmpdir;
6593
6594     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6595     local $ENV{VZDUMP_VMID} = $vmid;
6596     local $ENV{VZDUMP_USER} = $user;
6597
6598     my $conffile = PVE::QemuConfig->config_file($vmid);
6599     my $tmpfn = "$conffile.$$.tmp";
6600
6601     # disable interrupts (always do cleanups)
6602     local $SIG{INT} =
6603         local $SIG{TERM} =
6604         local $SIG{QUIT} =
6605         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6606
6607     eval {
6608         # enable interrupts
6609         local $SIG{INT} =
6610             local $SIG{TERM} =
6611             local $SIG{QUIT} =
6612             local $SIG{HUP} =
6613             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6614
6615         if ($archive eq '-') {
6616             print "extracting archive from STDIN\n";
6617             run_command($cmd, input => "<&STDIN");
6618         } else {
6619             print "extracting archive '$archive'\n";
6620             run_command($cmd);
6621         }
6622
6623         return if $opts->{info};
6624
6625         # read new mapping
6626         my $map = {};
6627         my $statfile = "$tmpdir/qmrestore.stat";
6628         if (my $fd = IO::File->new($statfile, "r")) {
6629             while (defined (my $line = <$fd>)) {
6630                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6631                     $map->{$1} = $2 if $1;
6632                 } else {
6633                     print STDERR "unable to parse line in statfile - $line\n";
6634                 }
6635             }
6636             $fd->close();
6637         }
6638
6639         my $confsrc = "$tmpdir/qemu-server.conf";
6640
6641         my $srcfd = new IO::File($confsrc, "r") ||
6642             die "unable to open file '$confsrc'\n";
6643
6644         my $outfd = new IO::File ($tmpfn, "w") ||
6645             die "unable to write config for VM $vmid\n";
6646
6647         my $cookie = { netcount => 0 };
6648         while (defined (my $line = <$srcfd>)) {
6649             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6650         }
6651
6652         $srcfd->close();
6653         $outfd->close();
6654     };
6655     my $err = $@;
6656
6657     if ($err) {
6658
6659         unlink $tmpfn;
6660
6661         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6662
6663         die $err;
6664     }
6665
6666     rmtree $tmpdir;
6667
6668     rename $tmpfn, $conffile ||
6669         die "unable to commit configuration file '$conffile'\n";
6670
6671     PVE::Cluster::cfs_update(); # make sure we read new file
6672
6673     eval { rescan($vmid, 1); };
6674     warn $@ if $@;
6675 };
6676
6677 sub foreach_storage_used_by_vm {
6678     my ($conf, $func) = @_;
6679
6680     my $sidhash = {};
6681
6682     foreach_drive($conf, sub {
6683         my ($ds, $drive) = @_;
6684         return if drive_is_cdrom($drive);
6685
6686         my $volid = $drive->{file};
6687
6688         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6689         $sidhash->{$sid} = $sid if $sid;
6690     });
6691
6692     foreach my $sid (sort keys %$sidhash) {
6693         &$func($sid);
6694     }
6695 }
6696
6697 sub do_snapshots_with_qemu {
6698     my ($storecfg, $volid) = @_;
6699
6700     my $storage_name = PVE::Storage::parse_volume_id($volid);
6701     my $scfg = $storecfg->{ids}->{$storage_name};
6702
6703     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6704         return 1;
6705     }
6706
6707     if ($volid =~ m/\.(qcow2|qed)$/){
6708         return 1;
6709     }
6710
6711     return undef;
6712 }
6713
6714 sub qga_check_running {
6715     my ($vmid, $nowarn) = @_;
6716
6717     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6718     if ($@) {
6719         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6720         return 0;
6721     }
6722     return 1;
6723 }
6724
6725 sub template_create {
6726     my ($vmid, $conf, $disk) = @_;
6727
6728     my $storecfg = PVE::Storage::config();
6729
6730     foreach_drive($conf, sub {
6731         my ($ds, $drive) = @_;
6732
6733         return if drive_is_cdrom($drive);
6734         return if $disk && $ds ne $disk;
6735
6736         my $volid = $drive->{file};
6737         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6738
6739         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6740         $drive->{file} = $voliddst;
6741         $conf->{$ds} = print_drive($vmid, $drive);
6742         PVE::QemuConfig->write_config($vmid, $conf);
6743     });
6744 }
6745
6746 sub convert_iscsi_path {
6747     my ($path) = @_;
6748
6749     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6750         my $portal = $1;
6751         my $target = $2;
6752         my $lun = $3;
6753
6754         my $initiator_name = get_initiator_name();
6755
6756         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6757                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6758     }
6759
6760     die "cannot convert iscsi path '$path', unkown format\n";
6761 }
6762
6763 sub qemu_img_convert {
6764     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6765
6766     my $storecfg = PVE::Storage::config();
6767     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6768     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6769
6770     if ($src_storeid && $dst_storeid) {
6771
6772         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6773
6774         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6775         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6776
6777         my $src_format = qemu_img_format($src_scfg, $src_volname);
6778         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6779
6780         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6781         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6782
6783         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6784         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6785
6786         my $cmd = [];
6787         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6788         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6789         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6790         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6791
6792         if ($src_is_iscsi) {
6793             push @$cmd, '--image-opts';
6794             $src_path = convert_iscsi_path($src_path);
6795         } else {
6796             push @$cmd, '-f', $src_format;
6797         }
6798
6799         if ($dst_is_iscsi) {
6800             push @$cmd, '--target-image-opts';
6801             $dst_path = convert_iscsi_path($dst_path);
6802         } else {
6803             push @$cmd, '-O', $dst_format;
6804         }
6805
6806         push @$cmd, $src_path;
6807
6808         if (!$dst_is_iscsi && $is_zero_initialized) {
6809             push @$cmd, "zeroinit:$dst_path";
6810         } else {
6811             push @$cmd, $dst_path;
6812         }
6813
6814         my $parser = sub {
6815             my $line = shift;
6816             if($line =~ m/\((\S+)\/100\%\)/){
6817                 my $percent = $1;
6818                 my $transferred = int($size * $percent / 100);
6819                 my $remaining = $size - $transferred;
6820
6821                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6822             }
6823
6824         };
6825
6826         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6827         my $err = $@;
6828         die "copy failed: $err" if $err;
6829     }
6830 }
6831
6832 sub qemu_img_format {
6833     my ($scfg, $volname) = @_;
6834
6835     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6836         return $1;
6837     } else {
6838         return "raw";
6839     }
6840 }
6841
6842 sub qemu_drive_mirror {
6843     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6844
6845     $jobs = {} if !$jobs;
6846
6847     my $qemu_target;
6848     my $format;
6849     $jobs->{"drive-$drive"} = {};
6850
6851     if ($dst_volid =~ /^nbd:/) {
6852         $qemu_target = $dst_volid;
6853         $format = "nbd";
6854     } else {
6855         my $storecfg = PVE::Storage::config();
6856         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6857
6858         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6859
6860         $format = qemu_img_format($dst_scfg, $dst_volname);
6861
6862         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6863
6864         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6865     }
6866
6867     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6868     $opts->{format} = $format if $format;
6869
6870     if (defined($bwlimit)) {
6871         $opts->{speed} = $bwlimit * 1024;
6872         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6873     } else {
6874         print "drive mirror is starting for drive-$drive\n";
6875     }
6876
6877     # if a job already runs for this device we get an error, catch it for cleanup
6878     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6879     if (my $err = $@) {
6880         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6881         warn "$@\n" if $@;
6882         die "mirroring error: $err\n";
6883     }
6884
6885     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6886 }
6887
6888 sub qemu_drive_mirror_monitor {
6889     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6890
6891     eval {
6892         my $err_complete = 0;
6893
6894         while (1) {
6895             die "storage migration timed out\n" if $err_complete > 300;
6896
6897             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6898
6899             my $running_mirror_jobs = {};
6900             foreach my $stat (@$stats) {
6901                 next if $stat->{type} ne 'mirror';
6902                 $running_mirror_jobs->{$stat->{device}} = $stat;
6903             }
6904
6905             my $readycounter = 0;
6906
6907             foreach my $job (keys %$jobs) {
6908
6909                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6910                     print "$job : finished\n";
6911                     delete $jobs->{$job};
6912                     next;
6913                 }
6914
6915                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6916
6917                 my $busy = $running_mirror_jobs->{$job}->{busy};
6918                 my $ready = $running_mirror_jobs->{$job}->{ready};
6919                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6920                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6921                     my $remaining = $total - $transferred;
6922                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6923
6924                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6925                 }
6926
6927                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6928             }
6929
6930             last if scalar(keys %$jobs) == 0;
6931
6932             if ($readycounter == scalar(keys %$jobs)) {
6933                 print "all mirroring jobs are ready \n";
6934                 last if $skipcomplete; #do the complete later
6935
6936                 if ($vmiddst && $vmiddst != $vmid) {
6937                     my $agent_running = $qga && qga_check_running($vmid);
6938                     if ($agent_running) {
6939                         print "freeze filesystem\n";
6940                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6941                     } else {
6942                         print "suspend vm\n";
6943                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6944                     }
6945
6946                     # if we clone a disk for a new target vm, we don't switch the disk
6947                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6948
6949                     if ($agent_running) {
6950                         print "unfreeze filesystem\n";
6951                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6952                     } else {
6953                         print "resume vm\n";
6954                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6955                     }
6956
6957                     last;
6958                 } else {
6959
6960                     foreach my $job (keys %$jobs) {
6961                         # try to switch the disk if source and destination are on the same guest
6962                         print "$job: Completing block job...\n";
6963
6964                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6965                         if ($@ =~ m/cannot be completed/) {
6966                             print "$job: Block job cannot be completed, try again.\n";
6967                             $err_complete++;
6968                         }else {
6969                             print "$job: Completed successfully.\n";
6970                             $jobs->{$job}->{complete} = 1;
6971                         }
6972                     }
6973                 }
6974             }
6975             sleep 1;
6976         }
6977     };
6978     my $err = $@;
6979
6980     if ($err) {
6981         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6982         die "mirroring error: $err";
6983     }
6984
6985 }
6986
6987 sub qemu_blockjobs_cancel {
6988     my ($vmid, $jobs) = @_;
6989
6990     foreach my $job (keys %$jobs) {
6991         print "$job: Cancelling block job\n";
6992         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6993         $jobs->{$job}->{cancel} = 1;
6994     }
6995
6996     while (1) {
6997         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6998
6999         my $running_jobs = {};
7000         foreach my $stat (@$stats) {
7001             $running_jobs->{$stat->{device}} = $stat;
7002         }
7003
7004         foreach my $job (keys %$jobs) {
7005
7006             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7007                 print "$job: Done.\n";
7008                 delete $jobs->{$job};
7009             }
7010         }
7011
7012         last if scalar(keys %$jobs) == 0;
7013
7014         sleep 1;
7015     }
7016 }
7017
7018 sub clone_disk {
7019     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7020         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7021
7022     my $newvolid;
7023
7024     if (!$full) {
7025         print "create linked clone of drive $drivename ($drive->{file})\n";
7026         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7027         push @$newvollist, $newvolid;
7028     } else {
7029
7030         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7031         $storeid = $storage if $storage;
7032
7033         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7034         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7035
7036         print "create full clone of drive $drivename ($drive->{file})\n";
7037         my $name = undef;
7038         if (drive_is_cloudinit($drive)) {
7039             $name = "vm-$newvmid-cloudinit";
7040             $snapname = undef;
7041             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7042             if ($dst_format ne 'raw') {
7043                 $name .= ".$dst_format";
7044             }
7045         }
7046         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7047         push @$newvollist, $newvolid;
7048
7049         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7050
7051         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7052         if (!$running || $snapname) {
7053             # TODO: handle bwlimits
7054             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7055         } else {
7056
7057             my $kvmver = get_running_qemu_version ($vmid);
7058             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7059                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7060                     if $drive->{iothread};
7061             }
7062
7063             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7064         }
7065     }
7066
7067     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7068
7069     my $disk = $drive;
7070     $disk->{format} = undef;
7071     $disk->{file} = $newvolid;
7072     $disk->{size} = $size;
7073
7074     return $disk;
7075 }
7076
7077 # this only works if VM is running
7078 sub get_current_qemu_machine {
7079     my ($vmid) = @_;
7080
7081     my $cmd = { execute => 'query-machines', arguments => {} };
7082     my $res = vm_qmp_command($vmid, $cmd);
7083
7084     my ($current, $default);
7085     foreach my $e (@$res) {
7086         $default = $e->{name} if $e->{'is-default'};
7087         $current = $e->{name} if $e->{'is-current'};
7088     }
7089
7090     # fallback to the default machine if current is not supported by qemu
7091     return $current || $default || 'pc';
7092 }
7093
7094 sub get_running_qemu_version {
7095     my ($vmid) = @_;
7096     my $cmd = { execute => 'query-version', arguments => {} };
7097     my $res = vm_qmp_command($vmid, $cmd);
7098     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7099 }
7100
7101 sub qemu_machine_feature_enabled {
7102     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7103
7104     my $current_major;
7105     my $current_minor;
7106
7107     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7108
7109         $current_major = $3;
7110         $current_minor = $4;
7111
7112     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7113
7114         $current_major = $1;
7115         $current_minor = $2;
7116     }
7117
7118     return 1 if $current_major > $version_major ||
7119                 ($current_major == $version_major &&
7120                  $current_minor >= $version_minor);
7121 }
7122
7123 sub qemu_machine_pxe {
7124     my ($vmid, $conf, $machine) = @_;
7125
7126     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7127
7128     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7129         $machine .= '.pxe';
7130     }
7131
7132     return $machine;
7133 }
7134
7135 sub qemu_use_old_bios_files {
7136     my ($machine_type) = @_;
7137
7138     return if !$machine_type;
7139
7140     my $use_old_bios_files = undef;
7141
7142     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7143         $machine_type = $1;
7144         $use_old_bios_files = 1;
7145     } else {
7146         my $kvmver = kvm_user_version();
7147         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7148         # load new efi bios files on migration. So this hack is required to allow
7149         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7150         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7151         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7152     }
7153
7154     return ($use_old_bios_files, $machine_type);
7155 }
7156
7157 sub create_efidisk($$$$$) {
7158     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7159
7160     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7161     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7162
7163     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7164     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7165     PVE::Storage::activate_volumes($storecfg, [$volid]);
7166
7167     my $path = PVE::Storage::path($storecfg, $volid);
7168     eval {
7169         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7170     };
7171     die "Copying EFI vars image failed: $@" if $@;
7172
7173     return ($volid, $vars_size);
7174 }
7175
7176 sub vm_iothreads_list {
7177     my ($vmid) = @_;
7178
7179     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7180
7181     my $iothreads = {};
7182     foreach my $iothread (@$res) {
7183         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7184     }
7185
7186     return $iothreads;
7187 }
7188
7189 sub scsihw_infos {
7190     my ($conf, $drive) = @_;
7191
7192     my $maxdev = 0;
7193
7194     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7195         $maxdev = 7;
7196     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7197         $maxdev = 1;
7198     } else {
7199         $maxdev = 256;
7200     }
7201
7202     my $controller = int($drive->{index} / $maxdev);
7203     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7204
7205     return ($maxdev, $controller, $controller_prefix);
7206 }
7207
7208 sub add_hyperv_enlightenments {
7209     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7210
7211     return if $winversion < 6;
7212     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7213
7214     if ($gpu_passthrough || defined($hv_vendor_id)) {
7215         $hv_vendor_id //= 'proxmox';
7216         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7217     }
7218
7219     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7220         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7221         push @$cpuFlags , 'hv_vapic';
7222         push @$cpuFlags , 'hv_time';
7223     } else {
7224         push @$cpuFlags , 'hv_spinlocks=0xffff';
7225     }
7226
7227     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7228         push @$cpuFlags , 'hv_reset';
7229         push @$cpuFlags , 'hv_vpindex';
7230         push @$cpuFlags , 'hv_runtime';
7231     }
7232
7233     if ($winversion >= 7) {
7234         push @$cpuFlags , 'hv_relaxed';
7235
7236         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7237             push @$cpuFlags , 'hv_synic';
7238             push @$cpuFlags , 'hv_stimer';
7239         }
7240
7241         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7242             push @$cpuFlags , 'hv_tlbflush';
7243             push @$cpuFlags , 'hv_ipi';
7244             # FIXME: AMD does not supports this currently, only add with special flag??
7245             #push @$cpuFlags , 'hv_evmcs';
7246         }
7247     }
7248 }
7249
7250 sub windows_version {
7251     my ($ostype) = @_;
7252
7253     return 0 if !$ostype;
7254
7255     my $winversion = 0;
7256
7257     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7258         $winversion = 5;
7259     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7260         $winversion = 6;
7261     } elsif ($ostype =~ m/^win(\d+)$/) {
7262         $winversion = $1;
7263     }
7264
7265     return $winversion;
7266 }
7267
7268 sub resolve_dst_disk_format {
7269         my ($storecfg, $storeid, $src_volname, $format) = @_;
7270         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7271
7272         if (!$format) {
7273             # if no target format is specified, use the source disk format as hint
7274             if ($src_volname) {
7275                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7276                 $format = qemu_img_format($scfg, $src_volname);
7277             } else {
7278                 return $defFormat;
7279             }
7280         }
7281
7282         # test if requested format is supported - else use default
7283         my $supported = grep { $_ eq $format } @$validFormats;
7284         $format = $defFormat if !$supported;
7285         return $format;
7286 }
7287
7288 sub resolve_first_disk {
7289     my $conf = shift;
7290     my @disks = PVE::QemuServer::valid_drive_names();
7291     my $firstdisk;
7292     foreach my $ds (reverse @disks) {
7293         next if !$conf->{$ds};
7294         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7295         next if PVE::QemuServer::drive_is_cdrom($disk);
7296         $firstdisk = $ds;
7297     }
7298     return $firstdisk;
7299 }
7300
7301 sub generate_uuid {
7302     my ($uuid, $uuid_str);
7303     UUID::generate($uuid);
7304     UUID::unparse($uuid, $uuid_str);
7305     return $uuid_str;
7306 }
7307
7308 sub generate_smbios1_uuid {
7309     return "uuid=".generate_uuid();
7310 }
7311
7312 sub nbd_stop {
7313     my ($vmid) = @_;
7314
7315     vm_mon_cmd($vmid, 'nbd-server-stop');
7316 }
7317
7318 # bash completion helper
7319
7320 sub complete_backup_archives {
7321     my ($cmdname, $pname, $cvalue) = @_;
7322
7323     my $cfg = PVE::Storage::config();
7324
7325     my $storeid;
7326
7327     if ($cvalue =~ m/^([^:]+):/) {
7328         $storeid = $1;
7329     }
7330
7331     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7332
7333     my $res = [];
7334     foreach my $id (keys %$data) {
7335         foreach my $item (@{$data->{$id}}) {
7336             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7337             push @$res, $item->{volid} if defined($item->{volid});
7338         }
7339     }
7340
7341     return $res;
7342 }
7343
7344 my $complete_vmid_full = sub {
7345     my ($running) = @_;
7346
7347     my $idlist = vmstatus();
7348
7349     my $res = [];
7350
7351     foreach my $id (keys %$idlist) {
7352         my $d = $idlist->{$id};
7353         if (defined($running)) {
7354             next if $d->{template};
7355             next if $running && $d->{status} ne 'running';
7356             next if !$running && $d->{status} eq 'running';
7357         }
7358         push @$res, $id;
7359
7360     }
7361     return $res;
7362 };
7363
7364 sub complete_vmid {
7365     return &$complete_vmid_full();
7366 }
7367
7368 sub complete_vmid_stopped {
7369     return &$complete_vmid_full(0);
7370 }
7371
7372 sub complete_vmid_running {
7373     return &$complete_vmid_full(1);
7374 }
7375
7376 sub complete_storage {
7377
7378     my $cfg = PVE::Storage::config();
7379     my $ids = $cfg->{ids};
7380
7381     my $res = [];
7382     foreach my $sid (keys %$ids) {
7383         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7384         next if !$ids->{$sid}->{content}->{images};
7385         push @$res, $sid;
7386     }
7387
7388     return $res;
7389 }
7390
7391 1;