PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     },
 641     audio0 => {
 642         type => 'string',
 643         enum => [qw(ich9-intel-hda intel-hda AC97)],
 644         description => "Configure a audio device, useful in combination with QXL/Spice.",
 645         optional => 1
 646     },
 647 };
 648
 649 my $cicustom_fmt = {
 650     meta => {
 651         type => 'string',
 652         optional => 1,
 653         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 654         format => 'pve-volume-id',
 655         format_description => 'volume',
 656     },
 657     network => {
 658         type => 'string',
 659         optional => 1,
 660         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 661         format => 'pve-volume-id',
 662         format_description => 'volume',
 663     },
 664     user => {
 665         type => 'string',
 666         optional => 1,
 667         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 668         format => 'pve-volume-id',
 669         format_description => 'volume',
 670     },
 671 };
 672 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 673
 674 my $confdesc_cloudinit = {
 675     citype => {
 676         optional => 1,
 677         type => 'string',
 678         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 679         enum => ['configdrive2', 'nocloud'],
 680     },
 681     ciuser => {
 682         optional => 1,
 683         type => 'string',
 684         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 685     },
 686     cipassword => {
 687         optional => 1,
 688         type => 'string',
 689         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 690     },
 691     cicustom => {
 692         optional => 1,
 693         type => 'string',
 694         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 695         format => 'pve-qm-cicustom',
 696     },
 697     searchdomain => {
 698         optional => 1,
 699         type => 'string',
 700         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 701     },
 702     nameserver => {
 703         optional => 1,
 704         type => 'string', format => 'address-list',
 705         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 706     },
 707     sshkeys => {
 708         optional => 1,
 709         type => 'string',
 710         format => 'urlencoded',
 711         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 712     },
 713 };
 714
 715 # what about other qemu settings ?
 716 #cpu => 'string',
 717 #machine => 'string',
 718 #fda => 'file',
 719 #fdb => 'file',
 720 #mtdblock => 'file',
 721 #sd => 'file',
 722 #pflash => 'file',
 723 #snapshot => 'bool',
 724 #bootp => 'file',
 725 ##tftp => 'dir',
 726 ##smb => 'dir',
 727 #kernel => 'file',
 728 #append => 'string',
 729 #initrd => 'file',
 730 ##soundhw => 'string',
 731
 732 while (my ($k, $v) = each %$confdesc) {
 733     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 734 }
 735
 736 my $MAX_IDE_DISKS = 4;
 737 my $MAX_SCSI_DISKS = 14;
 738 my $MAX_VIRTIO_DISKS = 16;
 739 my $MAX_SATA_DISKS = 6;
 740 my $MAX_USB_DEVICES = 5;
 741 my $MAX_NETS = 32;
 742 my $MAX_UNUSED_DISKS = 256;
 743 my $MAX_HOSTPCI_DEVICES = 4;
 744 my $MAX_SERIAL_PORTS = 4;
 745 my $MAX_PARALLEL_PORTS = 3;
 746 my $MAX_NUMA = 8;
 747
 748 my $numa_fmt = {
 749     cpus => {
 750         type => "string",
 751         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 752         description => "CPUs accessing this NUMA node.",
 753         format_description => "id[-id];...",
 754     },
 755     memory => {
 756         type => "number",
 757         description => "Amount of memory this NUMA node provides.",
 758         optional => 1,
 759     },
 760     hostnodes => {
 761         type => "string",
 762         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 763         description => "Host NUMA nodes to use.",
 764         format_description => "id[-id];...",
 765         optional => 1,
 766     },
 767     policy => {
 768         type => 'string',
 769         enum => [qw(preferred bind interleave)],
 770         description => "NUMA allocation policy.",
 771         optional => 1,
 772     },
 773 };
 774 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 775 my $numadesc = {
 776     optional => 1,
 777     type => 'string', format => $numa_fmt,
 778     description => "NUMA topology.",
 779 };
 780 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 781
 782 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 783     $confdesc->{"numa$i"} = $numadesc;
 784 }
 785
 786 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 787                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 788                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 789 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 790
 791 my $net_fmt_bridge_descr = <<__EOD__;
 792 Bridge to attach the network device to. The Proxmox VE standard bridge
 793 is called 'vmbr0'.
 794
 795 If you do not specify a bridge, we create a kvm user (NATed) network
 796 device, which provides DHCP and DNS services. The following addresses
 797 are used:
 798
 799  10.0.2.2   Gateway
 800  10.0.2.3   DNS Server
 801  10.0.2.4   SMB Server
 802
 803 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 804 __EOD__
 805
 806 my $net_fmt = {
 807     macaddr  => get_standard_option('mac-addr', {
 808         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 809     }),
 810     model => {
 811         type => 'string',
 812         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 813         enum => $nic_model_list,
 814         default_key => 1,
 815     },
 816     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 817     bridge => {
 818         type => 'string',
 819         description => $net_fmt_bridge_descr,
 820         format_description => 'bridge',
 821         optional => 1,
 822     },
 823     queues => {
 824         type => 'integer',
 825         minimum => 0, maximum => 16,
 826         description => 'Number of packet queues to be used on the device.',
 827         optional => 1,
 828     },
 829     rate => {
 830         type => 'number',
 831         minimum => 0,
 832         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 833         optional => 1,
 834     },
 835     tag => {
 836         type => 'integer',
 837         minimum => 1, maximum => 4094,
 838         description => 'VLAN tag to apply to packets on this interface.',
 839         optional => 1,
 840     },
 841     trunks => {
 842         type => 'string',
 843         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 844         description => 'VLAN trunks to pass through this interface.',
 845         format_description => 'vlanid[;vlanid...]',
 846         optional => 1,
 847     },
 848     firewall => {
 849         type => 'boolean',
 850         description => 'Whether this interface should be protected by the firewall.',
 851         optional => 1,
 852     },
 853     link_down => {
 854         type => 'boolean',
 855         description => 'Whether this interface should be disconnected (like pulling the plug).',
 856         optional => 1,
 857     },
 858 };
 859
 860 my $netdesc = {
 861     optional => 1,
 862     type => 'string', format => $net_fmt,
 863     description => "Specify network devices.",
 864 };
 865
 866 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 867
 868 my $ipconfig_fmt = {
 869     ip => {
 870         type => 'string',
 871         format => 'pve-ipv4-config',
 872         format_description => 'IPv4Format/CIDR',
 873         description => 'IPv4 address in CIDR format.',
 874         optional => 1,
 875         default => 'dhcp',
 876     },
 877     gw => {
 878         type => 'string',
 879         format => 'ipv4',
 880         format_description => 'GatewayIPv4',
 881         description => 'Default gateway for IPv4 traffic.',
 882         optional => 1,
 883         requires => 'ip',
 884     },
 885     ip6 => {
 886         type => 'string',
 887         format => 'pve-ipv6-config',
 888         format_description => 'IPv6Format/CIDR',
 889         description => 'IPv6 address in CIDR format.',
 890         optional => 1,
 891         default => 'dhcp',
 892     },
 893     gw6 => {
 894         type => 'string',
 895         format => 'ipv6',
 896         format_description => 'GatewayIPv6',
 897         description => 'Default gateway for IPv6 traffic.',
 898         optional => 1,
 899         requires => 'ip6',
 900     },
 901 };
 902 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 903 my $ipconfigdesc = {
 904     optional => 1,
 905     type => 'string', format => 'pve-qm-ipconfig',
 906     description => <<'EODESCR',
 907 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 908
 909 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 910
 911 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 912 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 913
 914 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 915 EODESCR
 916 };
 917 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 918
 919 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 920     $confdesc->{"net$i"} = $netdesc;
 921     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 922 }
 923
 924 foreach my $key (keys %$confdesc_cloudinit) {
 925     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 926 }
 927
 928 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 929 sub verify_volume_id_or_qm_path {
 930     my ($volid, $noerr) = @_;
 931
 932     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 933         return $volid;
 934     }
 935
 936     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 937     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 938     if ($@) {
 939         return undef if $noerr;
 940         die $@;
 941     }
 942     return $volid;
 943 }
 944
 945 my $drivename_hash;
 946
 947 my %drivedesc_base = (
 948     volume => { alias => 'file' },
 949     file => {
 950         type => 'string',
 951         format => 'pve-volume-id-or-qm-path',
 952         default_key => 1,
 953         format_description => 'volume',
 954         description => "The drive's backing volume.",
 955     },
 956     media => {
 957         type => 'string',
 958         enum => [qw(cdrom disk)],
 959         description => "The drive's media type.",
 960         default => 'disk',
 961         optional => 1
 962     },
 963     cyls => {
 964         type => 'integer',
 965         description => "Force the drive's physical geometry to have a specific cylinder count.",
 966         optional => 1
 967     },
 968     heads => {
 969         type => 'integer',
 970         description => "Force the drive's physical geometry to have a specific head count.",
 971         optional => 1
 972     },
 973     secs => {
 974         type => 'integer',
 975         description => "Force the drive's physical geometry to have a specific sector count.",
 976         optional => 1
 977     },
 978     trans => {
 979         type => 'string',
 980         enum => [qw(none lba auto)],
 981         description => "Force disk geometry bios translation mode.",
 982         optional => 1,
 983     },
 984     snapshot => {
 985         type => 'boolean',
 986         description => "Controls qemu's snapshot mode feature."
 987             . " If activated, changes made to the disk are temporary and will"
 988             . " be discarded when the VM is shutdown.",
 989         optional => 1,
 990     },
 991     cache => {
 992         type => 'string',
 993         enum => [qw(none writethrough writeback unsafe directsync)],
 994         description => "The drive's cache mode",
 995         optional => 1,
 996     },
 997     format => get_standard_option('pve-qm-image-format'),
 998     size => {
 999         type => 'string',
1000         format => 'disk-size',
1001         format_description => 'DiskSize',
1002         description => "Disk size. This is purely informational and has no effect.",
1003         optional => 1,
1004     },
1005     backup => {
1006         type => 'boolean',
1007         description => "Whether the drive should be included when making backups.",
1008         optional => 1,
1009     },
1010     replicate => {
1011         type => 'boolean',
1012         description => 'Whether the drive should considered for replication jobs.',
1013         optional => 1,
1014         default => 1,
1015     },
1016     rerror => {
1017         type => 'string',
1018         enum => [qw(ignore report stop)],
1019         description => 'Read error action.',
1020         optional => 1,
1021     },
1022     werror => {
1023         type => 'string',
1024         enum => [qw(enospc ignore report stop)],
1025         description => 'Write error action.',
1026         optional => 1,
1027     },
1028     aio => {
1029         type => 'string',
1030         enum => [qw(native threads)],
1031         description => 'AIO type to use.',
1032         optional => 1,
1033     },
1034     discard => {
1035         type => 'string',
1036         enum => [qw(ignore on)],
1037         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1038         optional => 1,
1039     },
1040     detect_zeroes => {
1041         type => 'boolean',
1042         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1043         optional => 1,
1044     },
1045     serial => {
1046         type => 'string',
1047         format => 'urlencoded',
1048         format_description => 'serial',
1049         maxLength => 20*3, # *3 since it's %xx url enoded
1050         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1051         optional => 1,
1052     },
1053     shared => {
1054         type => 'boolean',
1055         description => 'Mark this locally-managed volume as available on all nodes',
1056         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1057         optional => 1,
1058         default => 0,
1059     }
1060 );
1061
1062 my %iothread_fmt = ( iothread => {
1063         type => 'boolean',
1064         description => "Whether to use iothreads for this drive",
1065         optional => 1,
1066 });
1067
1068 my %model_fmt = (
1069     model => {
1070         type => 'string',
1071         format => 'urlencoded',
1072         format_description => 'model',
1073         maxLength => 40*3, # *3 since it's %xx url enoded
1074         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1075         optional => 1,
1076     },
1077 );
1078
1079 my %queues_fmt = (
1080     queues => {
1081         type => 'integer',
1082         description => "Number of queues.",
1083         minimum => 2,
1084         optional => 1
1085     }
1086 );
1087
1088 my %scsiblock_fmt = (
1089     scsiblock => {
1090         type => 'boolean',
1091         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1092         optional => 1,
1093         default => 0,
1094     },
1095 );
1096
1097 my %ssd_fmt = (
1098     ssd => {
1099         type => 'boolean',
1100         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1101         optional => 1,
1102     },
1103 );
1104
1105 my %wwn_fmt = (
1106     wwn => {
1107         type => 'string',
1108         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1109         format_description => 'wwn',
1110         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1111         optional => 1,
1112     },
1113 );
1114
1115 my $add_throttle_desc = sub {
1116     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1117     my $d = {
1118         type => $type,
1119         format_description => $unit,
1120         description => "Maximum $what in $longunit.",
1121         optional => 1,
1122     };
1123     $d->{minimum} = $minimum if defined($minimum);
1124     $drivedesc_base{$key} = $d;
1125 };
1126 # throughput: (leaky bucket)
1127 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1128 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1129 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1130 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1131 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1132 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1133 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1134 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1135 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1136
1137 # pools: (pool of IO before throttling starts taking effect)
1138 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1139 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1140 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1141 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1142 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1143 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1144
1145 # burst lengths
1146 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1147 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1148 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1150 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1151 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1152
1153 # legacy support
1154 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1155 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1156 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1157 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1158
1159 my $ide_fmt = {
1160     %drivedesc_base,
1161     %model_fmt,
1162     %ssd_fmt,
1163     %wwn_fmt,
1164 };
1165 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1166
1167 my $idedesc = {
1168     optional => 1,
1169     type => 'string', format => $ide_fmt,
1170     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1171 };
1172 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1173
1174 my $scsi_fmt = {
1175     %drivedesc_base,
1176     %iothread_fmt,
1177     %queues_fmt,
1178     %scsiblock_fmt,
1179     %ssd_fmt,
1180     %wwn_fmt,
1181 };
1182 my $scsidesc = {
1183     optional => 1,
1184     type => 'string', format => $scsi_fmt,
1185     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1186 };
1187 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1188
1189 my $sata_fmt = {
1190     %drivedesc_base,
1191     %ssd_fmt,
1192     %wwn_fmt,
1193 };
1194 my $satadesc = {
1195     optional => 1,
1196     type => 'string', format => $sata_fmt,
1197     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1198 };
1199 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1200
1201 my $virtio_fmt = {
1202     %drivedesc_base,
1203     %iothread_fmt,
1204 };
1205 my $virtiodesc = {
1206     optional => 1,
1207     type => 'string', format => $virtio_fmt,
1208     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1209 };
1210 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1211
1212 my $alldrive_fmt = {
1213     %drivedesc_base,
1214     %iothread_fmt,
1215     %model_fmt,
1216     %queues_fmt,
1217     %scsiblock_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221
1222 my $efidisk_fmt = {
1223     volume => { alias => 'file' },
1224     file => {
1225         type => 'string',
1226         format => 'pve-volume-id-or-qm-path',
1227         default_key => 1,
1228         format_description => 'volume',
1229         description => "The drive's backing volume.",
1230     },
1231     format => get_standard_option('pve-qm-image-format'),
1232     size => {
1233         type => 'string',
1234         format => 'disk-size',
1235         format_description => 'DiskSize',
1236         description => "Disk size. This is purely informational and has no effect.",
1237         optional => 1,
1238     },
1239 };
1240
1241 my $efidisk_desc = {
1242     optional => 1,
1243     type => 'string', format => $efidisk_fmt,
1244     description => "Configure a Disk for storing EFI vars",
1245 };
1246
1247 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1248
1249 my $usb_fmt = {
1250     host => {
1251         default_key => 1,
1252         type => 'string', format => 'pve-qm-usb-device',
1253         format_description => 'HOSTUSBDEVICE|spice',
1254         description => <<EODESCR,
1255 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1256
1257  'bus-port(.port)*' (decimal numbers) or
1258  'vendor_id:product_id' (hexadeciaml numbers) or
1259  'spice'
1260
1261 You can use the 'lsusb -t' command to list existing usb devices.
1262
1263 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1264
1265 The value 'spice' can be used to add a usb redirection devices for spice.
1266 EODESCR
1267     },
1268     usb3 => {
1269         optional => 1,
1270         type => 'boolean',
1271         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1272         default => 0,
1273     },
1274 };
1275
1276 my $usbdesc = {
1277     optional => 1,
1278     type => 'string', format => $usb_fmt,
1279     description => "Configure an USB device (n is 0 to 4).",
1280 };
1281 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1282
1283 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1284 my $hostpci_fmt = {
1285     host => {
1286         default_key => 1,
1287         type => 'string',
1288         pattern => qr/$PCIRE(;$PCIRE)*/,
1289         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1290         description => <<EODESCR,
1291 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1292 of PCI virtual functions of the host. HOSTPCIID syntax is:
1293
1294 'bus:dev.func' (hexadecimal numbers)
1295
1296 You can us the 'lspci' command to list existing PCI devices.
1297 EODESCR
1298     },
1299     rombar => {
1300         type => 'boolean',
1301         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1302         optional => 1,
1303         default => 1,
1304     },
1305     romfile => {
1306         type => 'string',
1307         pattern => '[^,;]+',
1308         format_description => 'string',
1309         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1310         optional => 1,
1311     },
1312     pcie => {
1313         type => 'boolean',
1314         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'x-vga' => {
1319         type => 'boolean',
1320         description =>  "Enable vfio-vga device support.",
1321         optional => 1,
1322         default => 0,
1323     },
1324     'mdev' => {
1325         type => 'string',
1326         format_description => 'string',
1327         pattern => '[^/\.:]+',
1328         optional => 1,
1329         description => <<EODESCR
1330 The type of mediated device to use.
1331 An instance of this type will be created on startup of the VM and
1332 will be cleaned up when the VM stops.
1333 EODESCR
1334     }
1335 };
1336 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1337
1338 my $hostpcidesc = {
1339         optional => 1,
1340         type => 'string', format => 'pve-qm-hostpci',
1341         description => "Map host PCI devices into guest.",
1342         verbose_description =>  <<EODESCR,
1343 Map host PCI devices into guest.
1344
1345 NOTE: This option allows direct access to host hardware. So it is no longer
1346 possible to migrate such machines - use with special care.
1347
1348 CAUTION: Experimental! User reported problems with this option.
1349 EODESCR
1350 };
1351 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1352
1353 my $serialdesc = {
1354         optional => 1,
1355         type => 'string',
1356         pattern => '(/dev/.+|socket)',
1357         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1358         verbose_description =>  <<EODESCR,
1359 Create a serial device inside the VM (n is 0 to 3), and pass through a
1360 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1361 host side (use 'qm terminal' to open a terminal connection).
1362
1363 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1364
1365 CAUTION: Experimental! User reported problems with this option.
1366 EODESCR
1367 };
1368
1369 my $paralleldesc= {
1370         optional => 1,
1371         type => 'string',
1372         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1373         description =>  "Map host parallel devices (n is 0 to 2).",
1374         verbose_description =>  <<EODESCR,
1375 Map host parallel devices (n is 0 to 2).
1376
1377 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1378
1379 CAUTION: Experimental! User reported problems with this option.
1380 EODESCR
1381 };
1382
1383 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1384     $confdesc->{"parallel$i"} = $paralleldesc;
1385 }
1386
1387 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1388     $confdesc->{"serial$i"} = $serialdesc;
1389 }
1390
1391 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1392     $confdesc->{"hostpci$i"} = $hostpcidesc;
1393 }
1394
1395 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1396     $drivename_hash->{"ide$i"} = 1;
1397     $confdesc->{"ide$i"} = $idedesc;
1398 }
1399
1400 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1401     $drivename_hash->{"sata$i"} = 1;
1402     $confdesc->{"sata$i"} = $satadesc;
1403 }
1404
1405 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1406     $drivename_hash->{"scsi$i"} = 1;
1407     $confdesc->{"scsi$i"} = $scsidesc ;
1408 }
1409
1410 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1411     $drivename_hash->{"virtio$i"} = 1;
1412     $confdesc->{"virtio$i"} = $virtiodesc;
1413 }
1414
1415 $drivename_hash->{efidisk0} = 1;
1416 $confdesc->{efidisk0} = $efidisk_desc;
1417
1418 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1419     $confdesc->{"usb$i"} = $usbdesc;
1420 }
1421
1422 my $unuseddesc = {
1423     optional => 1,
1424     type => 'string', format => 'pve-volume-id',
1425     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1426 };
1427
1428 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1429     $confdesc->{"unused$i"} = $unuseddesc;
1430 }
1431
1432 my $kvm_api_version = 0;
1433
1434 sub kvm_version {
1435     return $kvm_api_version if $kvm_api_version;
1436
1437     open my $fh, '<', '/dev/kvm'
1438         or return undef;
1439
1440     # 0xae00 => KVM_GET_API_VERSION
1441     $kvm_api_version = ioctl($fh, 0xae00, 0);
1442
1443     return $kvm_api_version;
1444 }
1445
1446 my $kvm_user_version;
1447
1448 sub kvm_user_version {
1449
1450     return $kvm_user_version if $kvm_user_version;
1451
1452     $kvm_user_version = 'unknown';
1453
1454     my $code = sub {
1455         my $line = shift;
1456         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1457             $kvm_user_version = $2;
1458         }
1459     };
1460
1461     eval { run_command("kvm -version", outfunc => $code); };
1462     warn $@ if $@;
1463
1464     return $kvm_user_version;
1465
1466 }
1467
1468 sub kernel_has_vhost_net {
1469     return -c '/dev/vhost-net';
1470 }
1471
1472 sub valid_drive_names {
1473     # order is important - used to autoselect boot disk
1474     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1475             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1476             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1477             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1478             'efidisk0');
1479 }
1480
1481 sub is_valid_drivename {
1482     my $dev = shift;
1483
1484     return defined($drivename_hash->{$dev});
1485 }
1486
1487 sub option_exists {
1488     my $key = shift;
1489     return defined($confdesc->{$key});
1490 }
1491
1492 sub nic_models {
1493     return $nic_model_list;
1494 }
1495
1496 sub os_list_description {
1497
1498     return {
1499         other => 'Other',
1500         wxp => 'Windows XP',
1501         w2k => 'Windows 2000',
1502         w2k3 =>, 'Windows 2003',
1503         w2k8 => 'Windows 2008',
1504         wvista => 'Windows Vista',
1505         win7 => 'Windows 7',
1506         win8 => 'Windows 8/2012',
1507         win10 => 'Windows 10/2016',
1508         l24 => 'Linux 2.4',
1509         l26 => 'Linux 2.6',
1510     };
1511 }
1512
1513 my $cdrom_path;
1514
1515 sub get_cdrom_path {
1516
1517     return  $cdrom_path if $cdrom_path;
1518
1519     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1520     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1521     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1522 }
1523
1524 sub get_iso_path {
1525     my ($storecfg, $vmid, $cdrom) = @_;
1526
1527     if ($cdrom eq 'cdrom') {
1528         return get_cdrom_path();
1529     } elsif ($cdrom eq 'none') {
1530         return '';
1531     } elsif ($cdrom =~ m|^/|) {
1532         return $cdrom;
1533     } else {
1534         return PVE::Storage::path($storecfg, $cdrom);
1535     }
1536 }
1537
1538 # try to convert old style file names to volume IDs
1539 sub filename_to_volume_id {
1540     my ($vmid, $file, $media) = @_;
1541
1542      if (!($file eq 'none' || $file eq 'cdrom' ||
1543           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1544
1545         return undef if $file =~ m|/|;
1546
1547         if ($media && $media eq 'cdrom') {
1548             $file = "local:iso/$file";
1549         } else {
1550             $file = "local:$vmid/$file";
1551         }
1552     }
1553
1554     return $file;
1555 }
1556
1557 sub verify_media_type {
1558     my ($opt, $vtype, $media) = @_;
1559
1560     return if !$media;
1561
1562     my $etype;
1563     if ($media eq 'disk') {
1564         $etype = 'images';
1565     } elsif ($media eq 'cdrom') {
1566         $etype = 'iso';
1567     } else {
1568         die "internal error";
1569     }
1570
1571     return if ($vtype eq $etype);
1572
1573     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1574 }
1575
1576 sub cleanup_drive_path {
1577     my ($opt, $storecfg, $drive) = @_;
1578
1579     # try to convert filesystem paths to volume IDs
1580
1581     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1582         ($drive->{file} !~ m|^/dev/.+|) &&
1583         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1584         ($drive->{file} !~ m/^\d+$/)) {
1585         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1586         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1587         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1588         verify_media_type($opt, $vtype, $drive->{media});
1589         $drive->{file} = $volid;
1590     }
1591
1592     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1593 }
1594
1595 sub parse_hotplug_features {
1596     my ($data) = @_;
1597
1598     my $res = {};
1599
1600     return $res if $data eq '0';
1601
1602     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1603
1604     foreach my $feature (PVE::Tools::split_list($data)) {
1605         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1606             $res->{$1} = 1;
1607         } else {
1608             die "invalid hotplug feature '$feature'\n";
1609         }
1610     }
1611     return $res;
1612 }
1613
1614 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1615 sub pve_verify_hotplug_features {
1616     my ($value, $noerr) = @_;
1617
1618     return $value if parse_hotplug_features($value);
1619
1620     return undef if $noerr;
1621
1622     die "unable to parse hotplug option\n";
1623 }
1624
1625 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1626 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1627 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1628 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1629 #        [,iothread=on][,serial=serial][,model=model]
1630
1631 sub parse_drive {
1632     my ($key, $data) = @_;
1633
1634     my ($interface, $index);
1635
1636     if ($key =~ m/^([^\d]+)(\d+)$/) {
1637         $interface = $1;
1638         $index = $2;
1639     } else {
1640         return undef;
1641     }
1642
1643     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1644                                      : $confdesc->{$key}->{format};
1645     if (!$desc) {
1646         warn "invalid drive key: $key\n";
1647         return undef;
1648     }
1649     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1650     return undef if !$res;
1651     $res->{interface} = $interface;
1652     $res->{index} = $index;
1653
1654     my $error = 0;
1655     foreach my $opt (qw(bps bps_rd bps_wr)) {
1656         if (my $bps = defined(delete $res->{$opt})) {
1657             if (defined($res->{"m$opt"})) {
1658                 warn "both $opt and m$opt specified\n";
1659                 ++$error;
1660                 next;
1661             }
1662             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1663         }
1664     }
1665
1666     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1667     for my $requirement (
1668         [mbps_max => 'mbps'],
1669         [mbps_rd_max => 'mbps_rd'],
1670         [mbps_wr_max => 'mbps_wr'],
1671         [miops_max => 'miops'],
1672         [miops_rd_max => 'miops_rd'],
1673         [miops_wr_max => 'miops_wr'],
1674         [bps_max_length => 'mbps_max'],
1675         [bps_rd_max_length => 'mbps_rd_max'],
1676         [bps_wr_max_length => 'mbps_wr_max'],
1677         [iops_max_length => 'iops_max'],
1678         [iops_rd_max_length => 'iops_rd_max'],
1679         [iops_wr_max_length => 'iops_wr_max']) {
1680         my ($option, $requires) = @$requirement;
1681         if ($res->{$option} && !$res->{$requires}) {
1682             warn "$option requires $requires\n";
1683             ++$error;
1684         }
1685     }
1686
1687     return undef if $error;
1688
1689     return undef if $res->{mbps_rd} && $res->{mbps};
1690     return undef if $res->{mbps_wr} && $res->{mbps};
1691     return undef if $res->{iops_rd} && $res->{iops};
1692     return undef if $res->{iops_wr} && $res->{iops};
1693
1694     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1695         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1696         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1697         return undef if $res->{interface} eq 'virtio';
1698     }
1699
1700     if (my $size = $res->{size}) {
1701         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1702     }
1703
1704     return $res;
1705 }
1706
1707 sub print_drive {
1708     my ($vmid, $drive) = @_;
1709     my $data = { %$drive };
1710     delete $data->{$_} for qw(index interface);
1711     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1712 }
1713
1714 sub scsi_inquiry {
1715     my($fh, $noerr) = @_;
1716
1717     my $SG_IO = 0x2285;
1718     my $SG_GET_VERSION_NUM = 0x2282;
1719
1720     my $versionbuf = "\x00" x 8;
1721     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1722     if (!$ret) {
1723         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1724         return undef;
1725     }
1726     my $version = unpack("I", $versionbuf);
1727     if ($version < 30000) {
1728         die "scsi generic interface too old\n"  if !$noerr;
1729         return undef;
1730     }
1731
1732     my $buf = "\x00" x 36;
1733     my $sensebuf = "\x00" x 8;
1734     my $cmd = pack("C x3 C x1", 0x12, 36);
1735
1736     # see /usr/include/scsi/sg.h
1737     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1738
1739     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1740                       length($sensebuf), 0, length($buf), $buf,
1741                       $cmd, $sensebuf, 6000);
1742
1743     $ret = ioctl($fh, $SG_IO, $packet);
1744     if (!$ret) {
1745         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my @res = unpack($sg_io_hdr_t, $packet);
1750     if ($res[17] || $res[18]) {
1751         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1752         return undef;
1753     }
1754
1755     my $res = {};
1756     (my $byte0, my $byte1, $res->{vendor},
1757      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1758
1759     $res->{removable} = $byte1 & 128 ? 1 : 0;
1760     $res->{type} = $byte0 & 31;
1761
1762     return $res;
1763 }
1764
1765 sub path_is_scsi {
1766     my ($path) = @_;
1767
1768     my $fh = IO::File->new("+<$path") || return undef;
1769     my $res = scsi_inquiry($fh, 1);
1770     close($fh);
1771
1772     return $res;
1773 }
1774
1775 sub machine_type_is_q35 {
1776     my ($conf) = @_;
1777
1778     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1779 }
1780
1781 sub print_tabletdevice_full {
1782     my ($conf, $arch) = @_;
1783
1784     my $q35 = machine_type_is_q35($conf);
1785
1786     # we use uhci for old VMs because tablet driver was buggy in older qemu
1787     my $usbbus;
1788     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1789         $usbbus = 'ehci';
1790     } else {
1791         $usbbus = 'uhci';
1792     }
1793
1794     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1795 }
1796
1797 sub print_keyboarddevice_full {
1798     my ($conf, $arch, $machine) = @_;
1799
1800     return undef if $arch ne 'aarch64';
1801
1802     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1803 }
1804
1805 sub print_drivedevice_full {
1806     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1807
1808     my $device = '';
1809     my $maxdev = 0;
1810
1811     if ($drive->{interface} eq 'virtio') {
1812         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1813         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1814         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1815     } elsif ($drive->{interface} eq 'scsi') {
1816
1817         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1818         my $unit = $drive->{index} % $maxdev;
1819         my $devicetype = 'hd';
1820         my $path = '';
1821         if (drive_is_cdrom($drive)) {
1822             $devicetype = 'cd';
1823         } else {
1824             if ($drive->{file} =~ m|^/|) {
1825                 $path = $drive->{file};
1826                 if (my $info = path_is_scsi($path)) {
1827                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1828                         $devicetype = 'block';
1829                     } elsif ($info->{type} == 1) { # tape
1830                         $devicetype = 'generic';
1831                     }
1832                 }
1833             } else {
1834                  $path = PVE::Storage::path($storecfg, $drive->{file});
1835             }
1836
1837             if($path =~ m/^iscsi\:\/\//){
1838                 $devicetype = 'generic';
1839             }
1840         }
1841
1842         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1843             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1844         } else {
1845             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1846         }
1847
1848         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1849             $device .= ",rotation_rate=1";
1850         }
1851         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1852
1853     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1854         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1855         my $controller = int($drive->{index} / $maxdev);
1856         my $unit = $drive->{index} % $maxdev;
1857         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1858
1859         $device = "ide-$devicetype";
1860         if ($drive->{interface} eq 'ide') {
1861             $device .= ",bus=ide.$controller,unit=$unit";
1862         } else {
1863             $device .= ",bus=ahci$controller.$unit";
1864         }
1865         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1866
1867         if ($devicetype eq 'hd') {
1868             if (my $model = $drive->{model}) {
1869                 $model = URI::Escape::uri_unescape($model);
1870                 $device .= ",model=$model";
1871             }
1872             if ($drive->{ssd}) {
1873                 $device .= ",rotation_rate=1";
1874             }
1875         }
1876         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1877     } elsif ($drive->{interface} eq 'usb') {
1878         die "implement me";
1879         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1880     } else {
1881         die "unsupported interface type";
1882     }
1883
1884     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1885
1886     if (my $serial = $drive->{serial}) {
1887         $serial = URI::Escape::uri_unescape($serial);
1888         $device .= ",serial=$serial";
1889     }
1890
1891
1892     return $device;
1893 }
1894
1895 sub get_initiator_name {
1896     my $initiator;
1897
1898     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1899     while (defined(my $line = <$fh>)) {
1900         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1901         $initiator = $1;
1902         last;
1903     }
1904     $fh->close();
1905
1906     return $initiator;
1907 }
1908
1909 sub print_drive_full {
1910     my ($storecfg, $vmid, $drive) = @_;
1911
1912     my $path;
1913     my $volid = $drive->{file};
1914     my $format;
1915
1916     if (drive_is_cdrom($drive)) {
1917         $path = get_iso_path($storecfg, $vmid, $volid);
1918     } else {
1919         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1920         if ($storeid) {
1921             $path = PVE::Storage::path($storecfg, $volid);
1922             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1923             $format = qemu_img_format($scfg, $volname);
1924         } else {
1925             $path = $volid;
1926             $format = "raw";
1927         }
1928    }
1929
1930     my $opts = '';
1931     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1932     foreach my $o (@qemu_drive_options) {
1933         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1934     }
1935
1936     # snapshot only accepts on|off
1937     if (defined($drive->{snapshot})) {
1938         my $v = $drive->{snapshot} ? 'on' : 'off';
1939         $opts .= ",snapshot=$v";
1940     }
1941
1942     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1943         my ($dir, $qmpname) = @$type;
1944         if (my $v = $drive->{"mbps$dir"}) {
1945             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1946         }
1947         if (my $v = $drive->{"mbps${dir}_max"}) {
1948             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1949         }
1950         if (my $v = $drive->{"bps${dir}_max_length"}) {
1951             $opts .= ",throttling.bps$qmpname-max-length=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}"}) {
1954             $opts .= ",throttling.iops$qmpname=$v";
1955         }
1956         if (my $v = $drive->{"iops${dir}_max"}) {
1957             $opts .= ",throttling.iops$qmpname-max=$v";
1958         }
1959         if (my $v = $drive->{"iops${dir}_max_length"}) {
1960             $opts .= ",throttling.iops$qmpname-max-length=$v";
1961         }
1962     }
1963
1964     $opts .= ",format=$format" if $format && !$drive->{format};
1965
1966     my $cache_direct = 0;
1967
1968     if (my $cache = $drive->{cache}) {
1969         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1970     } elsif (!drive_is_cdrom($drive)) {
1971         $opts .= ",cache=none";
1972         $cache_direct = 1;
1973     }
1974
1975     # aio native works only with O_DIRECT
1976     if (!$drive->{aio}) {
1977         if($cache_direct) {
1978             $opts .= ",aio=native";
1979         } else {
1980             $opts .= ",aio=threads";
1981         }
1982     }
1983
1984     if (!drive_is_cdrom($drive)) {
1985         my $detectzeroes;
1986         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1987             $detectzeroes = 'off';
1988         } elsif ($drive->{discard}) {
1989             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1990         } else {
1991             # This used to be our default with discard not being specified:
1992             $detectzeroes = 'on';
1993         }
1994         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1995     }
1996
1997     my $pathinfo = $path ? "file=$path," : '';
1998
1999     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2000 }
2001
2002 sub print_netdevice_full {
2003     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2004
2005     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2006
2007     my $device = $net->{model};
2008     if ($net->{model} eq 'virtio') {
2009          $device = 'virtio-net-pci';
2010      };
2011
2012     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2013     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2014     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2015         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2016         my $vectors = $net->{queues} * 2 + 2;
2017         $tmpstr .= ",vectors=$vectors,mq=on";
2018     }
2019     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2020
2021     if ($use_old_bios_files) {
2022         my $romfile;
2023         if ($device eq 'virtio-net-pci') {
2024             $romfile = 'pxe-virtio.rom';
2025         } elsif ($device eq 'e1000') {
2026             $romfile = 'pxe-e1000.rom';
2027         } elsif ($device eq 'ne2k') {
2028             $romfile = 'pxe-ne2k_pci.rom';
2029         } elsif ($device eq 'pcnet') {
2030             $romfile = 'pxe-pcnet.rom';
2031         } elsif ($device eq 'rtl8139') {
2032             $romfile = 'pxe-rtl8139.rom';
2033         }
2034         $tmpstr .= ",romfile=$romfile" if $romfile;
2035     }
2036
2037     return $tmpstr;
2038 }
2039
2040 sub print_netdev_full {
2041     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2042
2043     my $i = '';
2044     if ($netid =~ m/^net(\d+)$/) {
2045         $i = int($1);
2046     }
2047
2048     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2049
2050     my $ifname = "tap${vmid}i$i";
2051
2052     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2053     die "interface name '$ifname' is too long (max 15 character)\n"
2054         if length($ifname) >= 16;
2055
2056     my $vhostparam = '';
2057     if (is_native($arch)) {
2058         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2059     }
2060
2061     my $vmname = $conf->{name} || "vm$vmid";
2062
2063     my $netdev = "";
2064     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2065
2066     if ($net->{bridge}) {
2067         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2068     } else {
2069         $netdev = "type=user,id=$netid,hostname=$vmname";
2070     }
2071
2072     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2073
2074     return $netdev;
2075 }
2076
2077
2078 sub print_cpu_device {
2079     my ($conf, $id) = @_;
2080
2081     my $kvm = $conf->{kvm} // 1;
2082     my $cpu = $kvm ? "kvm64" : "qemu64";
2083     if (my $cputype = $conf->{cpu}) {
2084         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2085             or die "Cannot parse cpu description: $cputype\n";
2086         $cpu = $cpuconf->{cputype};
2087     }
2088
2089     my $cores = $conf->{cores} || 1;
2090
2091     my $current_core = ($id - 1) % $cores;
2092     my $current_socket = int(($id - 1 - $current_core)/$cores);
2093
2094     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2095 }
2096
2097 my $vga_map = {
2098     'cirrus' => 'cirrus-vga',
2099     'std' => 'VGA',
2100     'vmware' => 'vmware-svga',
2101     'virtio' => 'virtio-vga',
2102 };
2103
2104 sub print_vga_device {
2105     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2106
2107     my $type = $vga_map->{$vga->{type}};
2108     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2109         $type = 'virtio-gpu';
2110     }
2111     my $vgamem_mb = $vga->{memory};
2112     if ($qxlnum) {
2113         $type = $id ? 'qxl' : 'qxl-vga';
2114     }
2115     die "no devicetype for $vga->{type}\n" if !$type;
2116
2117     my $memory = "";
2118     if ($vgamem_mb) {
2119         if ($vga->{type} eq 'virtio') {
2120             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2121             $memory = ",max_hostmem=$bytes";
2122         } elsif ($qxlnum) {
2123             # from https://www.spice-space.org/multiple-monitors.html
2124             $memory = ",vgamem_mb=$vga->{memory}";
2125             my $ram = $vgamem_mb * 4;
2126             my $vram = $vgamem_mb * 2;
2127             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2128         } else {
2129             $memory = ",vgamem_mb=$vga->{memory}";
2130         }
2131     } elsif ($qxlnum && $id) {
2132         $memory = ",ram_size=67108864,vram_size=33554432";
2133     }
2134
2135     my $q35 = machine_type_is_q35($conf);
2136     my $vgaid = "vga" . ($id // '');
2137     my $pciaddr;
2138
2139     if ($q35 && $vgaid eq 'vga') {
2140         # the first display uses pcie.0 bus on q35 machines
2141         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2142     } else {
2143         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2144     }
2145
2146     return "$type,id=${vgaid}${memory}${pciaddr}";
2147 }
2148
2149 sub drive_is_cloudinit {
2150     my ($drive) = @_;
2151     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2152 }
2153
2154 sub drive_is_cdrom {
2155     my ($drive, $exclude_cloudinit) = @_;
2156
2157     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2158
2159     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2160
2161 }
2162
2163 sub parse_number_sets {
2164     my ($set) = @_;
2165     my $res = [];
2166     foreach my $part (split(/;/, $set)) {
2167         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2168             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2169             push @$res, [ $1, $2 ];
2170         } else {
2171             die "invalid range: $part\n";
2172         }
2173     }
2174     return $res;
2175 }
2176
2177 sub parse_numa {
2178     my ($data) = @_;
2179
2180     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2181     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2182     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2183     return $res;
2184 }
2185
2186 sub parse_hostpci {
2187     my ($value) = @_;
2188
2189     return undef if !$value;
2190
2191     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2192
2193     my @idlist = split(/;/, $res->{host});
2194     delete $res->{host};
2195     foreach my $id (@idlist) {
2196         if ($id =~ m/\./) { # full id 00:00.1
2197             push @{$res->{pciid}}, {
2198                 id => $id,
2199             };
2200         } else { # partial id 00:00
2201             $res->{pciid} = PVE::SysFSTools::lspci($id);
2202         }
2203     }
2204     return $res;
2205 }
2206
2207 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2208 sub parse_net {
2209     my ($data) = @_;
2210
2211     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2212     if ($@) {
2213         warn $@;
2214         return undef;
2215     }
2216     if (!defined($res->{macaddr})) {
2217         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2218         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2219     }
2220     return $res;
2221 }
2222
2223 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2224 sub parse_ipconfig {
2225     my ($data) = @_;
2226
2227     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2228     if ($@) {
2229         warn $@;
2230         return undef;
2231     }
2232
2233     if ($res->{gw} && !$res->{ip}) {
2234         warn 'gateway specified without specifying an IP address';
2235         return undef;
2236     }
2237     if ($res->{gw6} && !$res->{ip6}) {
2238         warn 'IPv6 gateway specified without specifying an IPv6 address';
2239         return undef;
2240     }
2241     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2242         warn 'gateway specified together with DHCP';
2243         return undef;
2244     }
2245     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2246         # gw6 + auto/dhcp
2247         warn "IPv6 gateway specified together with $res->{ip6} address";
2248         return undef;
2249     }
2250
2251     if (!$res->{ip} && !$res->{ip6}) {
2252         return { ip => 'dhcp', ip6 => 'dhcp' };
2253     }
2254
2255     return $res;
2256 }
2257
2258 sub print_net {
2259     my $net = shift;
2260
2261     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2262 }
2263
2264 sub add_random_macs {
2265     my ($settings) = @_;
2266
2267     foreach my $opt (keys %$settings) {
2268         next if $opt !~ m/^net(\d+)$/;
2269         my $net = parse_net($settings->{$opt});
2270         next if !$net;
2271         $settings->{$opt} = print_net($net);
2272     }
2273 }
2274
2275 sub vm_is_volid_owner {
2276     my ($storecfg, $vmid, $volid) = @_;
2277
2278     if ($volid !~  m|^/|) {
2279         my ($path, $owner);
2280         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2281         if ($owner && ($owner == $vmid)) {
2282             return 1;
2283         }
2284     }
2285
2286     return undef;
2287 }
2288
2289 sub split_flagged_list {
2290     my $text = shift || '';
2291     $text =~ s/[,;]/ /g;
2292     $text =~ s/^\s+//;
2293     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2294 }
2295
2296 sub join_flagged_list {
2297     my ($how, $lst) = @_;
2298     join $how, map { $lst->{$_} . $_ } keys %$lst;
2299 }
2300
2301 sub vmconfig_delete_pending_option {
2302     my ($conf, $key, $force) = @_;
2303
2304     delete $conf->{pending}->{$key};
2305     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2306     $pending_delete_hash->{$key} = $force ? '!' : '';
2307     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2308 }
2309
2310 sub vmconfig_undelete_pending_option {
2311     my ($conf, $key) = @_;
2312
2313     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2314     delete $pending_delete_hash->{$key};
2315
2316     if (%$pending_delete_hash) {
2317         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2318     } else {
2319         delete $conf->{pending}->{delete};
2320     }
2321 }
2322
2323 sub vmconfig_register_unused_drive {
2324     my ($storecfg, $vmid, $conf, $drive) = @_;
2325
2326     if (drive_is_cloudinit($drive)) {
2327         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2328         warn $@ if $@;
2329     } elsif (!drive_is_cdrom($drive)) {
2330         my $volid = $drive->{file};
2331         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2332             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2333         }
2334     }
2335 }
2336
2337 sub vmconfig_cleanup_pending {
2338     my ($conf) = @_;
2339
2340     # remove pending changes when nothing changed
2341     my $changes;
2342     foreach my $opt (keys %{$conf->{pending}}) {
2343         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2344             $changes = 1;
2345             delete $conf->{pending}->{$opt};
2346         }
2347     }
2348
2349     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2350     my $pending_delete_hash = {};
2351     while (my ($opt, $force) = each %$current_delete_hash) {
2352         if (defined($conf->{$opt})) {
2353             $pending_delete_hash->{$opt} = $force;
2354         } else {
2355             $changes = 1;
2356         }
2357     }
2358
2359     if (%$pending_delete_hash) {
2360         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2361     } else {
2362         delete $conf->{pending}->{delete};
2363     }
2364
2365     return $changes;
2366 }
2367
2368 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2369 my $smbios1_fmt = {
2370     uuid => {
2371         type => 'string',
2372         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2373         format_description => 'UUID',
2374         description => "Set SMBIOS1 UUID.",
2375         optional => 1,
2376     },
2377     version => {
2378         type => 'string',
2379         pattern => '[A-Za-z0-9+\/]+={0,2}',
2380         format_description => 'Base64 encoded string',
2381         description => "Set SMBIOS1 version.",
2382         optional => 1,
2383     },
2384     serial => {
2385         type => 'string',
2386         pattern => '[A-Za-z0-9+\/]+={0,2}',
2387         format_description => 'Base64 encoded string',
2388         description => "Set SMBIOS1 serial number.",
2389         optional => 1,
2390     },
2391     manufacturer => {
2392         type => 'string',
2393         pattern => '[A-Za-z0-9+\/]+={0,2}',
2394         format_description => 'Base64 encoded string',
2395         description => "Set SMBIOS1 manufacturer.",
2396         optional => 1,
2397     },
2398     product => {
2399         type => 'string',
2400         pattern => '[A-Za-z0-9+\/]+={0,2}',
2401         format_description => 'Base64 encoded string',
2402         description => "Set SMBIOS1 product ID.",
2403         optional => 1,
2404     },
2405     sku => {
2406         type => 'string',
2407         pattern => '[A-Za-z0-9+\/]+={0,2}',
2408         format_description => 'Base64 encoded string',
2409         description => "Set SMBIOS1 SKU string.",
2410         optional => 1,
2411     },
2412     family => {
2413         type => 'string',
2414         pattern => '[A-Za-z0-9+\/]+={0,2}',
2415         format_description => 'Base64 encoded string',
2416         description => "Set SMBIOS1 family string.",
2417         optional => 1,
2418     },
2419     base64 => {
2420         type => 'boolean',
2421         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2422         optional => 1,
2423     },
2424 };
2425
2426 sub parse_smbios1 {
2427     my ($data) = @_;
2428
2429     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2430     warn $@ if $@;
2431     return $res;
2432 }
2433
2434 sub print_smbios1 {
2435     my ($smbios1) = @_;
2436     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2437 }
2438
2439 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2440
2441 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2442 sub verify_bootdisk {
2443     my ($value, $noerr) = @_;
2444
2445     return $value if is_valid_drivename($value);
2446
2447     return undef if $noerr;
2448
2449     die "invalid boot disk '$value'\n";
2450 }
2451
2452 sub parse_watchdog {
2453     my ($value) = @_;
2454
2455     return undef if !$value;
2456
2457     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2458     warn $@ if $@;
2459     return $res;
2460 }
2461
2462 sub parse_guest_agent {
2463     my ($value) = @_;
2464
2465     return {} if !defined($value->{agent});
2466
2467     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2468     warn $@ if $@;
2469
2470     # if the agent is disabled ignore the other potentially set properties
2471     return {} if !$res->{enabled};
2472     return $res;
2473 }
2474
2475 sub parse_vga {
2476     my ($value) = @_;
2477
2478     return {} if !$value;
2479     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2480     warn $@ if $@;
2481     return $res;
2482 }
2483
2484 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2485 sub verify_usb_device {
2486     my ($value, $noerr) = @_;
2487
2488     return $value if parse_usb_device($value);
2489
2490     return undef if $noerr;
2491
2492     die "unable to parse usb device\n";
2493 }
2494
2495 # add JSON properties for create and set function
2496 sub json_config_properties {
2497     my $prop = shift;
2498
2499     foreach my $opt (keys %$confdesc) {
2500         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2501         $prop->{$opt} = $confdesc->{$opt};
2502     }
2503
2504     return $prop;
2505 }
2506
2507 # return copy of $confdesc_cloudinit to generate documentation
2508 sub cloudinit_config_properties {
2509
2510     return dclone($confdesc_cloudinit);
2511 }
2512
2513 sub check_type {
2514     my ($key, $value) = @_;
2515
2516     die "unknown setting '$key'\n" if !$confdesc->{$key};
2517
2518     my $type = $confdesc->{$key}->{type};
2519
2520     if (!defined($value)) {
2521         die "got undefined value\n";
2522     }
2523
2524     if ($value =~ m/[\n\r]/) {
2525         die "property contains a line feed\n";
2526     }
2527
2528     if ($type eq 'boolean') {
2529         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2530         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2531         die "type check ('boolean') failed - got '$value'\n";
2532     } elsif ($type eq 'integer') {
2533         return int($1) if $value =~ m/^(\d+)$/;
2534         die "type check ('integer') failed - got '$value'\n";
2535     } elsif ($type eq 'number') {
2536         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2537         die "type check ('number') failed - got '$value'\n";
2538     } elsif ($type eq 'string') {
2539         if (my $fmt = $confdesc->{$key}->{format}) {
2540             PVE::JSONSchema::check_format($fmt, $value);
2541             return $value;
2542         }
2543         $value =~ s/^\"(.*)\"$/$1/;
2544         return $value;
2545     } else {
2546         die "internal error"
2547     }
2548 }
2549
2550 sub touch_config {
2551     my ($vmid) = @_;
2552
2553     my $conf = PVE::QemuConfig->config_file($vmid);
2554     utime undef, undef, $conf;
2555 }
2556
2557 sub destroy_vm {
2558     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2559
2560     my $conffile = PVE::QemuConfig->config_file($vmid);
2561
2562     my $conf = PVE::QemuConfig->load_config($vmid);
2563
2564     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2565
2566     if ($conf->{template}) {
2567         # check if any base image is still used by a linked clone
2568         foreach_drive($conf, sub {
2569                 my ($ds, $drive) = @_;
2570
2571                 return if drive_is_cdrom($drive);
2572
2573                 my $volid = $drive->{file};
2574
2575                 return if !$volid || $volid =~ m|^/|;
2576
2577                 die "base volume '$volid' is still in use by linked cloned\n"
2578                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2579
2580         });
2581     }
2582
2583     # only remove disks owned by this VM
2584     foreach_drive($conf, sub {
2585         my ($ds, $drive) = @_;
2586
2587         return if drive_is_cdrom($drive, 1);
2588
2589         my $volid = $drive->{file};
2590
2591         return if !$volid || $volid =~ m|^/|;
2592
2593         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2594         return if !$path || !$owner || ($owner != $vmid);
2595
2596         eval {
2597             PVE::Storage::vdisk_free($storecfg, $volid);
2598         };
2599         warn "Could not remove disk '$volid', check manually: $@" if $@;
2600
2601     });
2602
2603     if ($keep_empty_config) {
2604         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2605     } else {
2606         unlink $conffile;
2607     }
2608
2609     # also remove unused disk
2610     eval {
2611         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2612
2613         eval {
2614             PVE::Storage::foreach_volid($dl, sub {
2615                 my ($volid, $sid, $volname, $d) = @_;
2616                 PVE::Storage::vdisk_free($storecfg, $volid);
2617             });
2618         };
2619         warn $@ if $@;
2620
2621     };
2622     warn $@ if $@;
2623 }
2624
2625 sub parse_vm_config {
2626     my ($filename, $raw) = @_;
2627
2628     return undef if !defined($raw);
2629
2630     my $res = {
2631         digest => Digest::SHA::sha1_hex($raw),
2632         snapshots => {},
2633         pending => {},
2634     };
2635
2636     $filename =~ m|/qemu-server/(\d+)\.conf$|
2637         || die "got strange filename '$filename'";
2638
2639     my $vmid = $1;
2640
2641     my $conf = $res;
2642     my $descr;
2643     my $section = '';
2644
2645     my @lines = split(/\n/, $raw);
2646     foreach my $line (@lines) {
2647         next if $line =~ m/^\s*$/;
2648
2649         if ($line =~ m/^\[PENDING\]\s*$/i) {
2650             $section = 'pending';
2651             if (defined($descr)) {
2652                 $descr =~ s/\s+$//;
2653                 $conf->{description} = $descr;
2654             }
2655             $descr = undef;
2656             $conf = $res->{$section} = {};
2657             next;
2658
2659         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2660             $section = $1;
2661             if (defined($descr)) {
2662                 $descr =~ s/\s+$//;
2663                 $conf->{description} = $descr;
2664             }
2665             $descr = undef;
2666             $conf = $res->{snapshots}->{$section} = {};
2667             next;
2668         }
2669
2670         if ($line =~ m/^\#(.*)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($1) . "\n";
2673             next;
2674         }
2675
2676         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2677             $descr = '' if !defined($descr);
2678             $descr .= PVE::Tools::decode_text($2);
2679         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2680             $conf->{snapstate} = $1;
2681         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2682             my $key = $1;
2683             my $value = $2;
2684             $conf->{$key} = $value;
2685         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2686             my $value = $1;
2687             if ($section eq 'pending') {
2688                 $conf->{delete} = $value; # we parse this later
2689             } else {
2690                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2691             }
2692         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2693             my $key = $1;
2694             my $value = $2;
2695             eval { $value = check_type($key, $value); };
2696             if ($@) {
2697                 warn "vm $vmid - unable to parse value of '$key' - $@";
2698             } else {
2699                 $key = 'ide2' if $key eq 'cdrom';
2700                 my $fmt = $confdesc->{$key}->{format};
2701                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2702                     my $v = parse_drive($key, $value);
2703                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2704                         $v->{file} = $volid;
2705                         $value = print_drive($vmid, $v);
2706                     } else {
2707                         warn "vm $vmid - unable to parse value of '$key'\n";
2708                         next;
2709                     }
2710                 }
2711
2712                 $conf->{$key} = $value;
2713             }
2714         }
2715     }
2716
2717     if (defined($descr)) {
2718         $descr =~ s/\s+$//;
2719         $conf->{description} = $descr;
2720     }
2721     delete $res->{snapstate}; # just to be sure
2722
2723     return $res;
2724 }
2725
2726 sub write_vm_config {
2727     my ($filename, $conf) = @_;
2728
2729     delete $conf->{snapstate}; # just to be sure
2730
2731     if ($conf->{cdrom}) {
2732         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2733         $conf->{ide2} = $conf->{cdrom};
2734         delete $conf->{cdrom};
2735     }
2736
2737     # we do not use 'smp' any longer
2738     if ($conf->{sockets}) {
2739         delete $conf->{smp};
2740     } elsif ($conf->{smp}) {
2741         $conf->{sockets} = $conf->{smp};
2742         delete $conf->{cores};
2743         delete $conf->{smp};
2744     }
2745
2746     my $used_volids = {};
2747
2748     my $cleanup_config = sub {
2749         my ($cref, $pending, $snapname) = @_;
2750
2751         foreach my $key (keys %$cref) {
2752             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2753                 $key eq 'snapstate' || $key eq 'pending';
2754             my $value = $cref->{$key};
2755             if ($key eq 'delete') {
2756                 die "propertry 'delete' is only allowed in [PENDING]\n"
2757                     if !$pending;
2758                 # fixme: check syntax?
2759                 next;
2760             }
2761             eval { $value = check_type($key, $value); };
2762             die "unable to parse value of '$key' - $@" if $@;
2763
2764             $cref->{$key} = $value;
2765
2766             if (!$snapname && is_valid_drivename($key)) {
2767                 my $drive = parse_drive($key, $value);
2768                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2769             }
2770         }
2771     };
2772
2773     &$cleanup_config($conf);
2774
2775     &$cleanup_config($conf->{pending}, 1);
2776
2777     foreach my $snapname (keys %{$conf->{snapshots}}) {
2778         die "internal error" if $snapname eq 'pending';
2779         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2780     }
2781
2782     # remove 'unusedX' settings if we re-add a volume
2783     foreach my $key (keys %$conf) {
2784         my $value = $conf->{$key};
2785         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2786             delete $conf->{$key};
2787         }
2788     }
2789
2790     my $generate_raw_config = sub {
2791         my ($conf, $pending) = @_;
2792
2793         my $raw = '';
2794
2795         # add description as comment to top of file
2796         if (defined(my $descr = $conf->{description})) {
2797             if ($descr) {
2798                 foreach my $cl (split(/\n/, $descr)) {
2799                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2800                 }
2801             } else {
2802                 $raw .= "#\n" if $pending;
2803             }
2804         }
2805
2806         foreach my $key (sort keys %$conf) {
2807             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2808             $raw .= "$key: $conf->{$key}\n";
2809         }
2810         return $raw;
2811     };
2812
2813     my $raw = &$generate_raw_config($conf);
2814
2815     if (scalar(keys %{$conf->{pending}})){
2816         $raw .= "\n[PENDING]\n";
2817         $raw .= &$generate_raw_config($conf->{pending}, 1);
2818     }
2819
2820     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2821         $raw .= "\n[$snapname]\n";
2822         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2823     }
2824
2825     return $raw;
2826 }
2827
2828 sub load_defaults {
2829
2830     my $res = {};
2831
2832     # we use static defaults from our JSON schema configuration
2833     foreach my $key (keys %$confdesc) {
2834         if (defined(my $default = $confdesc->{$key}->{default})) {
2835             $res->{$key} = $default;
2836         }
2837     }
2838
2839     return $res;
2840 }
2841
2842 sub config_list {
2843     my $vmlist = PVE::Cluster::get_vmlist();
2844     my $res = {};
2845     return $res if !$vmlist || !$vmlist->{ids};
2846     my $ids = $vmlist->{ids};
2847
2848     foreach my $vmid (keys %$ids) {
2849         my $d = $ids->{$vmid};
2850         next if !$d->{node} || $d->{node} ne $nodename;
2851         next if !$d->{type} || $d->{type} ne 'qemu';
2852         $res->{$vmid}->{exists} = 1;
2853     }
2854     return $res;
2855 }
2856
2857 # test if VM uses local resources (to prevent migration)
2858 sub check_local_resources {
2859     my ($conf, $noerr) = @_;
2860
2861     my @loc_res = ();
2862
2863     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2864     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2865
2866     push @loc_res, "ivshmem" if $conf->{ivshmem};
2867
2868     foreach my $k (keys %$conf) {
2869         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2870         # sockets are safe: they will recreated be on the target side post-migrate
2871         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2872         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2873     }
2874
2875     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2876
2877     return \@loc_res;
2878 }
2879
2880 # check if used storages are available on all nodes (use by migrate)
2881 sub check_storage_availability {
2882     my ($storecfg, $conf, $node) = @_;
2883
2884     foreach_drive($conf, sub {
2885         my ($ds, $drive) = @_;
2886
2887         my $volid = $drive->{file};
2888         return if !$volid;
2889
2890         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2891         return if !$sid;
2892
2893         # check if storage is available on both nodes
2894         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2895         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2896    });
2897 }
2898
2899 # list nodes where all VM images are available (used by has_feature API)
2900 sub shared_nodes {
2901     my ($conf, $storecfg) = @_;
2902
2903     my $nodelist = PVE::Cluster::get_nodelist();
2904     my $nodehash = { map { $_ => 1 } @$nodelist };
2905     my $nodename = PVE::INotify::nodename();
2906
2907     foreach_drive($conf, sub {
2908         my ($ds, $drive) = @_;
2909
2910         my $volid = $drive->{file};
2911         return if !$volid;
2912
2913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2914         if ($storeid) {
2915             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2916             if ($scfg->{disable}) {
2917                 $nodehash = {};
2918             } elsif (my $avail = $scfg->{nodes}) {
2919                 foreach my $node (keys %$nodehash) {
2920                     delete $nodehash->{$node} if !$avail->{$node};
2921                 }
2922             } elsif (!$scfg->{shared}) {
2923                 foreach my $node (keys %$nodehash) {
2924                     delete $nodehash->{$node} if $node ne $nodename
2925                 }
2926             }
2927         }
2928     });
2929
2930     return $nodehash
2931 }
2932
2933 sub check_local_storage_availability {
2934     my ($conf, $storecfg) = @_;
2935
2936     my $nodelist = PVE::Cluster::get_nodelist();
2937     my $nodehash = { map { $_ => {} } @$nodelist };
2938
2939     foreach_drive($conf, sub {
2940         my ($ds, $drive) = @_;
2941
2942         my $volid = $drive->{file};
2943         return if !$volid;
2944
2945         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2946         if ($storeid) {
2947             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2948
2949             if ($scfg->{disable}) {
2950                 foreach my $node (keys %$nodehash) {
2951                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2952                 }
2953             } elsif (my $avail = $scfg->{nodes}) {
2954                 foreach my $node (keys %$nodehash) {
2955                     if (!$avail->{$node}) {
2956                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2957                     }
2958                 }
2959             }
2960         }
2961     });
2962
2963     foreach my $node (values %$nodehash) {
2964         if (my $unavail = $node->{unavailable_storages}) {
2965             $node->{unavailable_storages} = [ sort keys %$unavail ];
2966         }
2967     }
2968
2969     return $nodehash
2970 }
2971
2972 sub check_cmdline {
2973     my ($pidfile, $pid) = @_;
2974
2975     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2976     if (defined($fh)) {
2977         my $line = <$fh>;
2978         $fh->close;
2979         return undef if !$line;
2980         my @param = split(/\0/, $line);
2981
2982         my $cmd = $param[0];
2983         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2984
2985         for (my $i = 0; $i < scalar (@param); $i++) {
2986             my $p = $param[$i];
2987             next if !$p;
2988             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2989                 my $p = $param[$i+1];
2990                 return 1 if $p && ($p eq $pidfile);
2991                 return undef;
2992             }
2993         }
2994     }
2995     return undef;
2996 }
2997
2998 sub check_running {
2999     my ($vmid, $nocheck, $node) = @_;
3000
3001     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3002
3003     die "unable to find configuration file for VM $vmid - no such machine\n"
3004         if !$nocheck && ! -f $filename;
3005
3006     my $pidfile = pidfile_name($vmid);
3007
3008     if (my $fd = IO::File->new("<$pidfile")) {
3009         my $st = stat($fd);
3010         my $line = <$fd>;
3011         close($fd);
3012
3013         my $mtime = $st->mtime;
3014         if ($mtime > time()) {
3015             warn "file '$filename' modified in future\n";
3016         }
3017
3018         if ($line =~ m/^(\d+)$/) {
3019             my $pid = $1;
3020             if (check_cmdline($pidfile, $pid)) {
3021                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3022                     return $pid;
3023                 }
3024             }
3025         }
3026     }
3027
3028     return undef;
3029 }
3030
3031 sub vzlist {
3032
3033     my $vzlist = config_list();
3034
3035     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3036
3037     while (defined(my $de = $fd->read)) {
3038         next if $de !~ m/^(\d+)\.pid$/;
3039         my $vmid = $1;
3040         next if !defined($vzlist->{$vmid});
3041         if (my $pid = check_running($vmid)) {
3042             $vzlist->{$vmid}->{pid} = $pid;
3043         }
3044     }
3045
3046     return $vzlist;
3047 }
3048
3049 sub disksize {
3050     my ($storecfg, $conf) = @_;
3051
3052     my $bootdisk = $conf->{bootdisk};
3053     return undef if !$bootdisk;
3054     return undef if !is_valid_drivename($bootdisk);
3055
3056     return undef if !$conf->{$bootdisk};
3057
3058     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3059     return undef if !defined($drive);
3060
3061     return undef if drive_is_cdrom($drive);
3062
3063     my $volid = $drive->{file};
3064     return undef if !$volid;
3065
3066     return $drive->{size};
3067 }
3068
3069 our $vmstatus_return_properties = {
3070     vmid => get_standard_option('pve-vmid'),
3071     status => {
3072         description => "Qemu process status.",
3073         type => 'string',
3074         enum => ['stopped', 'running'],
3075     },
3076     maxmem => {
3077         description => "Maximum memory in bytes.",
3078         type => 'integer',
3079         optional => 1,
3080         renderer => 'bytes',
3081     },
3082     maxdisk => {
3083         description => "Root disk size in bytes.",
3084         type => 'integer',
3085         optional => 1,
3086         renderer => 'bytes',
3087     },
3088     name => {
3089         description => "VM name.",
3090         type => 'string',
3091         optional => 1,
3092     },
3093     qmpstatus => {
3094         description => "Qemu QMP agent status.",
3095         type => 'string',
3096         optional => 1,
3097     },
3098     pid => {
3099         description => "PID of running qemu process.",
3100         type => 'integer',
3101         optional => 1,
3102     },
3103     uptime => {
3104         description => "Uptime.",
3105         type => 'integer',
3106         optional => 1,
3107         renderer => 'duration',
3108     },
3109     cpus => {
3110         description => "Maximum usable CPUs.",
3111         type => 'number',
3112         optional => 1,
3113     },
3114     lock => {
3115         description => "The current config lock, if any.",
3116         type => 'string',
3117         optional => 1,
3118     }
3119 };
3120
3121 my $last_proc_pid_stat;
3122
3123 # get VM status information
3124 # This must be fast and should not block ($full == false)
3125 # We only query KVM using QMP if $full == true (this can be slow)
3126 sub vmstatus {
3127     my ($opt_vmid, $full) = @_;
3128
3129     my $res = {};
3130
3131     my $storecfg = PVE::Storage::config();
3132
3133     my $list = vzlist();
3134     my $defaults = load_defaults();
3135
3136     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3137
3138     my $cpucount = $cpuinfo->{cpus} || 1;
3139
3140     foreach my $vmid (keys %$list) {
3141         next if $opt_vmid && ($vmid ne $opt_vmid);
3142
3143         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3144         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3145
3146         my $d = { vmid => $vmid };
3147         $d->{pid} = $list->{$vmid}->{pid};
3148
3149         # fixme: better status?
3150         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3151
3152         my $size = disksize($storecfg, $conf);
3153         if (defined($size)) {
3154             $d->{disk} = 0; # no info available
3155             $d->{maxdisk} = $size;
3156         } else {
3157             $d->{disk} = 0;
3158             $d->{maxdisk} = 0;
3159         }
3160
3161         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3162             * ($conf->{cores} || $defaults->{cores});
3163         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3164         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3165
3166         $d->{name} = $conf->{name} || "VM $vmid";
3167         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3168             : $defaults->{memory}*(1024*1024);
3169
3170         if ($conf->{balloon}) {
3171             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3172             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3173                 : $defaults->{shares};
3174         }
3175
3176         $d->{uptime} = 0;
3177         $d->{cpu} = 0;
3178         $d->{mem} = 0;
3179
3180         $d->{netout} = 0;
3181         $d->{netin} = 0;
3182
3183         $d->{diskread} = 0;
3184         $d->{diskwrite} = 0;
3185
3186         $d->{template} = PVE::QemuConfig->is_template($conf);
3187
3188         $d->{serial} = 1 if conf_has_serial($conf);
3189         $d->{lock} = $conf->{lock} if $conf->{lock};
3190
3191         $res->{$vmid} = $d;
3192     }
3193
3194     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3195     foreach my $dev (keys %$netdev) {
3196         next if $dev !~ m/^tap([1-9]\d*)i/;
3197         my $vmid = $1;
3198         my $d = $res->{$vmid};
3199         next if !$d;
3200
3201         $d->{netout} += $netdev->{$dev}->{receive};
3202         $d->{netin} += $netdev->{$dev}->{transmit};
3203
3204         if ($full) {
3205             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3206             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3207         }
3208
3209     }
3210
3211     my $ctime = gettimeofday;
3212
3213     foreach my $vmid (keys %$list) {
3214
3215         my $d = $res->{$vmid};
3216         my $pid = $d->{pid};
3217         next if !$pid;
3218
3219         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3220         next if !$pstat; # not running
3221
3222         my $used = $pstat->{utime} + $pstat->{stime};
3223
3224         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3225
3226         if ($pstat->{vsize}) {
3227             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3228         }
3229
3230         my $old = $last_proc_pid_stat->{$pid};
3231         if (!$old) {
3232             $last_proc_pid_stat->{$pid} = {
3233                 time => $ctime,
3234                 used => $used,
3235                 cpu => 0,
3236             };
3237             next;
3238         }
3239
3240         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3241
3242         if ($dtime > 1000) {
3243             my $dutime = $used -  $old->{used};
3244
3245             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3246             $last_proc_pid_stat->{$pid} = {
3247                 time => $ctime,
3248                 used => $used,
3249                 cpu => $d->{cpu},
3250             };
3251         } else {
3252             $d->{cpu} = $old->{cpu};
3253         }
3254     }
3255
3256     return $res if !$full;
3257
3258     my $qmpclient = PVE::QMPClient->new();
3259
3260     my $ballooncb = sub {
3261         my ($vmid, $resp) = @_;
3262
3263         my $info = $resp->{'return'};
3264         return if !$info->{max_mem};
3265
3266         my $d = $res->{$vmid};
3267
3268         # use memory assigned to VM
3269         $d->{maxmem} = $info->{max_mem};
3270         $d->{balloon} = $info->{actual};
3271
3272         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3273             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3274             $d->{freemem} = $info->{free_mem};
3275         }
3276
3277         $d->{ballooninfo} = $info;
3278     };
3279
3280     my $blockstatscb = sub {
3281         my ($vmid, $resp) = @_;
3282         my $data = $resp->{'return'} || [];
3283         my $totalrdbytes = 0;
3284         my $totalwrbytes = 0;
3285
3286         for my $blockstat (@$data) {
3287             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3288             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3289
3290             $blockstat->{device} =~ s/drive-//;
3291             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3292         }
3293         $res->{$vmid}->{diskread} = $totalrdbytes;
3294         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3295     };
3296
3297     my $statuscb = sub {
3298         my ($vmid, $resp) = @_;
3299
3300         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3301         # this fails if ballon driver is not loaded, so this must be
3302         # the last commnand (following command are aborted if this fails).
3303         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3304
3305         my $status = 'unknown';
3306         if (!defined($status = $resp->{'return'}->{status})) {
3307             warn "unable to get VM status\n";
3308             return;
3309         }
3310
3311         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3312     };
3313
3314     foreach my $vmid (keys %$list) {
3315         next if $opt_vmid && ($vmid ne $opt_vmid);
3316         next if !$res->{$vmid}->{pid}; # not running
3317         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3318     }
3319
3320     $qmpclient->queue_execute(undef, 2);
3321
3322     foreach my $vmid (keys %$list) {
3323         next if $opt_vmid && ($vmid ne $opt_vmid);
3324         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3325     }
3326
3327     return $res;
3328 }
3329
3330 sub foreach_drive {
3331     my ($conf, $func, @param) = @_;
3332
3333     foreach my $ds (valid_drive_names()) {
3334         next if !defined($conf->{$ds});
3335
3336         my $drive = parse_drive($ds, $conf->{$ds});
3337         next if !$drive;
3338
3339         &$func($ds, $drive, @param);
3340     }
3341 }
3342
3343 sub foreach_volid {
3344     my ($conf, $func, @param) = @_;
3345
3346     my $volhash = {};
3347
3348     my $test_volid = sub {
3349         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3350
3351         return if !$volid;
3352
3353         $volhash->{$volid}->{cdrom} //= 1;
3354         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3355
3356         $volhash->{$volid}->{replicate} //= 0;
3357         $volhash->{$volid}->{replicate} = 1 if $replicate;
3358
3359         $volhash->{$volid}->{shared} //= 0;
3360         $volhash->{$volid}->{shared} = 1 if $shared;
3361
3362         $volhash->{$volid}->{referenced_in_config} //= 0;
3363         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3364
3365         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3366             if defined($snapname);
3367         $volhash->{$volid}->{size} = $size if $size;
3368     };
3369
3370     foreach_drive($conf, sub {
3371         my ($ds, $drive) = @_;
3372         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3373     });
3374
3375     foreach my $snapname (keys %{$conf->{snapshots}}) {
3376         my $snap = $conf->{snapshots}->{$snapname};
3377         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3378         foreach_drive($snap, sub {
3379             my ($ds, $drive) = @_;
3380             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3381         });
3382     }
3383
3384     foreach my $volid (keys %$volhash) {
3385         &$func($volid, $volhash->{$volid}, @param);
3386     }
3387 }
3388
3389 sub conf_has_serial {
3390     my ($conf) = @_;
3391
3392     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3393         if ($conf->{"serial$i"}) {
3394             return 1;
3395         }
3396     }
3397
3398     return 0;
3399 }
3400
3401 sub vga_conf_has_spice {
3402     my ($vga) = @_;
3403
3404     my $vgaconf = parse_vga($vga);
3405     my $vgatype = $vgaconf->{type};
3406     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3407
3408     return $1 || 1;
3409 }
3410
3411 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3412 sub get_host_arch() {
3413     $host_arch = (POSIX::uname())[4] if !$host_arch;
3414     return $host_arch;
3415 }
3416
3417 sub is_native($) {
3418     my ($arch) = @_;
3419     return get_host_arch() eq $arch;
3420 }
3421
3422 my $default_machines = {
3423     x86_64 => 'pc',
3424     aarch64 => 'virt',
3425 };
3426
3427 sub get_basic_machine_info {
3428     my ($conf, $forcemachine) = @_;
3429
3430     my $arch = $conf->{arch} // get_host_arch();
3431     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3432     return ($arch, $machine);
3433 }
3434
3435 sub get_ovmf_files($) {
3436     my ($arch) = @_;
3437
3438     my $ovmf = $OVMF->{$arch}
3439         or die "no OVMF images known for architecture '$arch'\n";
3440
3441     return @$ovmf;
3442 }
3443
3444 my $Arch2Qemu = {
3445     aarch64 => '/usr/bin/qemu-system-aarch64',
3446     x86_64 => '/usr/bin/qemu-system-x86_64',
3447 };
3448 sub get_command_for_arch($) {
3449     my ($arch) = @_;
3450     return '/usr/bin/kvm' if is_native($arch);
3451
3452     my $cmd = $Arch2Qemu->{$arch}
3453         or die "don't know how to emulate architecture '$arch'\n";
3454     return $cmd;
3455 }
3456
3457 sub get_cpu_options {
3458     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3459
3460     my $cpuFlags = [];
3461     my $ostype = $conf->{ostype};
3462
3463     my $cpu = $kvm ? "kvm64" : "qemu64";
3464     if ($arch eq 'aarch64') {
3465         $cpu = 'cortex-a57';
3466     }
3467     my $hv_vendor_id;
3468     if (my $cputype = $conf->{cpu}) {
3469         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3470             or die "Cannot parse cpu description: $cputype\n";
3471         $cpu = $cpuconf->{cputype};
3472         $kvm_off = 1 if $cpuconf->{hidden};
3473         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3474
3475         if (defined(my $flags = $cpuconf->{flags})) {
3476             push @$cpuFlags, split(";", $flags);
3477         }
3478     }
3479
3480     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3481
3482     push @$cpuFlags , '-x2apic'
3483         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3484
3485     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3486
3487     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3488
3489     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3490
3491         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3492         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3493     }
3494
3495     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3496
3497     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3498
3499     push @$cpuFlags, 'kvm=off' if $kvm_off;
3500
3501     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3502         push @$cpuFlags, "vendor=${cpu_vendor}"
3503             if $cpu_vendor ne 'default';
3504     } elsif ($arch ne 'aarch64') {
3505         die "internal error"; # should not happen
3506     }
3507
3508     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3509
3510     return ('-cpu', $cpu);
3511 }
3512
3513 sub config_to_command {
3514     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3515
3516     my $cmd = [];
3517     my $globalFlags = [];
3518     my $machineFlags = [];
3519     my $rtcFlags = [];
3520     my $devices = [];
3521     my $pciaddr = '';
3522     my $bridges = {};
3523     my $kvmver = kvm_user_version();
3524     my $vernum = 0; # unknown
3525     my $ostype = $conf->{ostype};
3526     my $winversion = windows_version($ostype);
3527     my $kvm = $conf->{kvm};
3528
3529     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3530     $kvm //= 1 if is_native($arch);
3531
3532     if ($kvm) {
3533         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3534             if !defined kvm_version();
3535     }
3536
3537     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3538         $vernum = $1*1000000+$2*1000;
3539     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3540         $vernum = $1*1000000+$2*1000+$3;
3541     }
3542
3543     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3544
3545     my $have_ovz = -f '/proc/vz/vestat';
3546
3547     my $q35 = machine_type_is_q35($conf);
3548     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3549     my $use_old_bios_files = undef;
3550     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3551
3552     my $cpuunits = defined($conf->{cpuunits}) ?
3553             $conf->{cpuunits} : $defaults->{cpuunits};
3554
3555     push @$cmd, get_command_for_arch($arch);
3556
3557     push @$cmd, '-id', $vmid;
3558
3559     my $vmname = $conf->{name} || "vm$vmid";
3560
3561     push @$cmd, '-name', $vmname;
3562
3563     my $use_virtio = 0;
3564
3565     my $qmpsocket = qmp_socket($vmid);
3566     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3567     push @$cmd, '-mon', "chardev=qmp,mode=control";
3568
3569     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3570         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3571         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3572     }
3573
3574     push @$cmd, '-pidfile' , pidfile_name($vmid);
3575
3576     push @$cmd, '-daemonize';
3577
3578     if ($conf->{smbios1}) {
3579         my $smbios_conf = parse_smbios1($conf->{smbios1});
3580         if ($smbios_conf->{base64}) {
3581             # Do not pass base64 flag to qemu
3582             delete $smbios_conf->{base64};
3583             my $smbios_string = "";
3584             foreach my $key (keys %$smbios_conf) {
3585                 my $value;
3586                 if ($key eq "uuid") {
3587                     $value = $smbios_conf->{uuid}
3588                 } else {
3589                     $value = decode_base64($smbios_conf->{$key});
3590                 }
3591                 # qemu accepts any binary data, only commas need escaping by double comma
3592                 $value =~ s/,/,,/g;
3593                 $smbios_string .= "," . $key . "=" . $value if $value;
3594             }
3595             push @$cmd, '-smbios', "type=1" . $smbios_string;
3596         } else {
3597             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3598         }
3599     }
3600
3601     if ($conf->{vmgenid}) {
3602         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3603     }
3604
3605     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3606     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3607         die "uefi base image not found\n" if ! -f $ovmf_code;
3608
3609         my $path;
3610         my $format;
3611         if (my $efidisk = $conf->{efidisk0}) {
3612             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3613             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3614             $format = $d->{format};
3615             if ($storeid) {
3616                 $path = PVE::Storage::path($storecfg, $d->{file});
3617                 if (!defined($format)) {
3618                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3619                     $format = qemu_img_format($scfg, $volname);
3620                 }
3621             } else {
3622                 $path = $d->{file};
3623                 die "efidisk format must be specified\n"
3624                     if !defined($format);
3625             }
3626         } else {
3627             warn "no efidisk configured! Using temporary efivars disk.\n";
3628             $path = "/tmp/$vmid-ovmf.fd";
3629             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3630             $format = 'raw';
3631         }
3632
3633         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3634         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3635     }
3636
3637     # load q35 config
3638     if ($q35) {
3639         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3640         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3641             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3642         } else {
3643             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3644         }
3645     }
3646
3647     # add usb controllers
3648     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3649     push @$devices, @usbcontrollers if @usbcontrollers;
3650     my $vga = parse_vga($conf->{vga});
3651
3652     my $qxlnum = vga_conf_has_spice($conf->{vga});
3653     $vga->{type} = 'qxl' if $qxlnum;
3654
3655     if (!$vga->{type}) {
3656         if ($arch eq 'aarch64') {
3657             $vga->{type} = 'virtio';
3658         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3659             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3660         } else {
3661             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3662         }
3663     }
3664
3665     # enable absolute mouse coordinates (needed by vnc)
3666     my $tablet;
3667     if (defined($conf->{tablet})) {
3668         $tablet = $conf->{tablet};
3669     } else {
3670         $tablet = $defaults->{tablet};
3671         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3672         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3673     }
3674
3675     if ($tablet) {
3676         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3677         my $kbd = print_keyboarddevice_full($conf, $arch);
3678         push @$devices, '-device', $kbd if defined($kbd);
3679     }
3680
3681     my $kvm_off = 0;
3682     my $gpu_passthrough;
3683
3684     # host pci devices
3685     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3686         my $d = parse_hostpci($conf->{"hostpci$i"});
3687         next if !$d;
3688
3689         my $pcie = $d->{pcie};
3690         if ($pcie) {
3691             die "q35 machine model is not enabled" if !$q35;
3692             # win7 wants to have the pcie devices directly on the pcie bus
3693             # instead of in the root port
3694             if ($winversion == 7) {
3695                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3696             } else {
3697                 $pciaddr = print_pcie_addr("hostpci$i");
3698             }
3699         } else {
3700             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3701         }
3702
3703         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3704         my $romfile = $d->{romfile};
3705
3706         my $xvga = '';
3707         if ($d->{'x-vga'}) {
3708             $xvga = ',x-vga=on';
3709             $kvm_off = 1;
3710             $vga->{type} = 'none' if !defined($conf->{vga});
3711             $gpu_passthrough = 1;
3712
3713             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3714                 $xvga = "";
3715             }
3716         }
3717         my $pcidevices = $d->{pciid};
3718         my $multifunction = 1 if @$pcidevices > 1;
3719         my $sysfspath;
3720         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3721             my $id = $pcidevices->[0]->{id};
3722             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3723             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3724         } elsif ($d->{mdev}) {
3725             warn "ignoring mediated device with multifunction device\n";
3726         }
3727
3728         my $j=0;
3729         foreach my $pcidevice (@$pcidevices) {
3730
3731             my $id = "hostpci$i";
3732             $id .= ".$j" if $multifunction;
3733             my $addr = $pciaddr;
3734             $addr .= ".$j" if $multifunction;
3735             my $devicestr = "vfio-pci";
3736             if ($sysfspath) {
3737                 $devicestr .= ",sysfsdev=$sysfspath";
3738             } else {
3739                 $devicestr .= ",host=$pcidevice->{id}";
3740             }
3741             $devicestr .= ",id=$id$addr";
3742
3743             if($j == 0){
3744                 $devicestr .= "$rombar$xvga";
3745                 $devicestr .= ",multifunction=on" if $multifunction;
3746                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3747             }
3748
3749             push @$devices, '-device', $devicestr;
3750             $j++;
3751         }
3752     }
3753
3754     # usb devices
3755     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3756     push @$devices, @usbdevices if @usbdevices;
3757     # serial devices
3758     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3759         if (my $path = $conf->{"serial$i"}) {
3760             if ($path eq 'socket') {
3761                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3762                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3763                 # On aarch64, serial0 is the UART device. Qemu only allows
3764                 # connecting UART devices via the '-serial' command line, as
3765                 # the device has a fixed slot on the hardware...
3766                 if ($arch eq 'aarch64' && $i == 0) {
3767                     push @$devices, '-serial', "chardev:serial$i";
3768                 } else {
3769                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3770                 }
3771             } else {
3772                 die "no such serial device\n" if ! -c $path;
3773                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3775             }
3776         }
3777     }
3778
3779     # parallel devices
3780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3781         if (my $path = $conf->{"parallel$i"}) {
3782             die "no such parallel device\n" if ! -c $path;
3783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3786         }
3787     }
3788
3789     if (my $audiodevice = $conf->{audio0}) {
3790         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3791
3792         if ($audiodevice eq 'AC97') {
3793             push @$devices, '-device', "AC97,id=sound0${audiopciaddr}";
3794         } elsif ($audiodevice =~ /intel\-hda$/) {
3795             push @$devices, '-device', "${audiodevice},id=sound5${audiopciaddr}";
3796             push @$devices, '-device', "hda-micro,id=sound5-codec0,bus=sound5.0,cad=0";
3797             push @$devices, '-device', "hda-duplex,id=sound5-codec1,bus=sound5.0,cad=1";
3798         } else {
3799             die "unkown audio device '$audiodevice', implement me!";
3800         }
3801     }
3802
3803     my $sockets = 1;
3804     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3805     $sockets = $conf->{sockets} if  $conf->{sockets};
3806
3807     my $cores = $conf->{cores} || 1;
3808
3809     my $maxcpus = $sockets * $cores;
3810
3811     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3812
3813     my $allowed_vcpus = $cpuinfo->{cpus};
3814
3815     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3816         if ($allowed_vcpus < $maxcpus);
3817
3818     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3819
3820         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3821         for (my $i = 2; $i <= $vcpus; $i++)  {
3822             my $cpustr = print_cpu_device($conf,$i);
3823             push @$cmd, '-device', $cpustr;
3824         }
3825
3826     } else {
3827
3828         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3829     }
3830     push @$cmd, '-nodefaults';
3831
3832     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3833
3834     my $bootindex_hash = {};
3835     my $i = 1;
3836     foreach my $o (split(//, $bootorder)) {
3837         $bootindex_hash->{$o} = $i*100;
3838         $i++;
3839     }
3840
3841     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3842
3843     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3844
3845     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3846
3847     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3848         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3849         my $socket = vnc_socket($vmid);
3850         push @$cmd,  '-vnc', "unix:$socket,password";
3851     } else {
3852         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3853         push @$cmd, '-nographic';
3854     }
3855
3856     # time drift fix
3857     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3858
3859     my $useLocaltime = $conf->{localtime};
3860
3861     if ($winversion >= 5) { # windows
3862         $useLocaltime = 1 if !defined($conf->{localtime});
3863
3864         # use time drift fix when acpi is enabled
3865         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3866             $tdf = 1 if !defined($conf->{tdf});
3867         }
3868     }
3869
3870     if ($winversion >= 6) {
3871         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3872         push @$cmd, '-no-hpet';
3873     }
3874
3875     push @$rtcFlags, 'driftfix=slew' if $tdf;
3876
3877     if (!$kvm) {
3878         push @$machineFlags, 'accel=tcg';
3879     }
3880
3881     if ($machine_type) {
3882         push @$machineFlags, "type=${machine_type}";
3883     }
3884
3885     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3886         push @$rtcFlags, "base=$conf->{startdate}";
3887     } elsif ($useLocaltime) {
3888         push @$rtcFlags, 'base=localtime';
3889     }
3890
3891     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3892
3893     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3894
3895     push @$cmd, '-S' if $conf->{freeze};
3896
3897     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3898
3899     if (parse_guest_agent($conf)->{enabled}) {
3900         my $qgasocket = qmp_socket($vmid, 1);
3901         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3902         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3903         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3904         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3905     }
3906
3907     my $spice_port;
3908
3909     if ($qxlnum) {
3910         if ($qxlnum > 1) {
3911             if ($winversion){
3912                 for(my $i = 1; $i < $qxlnum; $i++){
3913                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3914                 }
3915             } else {
3916                 # assume other OS works like Linux
3917                 my ($ram, $vram) = ("134217728", "67108864");
3918                 if ($vga->{memory}) {
3919                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3920                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3921                 }
3922                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3923                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3924             }
3925         }
3926
3927         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3928
3929         my $nodename = PVE::INotify::nodename();
3930         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3931         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3932         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3933         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3934         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3935
3936         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3937
3938         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3939         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3940         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3941     }
3942
3943     # enable balloon by default, unless explicitly disabled
3944     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3945         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3946         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3947     }
3948
3949     if ($conf->{watchdog}) {
3950         my $wdopts = parse_watchdog($conf->{watchdog});
3951         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3952         my $watchdog = $wdopts->{model} || 'i6300esb';
3953         push @$devices, '-device', "$watchdog$pciaddr";
3954         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3955     }
3956
3957     my $vollist = [];
3958     my $scsicontroller = {};
3959     my $ahcicontroller = {};
3960     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3961
3962     # Add iscsi initiator name if available
3963     if (my $initiator = get_initiator_name()) {
3964         push @$devices, '-iscsi', "initiator-name=$initiator";
3965     }
3966
3967     foreach_drive($conf, sub {
3968         my ($ds, $drive) = @_;
3969
3970         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3971             push @$vollist, $drive->{file};
3972         }
3973
3974         # ignore efidisk here, already added in bios/fw handling code above
3975         return if $drive->{interface} eq 'efidisk';
3976
3977         $use_virtio = 1 if $ds =~ m/^virtio/;
3978
3979         if (drive_is_cdrom ($drive)) {
3980             if ($bootindex_hash->{d}) {
3981                 $drive->{bootindex} = $bootindex_hash->{d};
3982                 $bootindex_hash->{d} += 1;
3983             }
3984         } else {
3985             if ($bootindex_hash->{c}) {
3986                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3987                 $bootindex_hash->{c} += 1;
3988             }
3989         }
3990
3991         if($drive->{interface} eq 'virtio'){
3992            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3993         }
3994
3995         if ($drive->{interface} eq 'scsi') {
3996
3997             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3998
3999             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4000             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4001
4002             my $iothread = '';
4003             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4004                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4005                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4006             } elsif ($drive->{iothread}) {
4007                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4008             }
4009
4010             my $queues = '';
4011             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4012                 $queues = ",num_queues=$drive->{queues}";
4013             }
4014
4015             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4016             $scsicontroller->{$controller}=1;
4017         }
4018
4019         if ($drive->{interface} eq 'sata') {
4020            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4021            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4022            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4023            $ahcicontroller->{$controller}=1;
4024         }
4025
4026         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4027         push @$devices, '-drive',$drive_cmd;
4028         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4029     });
4030
4031     for (my $i = 0; $i < $MAX_NETS; $i++) {
4032          next if !$conf->{"net$i"};
4033          my $d = parse_net($conf->{"net$i"});
4034          next if !$d;
4035
4036          $use_virtio = 1 if $d->{model} eq 'virtio';
4037
4038          if ($bootindex_hash->{n}) {
4039             $d->{bootindex} = $bootindex_hash->{n};
4040             $bootindex_hash->{n} += 1;
4041          }
4042
4043          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4044          push @$devices, '-netdev', $netdevfull;
4045
4046          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4047          push @$devices, '-device', $netdevicefull;
4048     }
4049
4050     if ($conf->{ivshmem}) {
4051         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4052
4053         my $bus;
4054         if ($q35) {
4055             $bus = print_pcie_addr("ivshmem");
4056         } else {
4057             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4058         }
4059
4060         my $ivshmem_name = $ivshmem->{name} // $vmid;
4061         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4062
4063         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4064         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4065     }
4066
4067     if (!$q35) {
4068         # add pci bridges
4069         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4070            $bridges->{1} = 1;
4071            $bridges->{2} = 1;
4072         }
4073
4074         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4075
4076         while (my ($k, $v) = each %$bridges) {
4077             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4078             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4079         }
4080     }
4081
4082     push @$cmd, @$devices;
4083     push @$cmd, '-rtc', join(',', @$rtcFlags)
4084         if scalar(@$rtcFlags);
4085     push @$cmd, '-machine', join(',', @$machineFlags)
4086         if scalar(@$machineFlags);
4087     push @$cmd, '-global', join(',', @$globalFlags)
4088         if scalar(@$globalFlags);
4089
4090     if (my $vmstate = $conf->{vmstate}) {
4091         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4092         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4093         push @$cmd, '-loadstate', $statepath;
4094     }
4095
4096     # add custom args
4097     if ($conf->{args}) {
4098         my $aa = PVE::Tools::split_args($conf->{args});
4099         push @$cmd, @$aa;
4100     }
4101
4102     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4103 }
4104
4105 sub vnc_socket {
4106     my ($vmid) = @_;
4107     return "${var_run_tmpdir}/$vmid.vnc";
4108 }
4109
4110 sub spice_port {
4111     my ($vmid) = @_;
4112
4113     my $res = vm_mon_cmd($vmid, 'query-spice');
4114
4115     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4116 }
4117
4118 sub qmp_socket {
4119     my ($vmid, $qga, $name) = @_;
4120     my $sockettype = $qga ? 'qga' : 'qmp';
4121     my $ext = $name ? '-'.$name : '';
4122     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4123 }
4124
4125 sub pidfile_name {
4126     my ($vmid) = @_;
4127     return "${var_run_tmpdir}/$vmid.pid";
4128 }
4129
4130 sub vm_devices_list {
4131     my ($vmid) = @_;
4132
4133     my $res = vm_mon_cmd($vmid, 'query-pci');
4134     my $devices_to_check = [];
4135     my $devices = {};
4136     foreach my $pcibus (@$res) {
4137         push @$devices_to_check, @{$pcibus->{devices}},
4138     }
4139
4140     while (@$devices_to_check) {
4141         my $to_check = [];
4142         for my $d (@$devices_to_check) {
4143             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4144             next if !$d->{'pci_bridge'};
4145
4146             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4147             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4148         }
4149         $devices_to_check = $to_check;
4150     }
4151
4152     my $resblock = vm_mon_cmd($vmid, 'query-block');
4153     foreach my $block (@$resblock) {
4154         if($block->{device} =~ m/^drive-(\S+)/){
4155                 $devices->{$1} = 1;
4156         }
4157     }
4158
4159     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4160     foreach my $mice (@$resmice) {
4161         if ($mice->{name} eq 'QEMU HID Tablet') {
4162             $devices->{tablet} = 1;
4163             last;
4164         }
4165     }
4166
4167     # for usb devices there is no query-usb
4168     # but we can iterate over the entries in
4169     # qom-list path=/machine/peripheral
4170     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4171     foreach my $per (@$resperipheral) {
4172         if ($per->{name} =~ m/^usb\d+$/) {
4173             $devices->{$per->{name}} = 1;
4174         }
4175     }
4176
4177     return $devices;
4178 }
4179
4180 sub vm_deviceplug {
4181     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4182
4183     my $q35 = machine_type_is_q35($conf);
4184
4185     my $devices_list = vm_devices_list($vmid);
4186     return 1 if defined($devices_list->{$deviceid});
4187
4188     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4189
4190     if ($deviceid eq 'tablet') {
4191
4192         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4193
4194     } elsif ($deviceid eq 'keyboard') {
4195
4196         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4197
4198     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4199
4200         die "usb hotplug currently not reliable\n";
4201         # since we can't reliably hot unplug all added usb devices
4202         # and usb passthrough disables live migration
4203         # we disable usb hotplugging for now
4204         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4205
4206     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4207
4208         qemu_iothread_add($vmid, $deviceid, $device);
4209
4210         qemu_driveadd($storecfg, $vmid, $device);
4211         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4212
4213         qemu_deviceadd($vmid, $devicefull);
4214         eval { qemu_deviceaddverify($vmid, $deviceid); };
4215         if (my $err = $@) {
4216             eval { qemu_drivedel($vmid, $deviceid); };
4217             warn $@ if $@;
4218             die $err;
4219         }
4220
4221     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4222
4223
4224         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4225         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4226         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4227
4228         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4229
4230         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4231             qemu_iothread_add($vmid, $deviceid, $device);
4232             $devicefull .= ",iothread=iothread-$deviceid";
4233         }
4234
4235         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4236             $devicefull .= ",num_queues=$device->{queues}";
4237         }
4238
4239         qemu_deviceadd($vmid, $devicefull);
4240         qemu_deviceaddverify($vmid, $deviceid);
4241
4242     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4243
4244         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4245         qemu_driveadd($storecfg, $vmid, $device);
4246
4247         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4248         eval { qemu_deviceadd($vmid, $devicefull); };
4249         if (my $err = $@) {
4250             eval { qemu_drivedel($vmid, $deviceid); };
4251             warn $@ if $@;
4252             die $err;
4253         }
4254
4255     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4256
4257         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4258
4259         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4260         my $use_old_bios_files = undef;
4261         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4262
4263         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4264         qemu_deviceadd($vmid, $netdevicefull);
4265         eval {
4266             qemu_deviceaddverify($vmid, $deviceid);
4267             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4268         };
4269         if (my $err = $@) {
4270             eval { qemu_netdevdel($vmid, $deviceid); };
4271             warn $@ if $@;
4272             die $err;
4273         }
4274
4275     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4276
4277         my $bridgeid = $2;
4278         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4279         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4280
4281         qemu_deviceadd($vmid, $devicefull);
4282         qemu_deviceaddverify($vmid, $deviceid);
4283
4284     } else {
4285         die "can't hotplug device '$deviceid'\n";
4286     }
4287
4288     return 1;
4289 }
4290
4291 # fixme: this should raise exceptions on error!
4292 sub vm_deviceunplug {
4293     my ($vmid, $conf, $deviceid) = @_;
4294
4295     my $devices_list = vm_devices_list($vmid);
4296     return 1 if !defined($devices_list->{$deviceid});
4297
4298     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4299
4300     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4301
4302         qemu_devicedel($vmid, $deviceid);
4303
4304     } elsif ($deviceid =~ m/^usb\d+$/) {
4305
4306         die "usb hotplug currently not reliable\n";
4307         # when unplugging usb devices this way,
4308         # there may be remaining usb controllers/hubs
4309         # so we disable it for now
4310         qemu_devicedel($vmid, $deviceid);
4311         qemu_devicedelverify($vmid, $deviceid);
4312
4313     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4314
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_drivedel($vmid, $deviceid);
4318         qemu_iothread_del($conf, $vmid, $deviceid);
4319
4320     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4321
4322         qemu_devicedel($vmid, $deviceid);
4323         qemu_devicedelverify($vmid, $deviceid);
4324         qemu_iothread_del($conf, $vmid, $deviceid);
4325
4326     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4327
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_drivedel($vmid, $deviceid);
4330         qemu_deletescsihw($conf, $vmid, $deviceid);
4331
4332     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4333
4334         qemu_devicedel($vmid, $deviceid);
4335         qemu_devicedelverify($vmid, $deviceid);
4336         qemu_netdevdel($vmid, $deviceid);
4337
4338     } else {
4339         die "can't unplug device '$deviceid'\n";
4340     }
4341
4342     return 1;
4343 }
4344
4345 sub qemu_deviceadd {
4346     my ($vmid, $devicefull) = @_;
4347
4348     $devicefull = "driver=".$devicefull;
4349     my %options =  split(/[=,]/, $devicefull);
4350
4351     vm_mon_cmd($vmid, "device_add" , %options);
4352 }
4353
4354 sub qemu_devicedel {
4355     my ($vmid, $deviceid) = @_;
4356
4357     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4358 }
4359
4360 sub qemu_iothread_add {
4361     my($vmid, $deviceid, $device) = @_;
4362
4363     if ($device->{iothread}) {
4364         my $iothreads = vm_iothreads_list($vmid);
4365         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4366     }
4367 }
4368
4369 sub qemu_iothread_del {
4370     my($conf, $vmid, $deviceid) = @_;
4371
4372     my $confid = $deviceid;
4373     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4374         $confid = 'scsi' . $1;
4375     }
4376     my $device = parse_drive($confid, $conf->{$confid});
4377     if ($device->{iothread}) {
4378         my $iothreads = vm_iothreads_list($vmid);
4379         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4380     }
4381 }
4382
4383 sub qemu_objectadd {
4384     my($vmid, $objectid, $qomtype) = @_;
4385
4386     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4387
4388     return 1;
4389 }
4390
4391 sub qemu_objectdel {
4392     my($vmid, $objectid) = @_;
4393
4394     vm_mon_cmd($vmid, "object-del", id => $objectid);
4395
4396     return 1;
4397 }
4398
4399 sub qemu_driveadd {
4400     my ($storecfg, $vmid, $device) = @_;
4401
4402     my $drive = print_drive_full($storecfg, $vmid, $device);
4403     $drive =~ s/\\/\\\\/g;
4404     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4405
4406     # If the command succeeds qemu prints: "OK"
4407     return 1 if $ret =~ m/OK/s;
4408
4409     die "adding drive failed: $ret\n";
4410 }
4411
4412 sub qemu_drivedel {
4413     my($vmid, $deviceid) = @_;
4414
4415     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4416     $ret =~ s/^\s+//;
4417
4418     return 1 if $ret eq "";
4419
4420     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4421     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4422
4423     die "deleting drive $deviceid failed : $ret\n";
4424 }
4425
4426 sub qemu_deviceaddverify {
4427     my ($vmid, $deviceid) = @_;
4428
4429     for (my $i = 0; $i <= 5; $i++) {
4430          my $devices_list = vm_devices_list($vmid);
4431          return 1 if defined($devices_list->{$deviceid});
4432          sleep 1;
4433     }
4434
4435     die "error on hotplug device '$deviceid'\n";
4436 }
4437
4438
4439 sub qemu_devicedelverify {
4440     my ($vmid, $deviceid) = @_;
4441
4442     # need to verify that the device is correctly removed as device_del
4443     # is async and empty return is not reliable
4444
4445     for (my $i = 0; $i <= 5; $i++) {
4446          my $devices_list = vm_devices_list($vmid);
4447          return 1 if !defined($devices_list->{$deviceid});
4448          sleep 1;
4449     }
4450
4451     die "error on hot-unplugging device '$deviceid'\n";
4452 }
4453
4454 sub qemu_findorcreatescsihw {
4455     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4456
4457     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4458
4459     my $scsihwid="$controller_prefix$controller";
4460     my $devices_list = vm_devices_list($vmid);
4461
4462     if(!defined($devices_list->{$scsihwid})) {
4463         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4464     }
4465
4466     return 1;
4467 }
4468
4469 sub qemu_deletescsihw {
4470     my ($conf, $vmid, $opt) = @_;
4471
4472     my $device = parse_drive($opt, $conf->{$opt});
4473
4474     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4475         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4476         return 1;
4477     }
4478
4479     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4480
4481     my $devices_list = vm_devices_list($vmid);
4482     foreach my $opt (keys %{$devices_list}) {
4483         if (PVE::QemuServer::is_valid_drivename($opt)) {
4484             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4485             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4486                 return 1;
4487             }
4488         }
4489     }
4490
4491     my $scsihwid="scsihw$controller";
4492
4493     vm_deviceunplug($vmid, $conf, $scsihwid);
4494
4495     return 1;
4496 }
4497
4498 sub qemu_add_pci_bridge {
4499     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4500
4501     my $bridges = {};
4502
4503     my $bridgeid;
4504
4505     print_pci_addr($device, $bridges, $arch, $machine_type);
4506
4507     while (my ($k, $v) = each %$bridges) {
4508         $bridgeid = $k;
4509     }
4510     return 1 if !defined($bridgeid) || $bridgeid < 1;
4511
4512     my $bridge = "pci.$bridgeid";
4513     my $devices_list = vm_devices_list($vmid);
4514
4515     if (!defined($devices_list->{$bridge})) {
4516         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4517     }
4518
4519     return 1;
4520 }
4521
4522 sub qemu_set_link_status {
4523     my ($vmid, $device, $up) = @_;
4524
4525     vm_mon_cmd($vmid, "set_link", name => $device,
4526                up => $up ? JSON::true : JSON::false);
4527 }
4528
4529 sub qemu_netdevadd {
4530     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4531
4532     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4533     my %options =  split(/[=,]/, $netdev);
4534
4535     vm_mon_cmd($vmid, "netdev_add",  %options);
4536     return 1;
4537 }
4538
4539 sub qemu_netdevdel {
4540     my ($vmid, $deviceid) = @_;
4541
4542     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4543 }
4544
4545 sub qemu_usb_hotplug {
4546     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4547
4548     return if !$device;
4549
4550     # remove the old one first
4551     vm_deviceunplug($vmid, $conf, $deviceid);
4552
4553     # check if xhci controller is necessary and available
4554     if ($device->{usb3}) {
4555
4556         my $devicelist = vm_devices_list($vmid);
4557
4558         if (!$devicelist->{xhci}) {
4559             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4560             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4561         }
4562     }
4563     my $d = parse_usb_device($device->{host});
4564     $d->{usb3} = $device->{usb3};
4565
4566     # add the new one
4567     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4568 }
4569
4570 sub qemu_cpu_hotplug {
4571     my ($vmid, $conf, $vcpus) = @_;
4572
4573     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4574
4575     my $sockets = 1;
4576     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4577     $sockets = $conf->{sockets} if  $conf->{sockets};
4578     my $cores = $conf->{cores} || 1;
4579     my $maxcpus = $sockets * $cores;
4580
4581     $vcpus = $maxcpus if !$vcpus;
4582
4583     die "you can't add more vcpus than maxcpus\n"
4584         if $vcpus > $maxcpus;
4585
4586     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4587
4588     if ($vcpus < $currentvcpus) {
4589
4590         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4591
4592             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4593                 qemu_devicedel($vmid, "cpu$i");
4594                 my $retry = 0;
4595                 my $currentrunningvcpus = undef;
4596                 while (1) {
4597                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4598                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4599                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4600                     $retry++;
4601                     sleep 1;
4602                 }
4603                 #update conf after each succesfull cpu unplug
4604                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4605                 PVE::QemuConfig->write_config($vmid, $conf);
4606             }
4607         } else {
4608             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4609         }
4610
4611         return;
4612     }
4613
4614     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4615     die "vcpus in running vm does not match its configuration\n"
4616         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4617
4618     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4619
4620         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4621             my $cpustr = print_cpu_device($conf, $i);
4622             qemu_deviceadd($vmid, $cpustr);
4623
4624             my $retry = 0;
4625             my $currentrunningvcpus = undef;
4626             while (1) {
4627                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4628                 last if scalar(@{$currentrunningvcpus}) == $i;
4629                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4630                 sleep 1;
4631                 $retry++;
4632             }
4633             #update conf after each succesfull cpu hotplug
4634             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4635             PVE::QemuConfig->write_config($vmid, $conf);
4636         }
4637     } else {
4638
4639         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4640             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4641         }
4642     }
4643 }
4644
4645 sub qemu_block_set_io_throttle {
4646     my ($vmid, $deviceid,
4647         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4648         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4649         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4650         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4651
4652     return if !check_running($vmid) ;
4653
4654     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4655         bps => int($bps),
4656         bps_rd => int($bps_rd),
4657         bps_wr => int($bps_wr),
4658         iops => int($iops),
4659         iops_rd => int($iops_rd),
4660         iops_wr => int($iops_wr),
4661         bps_max => int($bps_max),
4662         bps_rd_max => int($bps_rd_max),
4663         bps_wr_max => int($bps_wr_max),
4664         iops_max => int($iops_max),
4665         iops_rd_max => int($iops_rd_max),
4666         iops_wr_max => int($iops_wr_max),
4667         bps_max_length => int($bps_max_length),
4668         bps_rd_max_length => int($bps_rd_max_length),
4669         bps_wr_max_length => int($bps_wr_max_length),
4670         iops_max_length => int($iops_max_length),
4671         iops_rd_max_length => int($iops_rd_max_length),
4672         iops_wr_max_length => int($iops_wr_max_length),
4673     );
4674
4675 }
4676
4677 # old code, only used to shutdown old VM after update
4678 sub __read_avail {
4679     my ($fh, $timeout) = @_;
4680
4681     my $sel = new IO::Select;
4682     $sel->add($fh);
4683
4684     my $res = '';
4685     my $buf;
4686
4687     my @ready;
4688     while (scalar (@ready = $sel->can_read($timeout))) {
4689         my $count;
4690         if ($count = $fh->sysread($buf, 8192)) {
4691             if ($buf =~ /^(.*)\(qemu\) $/s) {
4692                 $res .= $1;
4693                 last;
4694             } else {
4695                 $res .= $buf;
4696             }
4697         } else {
4698             if (!defined($count)) {
4699                 die "$!\n";
4700             }
4701             last;
4702         }
4703     }
4704
4705     die "monitor read timeout\n" if !scalar(@ready);
4706
4707     return $res;
4708 }
4709
4710 sub qemu_block_resize {
4711     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4712
4713     my $running = check_running($vmid);
4714
4715     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4716
4717     return if !$running;
4718
4719     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4720
4721 }
4722
4723 sub qemu_volume_snapshot {
4724     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4725
4726     my $running = check_running($vmid);
4727
4728     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4729         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4730     } else {
4731         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4732     }
4733 }
4734
4735 sub qemu_volume_snapshot_delete {
4736     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4737
4738     my $running = check_running($vmid);
4739
4740     if($running) {
4741
4742         $running = undef;
4743         my $conf = PVE::QemuConfig->load_config($vmid);
4744         foreach_drive($conf, sub {
4745             my ($ds, $drive) = @_;
4746             $running = 1 if $drive->{file} eq $volid;
4747         });
4748     }
4749
4750     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4751         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4752     } else {
4753         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4754     }
4755 }
4756
4757 sub set_migration_caps {
4758     my ($vmid) = @_;
4759
4760     my $cap_ref = [];
4761
4762     my $enabled_cap = {
4763         "auto-converge" => 1,
4764         "xbzrle" => 1,
4765         "x-rdma-pin-all" => 0,
4766         "zero-blocks" => 0,
4767         "compress" => 0
4768     };
4769
4770     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4771
4772     for my $supported_capability (@$supported_capabilities) {
4773         push @$cap_ref, {
4774             capability => $supported_capability->{capability},
4775             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4776         };
4777     }
4778
4779     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4780 }
4781
4782 my $fast_plug_option = {
4783     'lock' => 1,
4784     'name' => 1,
4785     'onboot' => 1,
4786     'shares' => 1,
4787     'startup' => 1,
4788     'description' => 1,
4789     'protection' => 1,
4790     'vmstatestorage' => 1,
4791     'hookscript' => 1,
4792 };
4793
4794 # hotplug changes in [PENDING]
4795 # $selection hash can be used to only apply specified options, for
4796 # example: { cores => 1 } (only apply changed 'cores')
4797 # $errors ref is used to return error messages
4798 sub vmconfig_hotplug_pending {
4799     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4800
4801     my $defaults = load_defaults();
4802     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4803
4804     # commit values which do not have any impact on running VM first
4805     # Note: those option cannot raise errors, we we do not care about
4806     # $selection and always apply them.
4807
4808     my $add_error = sub {
4809         my ($opt, $msg) = @_;
4810         $errors->{$opt} = "hotplug problem - $msg";
4811     };
4812
4813     my $changes = 0;
4814     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4815         if ($fast_plug_option->{$opt}) {
4816             $conf->{$opt} = $conf->{pending}->{$opt};
4817             delete $conf->{pending}->{$opt};
4818             $changes = 1;
4819         }
4820     }
4821
4822     if ($changes) {
4823         PVE::QemuConfig->write_config($vmid, $conf);
4824         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4825     }
4826
4827     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4828
4829     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4830     while (my ($opt, $force) = each %$pending_delete_hash) {
4831         next if $selection && !$selection->{$opt};
4832         eval {
4833             if ($opt eq 'hotplug') {
4834                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4835             } elsif ($opt eq 'tablet') {
4836                 die "skip\n" if !$hotplug_features->{usb};
4837                 if ($defaults->{tablet}) {
4838                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4839                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4840                         if $arch eq 'aarch64';
4841                 } else {
4842                     vm_deviceunplug($vmid, $conf, 'tablet');
4843                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4844                 }
4845             } elsif ($opt =~ m/^usb\d+/) {
4846                 die "skip\n";
4847                 # since we cannot reliably hot unplug usb devices
4848                 # we are disabling it
4849                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4850                 vm_deviceunplug($vmid, $conf, $opt);
4851             } elsif ($opt eq 'vcpus') {
4852                 die "skip\n" if !$hotplug_features->{cpu};
4853                 qemu_cpu_hotplug($vmid, $conf, undef);
4854             } elsif ($opt eq 'balloon') {
4855                 # enable balloon device is not hotpluggable
4856                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4857                 # here we reset the ballooning value to memory
4858                 my $balloon = $conf->{memory} || $defaults->{memory};
4859                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4860             } elsif ($fast_plug_option->{$opt}) {
4861                 # do nothing
4862             } elsif ($opt =~ m/^net(\d+)$/) {
4863                 die "skip\n" if !$hotplug_features->{network};
4864                 vm_deviceunplug($vmid, $conf, $opt);
4865             } elsif (is_valid_drivename($opt)) {
4866                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4867                 vm_deviceunplug($vmid, $conf, $opt);
4868                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4869             } elsif ($opt =~ m/^memory$/) {
4870                 die "skip\n" if !$hotplug_features->{memory};
4871                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4872             } elsif ($opt eq 'cpuunits') {
4873                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4874             } elsif ($opt eq 'cpulimit') {
4875                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4876             } else {
4877                 die "skip\n";
4878             }
4879         };
4880         if (my $err = $@) {
4881             &$add_error($opt, $err) if $err ne "skip\n";
4882         } else {
4883             # save new config if hotplug was successful
4884             delete $conf->{$opt};
4885             vmconfig_undelete_pending_option($conf, $opt);
4886             PVE::QemuConfig->write_config($vmid, $conf);
4887             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4888         }
4889     }
4890
4891     my $apply_pending_cloudinit;
4892     $apply_pending_cloudinit = sub {
4893         my ($key, $value) = @_;
4894         $apply_pending_cloudinit = sub {}; # once is enough
4895
4896         my @cloudinit_opts = keys %$confdesc_cloudinit;
4897         foreach my $opt (keys %{$conf->{pending}}) {
4898             next if !grep { $_ eq $opt } @cloudinit_opts;
4899             $conf->{$opt} = delete $conf->{pending}->{$opt};
4900         }
4901
4902         my $new_conf = { %$conf };
4903         $new_conf->{$key} = $value;
4904         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4905     };
4906
4907     foreach my $opt (keys %{$conf->{pending}}) {
4908         next if $selection && !$selection->{$opt};
4909         my $value = $conf->{pending}->{$opt};
4910         eval {
4911             if ($opt eq 'hotplug') {
4912                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4913             } elsif ($opt eq 'tablet') {
4914                 die "skip\n" if !$hotplug_features->{usb};
4915                 if ($value == 1) {
4916                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4917                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4918                         if $arch eq 'aarch64';
4919                 } elsif ($value == 0) {
4920                     vm_deviceunplug($vmid, $conf, 'tablet');
4921                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4922                 }
4923             } elsif ($opt =~ m/^usb\d+$/) {
4924                 die "skip\n";
4925                 # since we cannot reliably hot unplug usb devices
4926                 # we are disabling it
4927                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4928                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4929                 die "skip\n" if !$d;
4930                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4931             } elsif ($opt eq 'vcpus') {
4932                 die "skip\n" if !$hotplug_features->{cpu};
4933                 qemu_cpu_hotplug($vmid, $conf, $value);
4934             } elsif ($opt eq 'balloon') {
4935                 # enable/disable balloning device is not hotpluggable
4936                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4937                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4938                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4939
4940                 # allow manual ballooning if shares is set to zero
4941                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4942                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4943                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4944                 }
4945             } elsif ($opt =~ m/^net(\d+)$/) {
4946                 # some changes can be done without hotplug
4947                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4948                                     $vmid, $opt, $value, $arch, $machine_type);
4949             } elsif (is_valid_drivename($opt)) {
4950                 # some changes can be done without hotplug
4951                 my $drive = parse_drive($opt, $value);
4952                 if (drive_is_cloudinit($drive)) {
4953                     &$apply_pending_cloudinit($opt, $value);
4954                 }
4955                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4956                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4957             } elsif ($opt =~ m/^memory$/) { #dimms
4958                 die "skip\n" if !$hotplug_features->{memory};
4959                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4960             } elsif ($opt eq 'cpuunits') {
4961                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4962             } elsif ($opt eq 'cpulimit') {
4963                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4964                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4965             } else {
4966                 die "skip\n";  # skip non-hot-pluggable options
4967             }
4968         };
4969         if (my $err = $@) {
4970             &$add_error($opt, $err) if $err ne "skip\n";
4971         } else {
4972             # save new config if hotplug was successful
4973             $conf->{$opt} = $value;
4974             delete $conf->{pending}->{$opt};
4975             PVE::QemuConfig->write_config($vmid, $conf);
4976             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4977         }
4978     }
4979 }
4980
4981 sub try_deallocate_drive {
4982     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4983
4984     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4985         my $volid = $drive->{file};
4986         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4987             my $sid = PVE::Storage::parse_volume_id($volid);
4988             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4989
4990             # check if the disk is really unused
4991             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4992                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4993             PVE::Storage::vdisk_free($storecfg, $volid);
4994             return 1;
4995         } else {
4996             # If vm is not owner of this disk remove from config
4997             return 1;
4998         }
4999     }
5000
5001     return undef;
5002 }
5003
5004 sub vmconfig_delete_or_detach_drive {
5005     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5006
5007     my $drive = parse_drive($opt, $conf->{$opt});
5008
5009     my $rpcenv = PVE::RPCEnvironment::get();
5010     my $authuser = $rpcenv->get_user();
5011
5012     if ($force) {
5013         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5014         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5015     } else {
5016         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5017     }
5018 }
5019
5020 sub vmconfig_apply_pending {
5021     my ($vmid, $conf, $storecfg) = @_;
5022
5023     # cold plug
5024
5025     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5026     while (my ($opt, $force) = each %$pending_delete_hash) {
5027         die "internal error" if $opt =~ m/^unused/;
5028         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5029         if (!defined($conf->{$opt})) {
5030             vmconfig_undelete_pending_option($conf, $opt);
5031             PVE::QemuConfig->write_config($vmid, $conf);
5032         } elsif (is_valid_drivename($opt)) {
5033             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5034             vmconfig_undelete_pending_option($conf, $opt);
5035             delete $conf->{$opt};
5036             PVE::QemuConfig->write_config($vmid, $conf);
5037         } else {
5038             vmconfig_undelete_pending_option($conf, $opt);
5039             delete $conf->{$opt};
5040             PVE::QemuConfig->write_config($vmid, $conf);
5041         }
5042     }
5043
5044     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5045
5046     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5047         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5048
5049         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5050             # skip if nothing changed
5051         } elsif (is_valid_drivename($opt)) {
5052             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5053                 if defined($conf->{$opt});
5054             $conf->{$opt} = $conf->{pending}->{$opt};
5055         } else {
5056             $conf->{$opt} = $conf->{pending}->{$opt};
5057         }
5058
5059         delete $conf->{pending}->{$opt};
5060         PVE::QemuConfig->write_config($vmid, $conf);
5061     }
5062 }
5063
5064 my $safe_num_ne = sub {
5065     my ($a, $b) = @_;
5066
5067     return 0 if !defined($a) && !defined($b);
5068     return 1 if !defined($a);
5069     return 1 if !defined($b);
5070
5071     return $a != $b;
5072 };
5073
5074 my $safe_string_ne = sub {
5075     my ($a, $b) = @_;
5076
5077     return 0 if !defined($a) && !defined($b);
5078     return 1 if !defined($a);
5079     return 1 if !defined($b);
5080
5081     return $a ne $b;
5082 };
5083
5084 sub vmconfig_update_net {
5085     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5086
5087     my $newnet = parse_net($value);
5088
5089     if ($conf->{$opt}) {
5090         my $oldnet = parse_net($conf->{$opt});
5091
5092         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5093             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5094             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5095             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5096
5097             # for non online change, we try to hot-unplug
5098             die "skip\n" if !$hotplug;
5099             vm_deviceunplug($vmid, $conf, $opt);
5100         } else {
5101
5102             die "internal error" if $opt !~ m/net(\d+)/;
5103             my $iface = "tap${vmid}i$1";
5104
5105             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5106                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5107                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5108                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5109                 PVE::Network::tap_unplug($iface);
5110                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5111             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5112                 # Rate can be applied on its own but any change above needs to
5113                 # include the rate in tap_plug since OVS resets everything.
5114                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5115             }
5116
5117             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5118                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5119             }
5120
5121             return 1;
5122         }
5123     }
5124
5125     if ($hotplug) {
5126         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5127     } else {
5128         die "skip\n";
5129     }
5130 }
5131
5132 sub vmconfig_update_disk {
5133     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5134
5135     # fixme: do we need force?
5136
5137     my $drive = parse_drive($opt, $value);
5138
5139     if ($conf->{$opt}) {
5140
5141         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5142
5143             my $media = $drive->{media} || 'disk';
5144             my $oldmedia = $old_drive->{media} || 'disk';
5145             die "unable to change media type\n" if $media ne $oldmedia;
5146
5147             if (!drive_is_cdrom($old_drive)) {
5148
5149                 if ($drive->{file} ne $old_drive->{file}) {
5150
5151                     die "skip\n" if !$hotplug;
5152
5153                     # unplug and register as unused
5154                     vm_deviceunplug($vmid, $conf, $opt);
5155                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5156
5157                 } else {
5158                     # update existing disk
5159
5160                     # skip non hotpluggable value
5161                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5162                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5163                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5164                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5165                         die "skip\n";
5166                     }
5167
5168                     # apply throttle
5169                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5170                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5171                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5172                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5173                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5174                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5175                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5176                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5177                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5178                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5179                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5180                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5181                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5182                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5183                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5184                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5185                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5186                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5187
5188                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5189                                                    ($drive->{mbps} || 0)*1024*1024,
5190                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5191                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5192                                                    $drive->{iops} || 0,
5193                                                    $drive->{iops_rd} || 0,
5194                                                    $drive->{iops_wr} || 0,
5195                                                    ($drive->{mbps_max} || 0)*1024*1024,
5196                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5197                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5198                                                    $drive->{iops_max} || 0,
5199                                                    $drive->{iops_rd_max} || 0,
5200                                                    $drive->{iops_wr_max} || 0,
5201                                                    $drive->{bps_max_length} || 1,
5202                                                    $drive->{bps_rd_max_length} || 1,
5203                                                    $drive->{bps_wr_max_length} || 1,
5204                                                    $drive->{iops_max_length} || 1,
5205                                                    $drive->{iops_rd_max_length} || 1,
5206                                                    $drive->{iops_wr_max_length} || 1);
5207
5208                     }
5209
5210                     return 1;
5211                 }
5212
5213             } else { # cdrom
5214
5215                 if ($drive->{file} eq 'none') {
5216                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5217                     if (drive_is_cloudinit($old_drive)) {
5218                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5219                     }
5220                 } else {
5221                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5222                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5223                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5224                 }
5225
5226                 return 1;
5227             }
5228         }
5229     }
5230
5231     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5232     # hotplug new disks
5233     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5234     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5235 }
5236
5237 sub vm_start {
5238     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5239         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5240
5241     PVE::QemuConfig->lock_config($vmid, sub {
5242         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5243
5244         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5245
5246         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5247
5248         PVE::QemuConfig->check_lock($conf)
5249             if !($skiplock || $is_suspended);
5250
5251         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5252
5253         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5254             vmconfig_apply_pending($vmid, $conf, $storecfg);
5255             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5256         }
5257
5258         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5259
5260         my $defaults = load_defaults();
5261
5262         # set environment variable useful inside network script
5263         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5264
5265         my $local_volumes = {};
5266
5267         if ($targetstorage) {
5268             foreach_drive($conf, sub {
5269                 my ($ds, $drive) = @_;
5270
5271                 return if drive_is_cdrom($drive);
5272
5273                 my $volid = $drive->{file};
5274
5275                 return if !$volid;
5276
5277                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5278
5279                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5280                 return if $scfg->{shared};
5281                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5282             });
5283
5284             my $format = undef;
5285
5286             foreach my $opt (sort keys %$local_volumes) {
5287
5288                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5289                 my $drive = parse_drive($opt, $conf->{$opt});
5290
5291                 #if remote storage is specified, use default format
5292                 if ($targetstorage && $targetstorage ne "1") {
5293                     $storeid = $targetstorage;
5294                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5295                     $format = $defFormat;
5296                 } else {
5297                     #else we use same format than original
5298                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5299                     $format = qemu_img_format($scfg, $volid);
5300                 }
5301
5302                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5303                 my $newdrive = $drive;
5304                 $newdrive->{format} = $format;
5305                 $newdrive->{file} = $newvolid;
5306                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5307                 $local_volumes->{$opt} = $drivestr;
5308                 #pass drive to conf for command line
5309                 $conf->{$opt} = $drivestr;
5310             }
5311         }
5312
5313         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5314
5315         if ($is_suspended) {
5316             # enforce machine type on suspended vm to ensure HW compatibility
5317             $forcemachine = $conf->{runningmachine};
5318             print "Resuming suspended VM\n";
5319         }
5320
5321         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5322
5323         my $migrate_port = 0;
5324         my $migrate_uri;
5325         if ($statefile) {
5326             if ($statefile eq 'tcp') {
5327                 my $localip = "localhost";
5328                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5329                 my $nodename = PVE::INotify::nodename();
5330
5331                 if (!defined($migration_type)) {
5332                     if (defined($datacenterconf->{migration}->{type})) {
5333                         $migration_type = $datacenterconf->{migration}->{type};
5334                     } else {
5335                         $migration_type = 'secure';
5336                     }
5337                 }
5338
5339                 if ($migration_type eq 'insecure') {
5340                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5341                     if ($migrate_network_addr) {
5342                         $localip = $migrate_network_addr;
5343                     } else {
5344                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5345                     }
5346
5347                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5348                 }
5349
5350                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5351                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5352                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5353                 push @$cmd, '-incoming', $migrate_uri;
5354                 push @$cmd, '-S';
5355
5356             } elsif ($statefile eq 'unix') {
5357                 # should be default for secure migrations as a ssh TCP forward
5358                 # tunnel is not deterministic reliable ready and fails regurarly
5359                 # to set up in time, so use UNIX socket forwards
5360                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5361                 unlink $socket_addr;
5362
5363                 $migrate_uri = "unix:$socket_addr";
5364
5365                 push @$cmd, '-incoming', $migrate_uri;
5366                 push @$cmd, '-S';
5367
5368             } else {
5369                 push @$cmd, '-loadstate', $statefile;
5370             }
5371         } elsif ($paused) {
5372             push @$cmd, '-S';
5373         }
5374
5375         # host pci devices
5376         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5377           my $d = parse_hostpci($conf->{"hostpci$i"});
5378           next if !$d;
5379           my $pcidevices = $d->{pciid};
5380           foreach my $pcidevice (@$pcidevices) {
5381                 my $pciid = $pcidevice->{id};
5382
5383                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5384                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5385                 die "no pci device info for device '$pciid'\n" if !$info;
5386
5387                 if ($d->{mdev}) {
5388                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5389                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5390                 } else {
5391                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5392                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5393                     die "can't reset pci device '$pciid'\n"
5394                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5395                 }
5396           }
5397         }
5398
5399         PVE::Storage::activate_volumes($storecfg, $vollist);
5400
5401         eval {
5402             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5403                 outfunc => sub {}, errfunc => sub {});
5404         };
5405         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5406         # timeout should be more than enough here...
5407         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5408
5409         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5410                                                   : $defaults->{cpuunits};
5411
5412         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5413         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5414
5415         my %properties = (
5416             Slice => 'qemu.slice',
5417             KillMode => 'none',
5418             CPUShares => $cpuunits
5419         );
5420
5421         if (my $cpulimit = $conf->{cpulimit}) {
5422             $properties{CPUQuota} = int($cpulimit * 100);
5423         }
5424         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5425
5426         my $run_qemu = sub {
5427             PVE::Tools::run_fork sub {
5428                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5429                 run_command($cmd, %run_params);
5430             };
5431         };
5432
5433         if ($conf->{hugepages}) {
5434
5435             my $code = sub {
5436                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5437                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5438
5439                 PVE::QemuServer::Memory::hugepages_mount();
5440                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5441
5442                 eval { $run_qemu->() };
5443                 if (my $err = $@) {
5444                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5445                     die $err;
5446                 }
5447
5448                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5449             };
5450             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5451
5452         } else {
5453             eval { $run_qemu->() };
5454         }
5455
5456         if (my $err = $@) {
5457             # deactivate volumes if start fails
5458             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5459             die "start failed: $err";
5460         }
5461
5462         print "migration listens on $migrate_uri\n" if $migrate_uri;
5463
5464         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5465             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5466             warn $@ if $@;
5467         }
5468
5469         #start nbd server for storage migration
5470         if ($targetstorage) {
5471             my $nodename = PVE::INotify::nodename();
5472             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5473             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5474             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5475             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5476
5477             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5478
5479             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5480
5481             foreach my $opt (sort keys %$local_volumes) {
5482                 my $volid = $local_volumes->{$opt};
5483                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5484                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5485                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5486             }
5487         }
5488
5489         if ($migratedfrom) {
5490             eval {
5491                 set_migration_caps($vmid);
5492             };
5493             warn $@ if $@;
5494
5495             if ($spice_port) {
5496                 print "spice listens on port $spice_port\n";
5497                 if ($spice_ticket) {
5498                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5499                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5500                 }
5501             }
5502
5503         } else {
5504             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5505                 if !$statefile && $conf->{balloon};
5506
5507             foreach my $opt (keys %$conf) {
5508                 next if $opt !~  m/^net\d+$/;
5509                 my $nicconf = parse_net($conf->{$opt});
5510                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5511             }
5512         }
5513
5514         vm_mon_cmd_nocheck($vmid, 'qom-set',
5515                     path => "machine/peripheral/balloon0",
5516                     property => "guest-stats-polling-interval",
5517                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5518
5519         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5520             print "Resumed VM, removing state\n";
5521             delete $conf->@{qw(lock vmstate runningmachine)};
5522             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5523             PVE::Storage::vdisk_free($storecfg, $vmstate);
5524             PVE::QemuConfig->write_config($vmid, $conf);
5525         }
5526
5527         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5528     });
5529 }
5530
5531 sub vm_mon_cmd {
5532     my ($vmid, $execute, %params) = @_;
5533
5534     my $cmd = { execute => $execute, arguments => \%params };
5535     vm_qmp_command($vmid, $cmd);
5536 }
5537
5538 sub vm_mon_cmd_nocheck {
5539     my ($vmid, $execute, %params) = @_;
5540
5541     my $cmd = { execute => $execute, arguments => \%params };
5542     vm_qmp_command($vmid, $cmd, 1);
5543 }
5544
5545 sub vm_qmp_command {
5546     my ($vmid, $cmd, $nocheck) = @_;
5547
5548     my $res;
5549
5550     my $timeout;
5551     if ($cmd->{arguments}) {
5552         $timeout = delete $cmd->{arguments}->{timeout};
5553     }
5554
5555     eval {
5556         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5557         my $sname = qmp_socket($vmid);
5558         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5559             my $qmpclient = PVE::QMPClient->new();
5560
5561             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5562         } else {
5563             die "unable to open monitor socket\n";
5564         }
5565     };
5566     if (my $err = $@) {
5567         syslog("err", "VM $vmid qmp command failed - $err");
5568         die $err;
5569     }
5570
5571     return $res;
5572 }
5573
5574 sub vm_human_monitor_command {
5575     my ($vmid, $cmdline) = @_;
5576
5577     my $cmd = {
5578         execute => 'human-monitor-command',
5579         arguments => { 'command-line' => $cmdline},
5580     };
5581
5582     return vm_qmp_command($vmid, $cmd);
5583 }
5584
5585 sub vm_commandline {
5586     my ($storecfg, $vmid, $snapname) = @_;
5587
5588     my $conf = PVE::QemuConfig->load_config($vmid);
5589
5590     if ($snapname) {
5591         my $snapshot = $conf->{snapshots}->{$snapname};
5592         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5593
5594         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5595
5596         $conf = $snapshot;
5597     }
5598
5599     my $defaults = load_defaults();
5600
5601     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5602
5603     return PVE::Tools::cmd2string($cmd);
5604 }
5605
5606 sub vm_reset {
5607     my ($vmid, $skiplock) = @_;
5608
5609     PVE::QemuConfig->lock_config($vmid, sub {
5610
5611         my $conf = PVE::QemuConfig->load_config($vmid);
5612
5613         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5614
5615         vm_mon_cmd($vmid, "system_reset");
5616     });
5617 }
5618
5619 sub get_vm_volumes {
5620     my ($conf) = @_;
5621
5622     my $vollist = [];
5623     foreach_volid($conf, sub {
5624         my ($volid, $attr) = @_;
5625
5626         return if $volid =~ m|^/|;
5627
5628         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5629         return if !$sid;
5630
5631         push @$vollist, $volid;
5632     });
5633
5634     return $vollist;
5635 }
5636
5637 sub vm_stop_cleanup {
5638     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5639
5640     eval {
5641
5642         if (!$keepActive) {
5643             my $vollist = get_vm_volumes($conf);
5644             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5645         }
5646
5647         foreach my $ext (qw(mon qmp pid vnc qga)) {
5648             unlink "/var/run/qemu-server/${vmid}.$ext";
5649         }
5650
5651         if ($conf->{ivshmem}) {
5652             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5653             # just delete it for now, VMs which have this already open do not
5654             # are affected, but new VMs will get a separated one. If this
5655             # becomes an issue we either add some sort of ref-counting or just
5656             # add a "don't delete on stop" flag to the ivshmem format.
5657             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5658         }
5659
5660         foreach my $key (keys %$conf) {
5661             next if $key !~ m/^hostpci(\d+)$/;
5662             my $hostpciindex = $1;
5663             my $d = parse_hostpci($conf->{$key});
5664             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5665
5666             foreach my $pci (@{$d->{pciid}}) {
5667                 my $pciid = $pci->{id};
5668                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5669             }
5670         }
5671
5672         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5673     };
5674     warn $@ if $@; # avoid errors - just warn
5675 }
5676
5677 # Note: use $nockeck to skip tests if VM configuration file exists.
5678 # We need that when migration VMs to other nodes (files already moved)
5679 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5680 sub vm_stop {
5681     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5682
5683     $force = 1 if !defined($force) && !$shutdown;
5684
5685     if ($migratedfrom){
5686         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5687         kill 15, $pid if $pid;
5688         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5689         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5690         return;
5691     }
5692
5693     PVE::QemuConfig->lock_config($vmid, sub {
5694
5695         my $pid = check_running($vmid, $nocheck);
5696         return if !$pid;
5697
5698         my $conf;
5699         if (!$nocheck) {
5700             $conf = PVE::QemuConfig->load_config($vmid);
5701             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5702             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5703                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5704                 $timeout = $opts->{down} if $opts->{down};
5705             }
5706             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5707         }
5708
5709         eval {
5710             if ($shutdown) {
5711                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5712                     vm_qmp_command($vmid, {
5713                         execute => "guest-shutdown",
5714                         arguments => { timeout => $timeout }
5715                     }, $nocheck);
5716                 } else {
5717                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5718                 }
5719             } else {
5720                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5721             }
5722         };
5723         my $err = $@;
5724
5725         if (!$err) {
5726             $timeout = 60 if !defined($timeout);
5727
5728             my $count = 0;
5729             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5730                 $count++;
5731                 sleep 1;
5732             }
5733
5734             if ($count >= $timeout) {
5735                 if ($force) {
5736                     warn "VM still running - terminating now with SIGTERM\n";
5737                     kill 15, $pid;
5738                 } else {
5739                     die "VM quit/powerdown failed - got timeout\n";
5740                 }
5741             } else {
5742                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5743                 return;
5744             }
5745         } else {
5746             if ($force) {
5747                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5748                 kill 15, $pid;
5749             } else {
5750                 die "VM quit/powerdown failed\n";
5751             }
5752         }
5753
5754         # wait again
5755         $timeout = 10;
5756
5757         my $count = 0;
5758         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5759             $count++;
5760             sleep 1;
5761         }
5762
5763         if ($count >= $timeout) {
5764             warn "VM still running - terminating now with SIGKILL\n";
5765             kill 9, $pid;
5766             sleep 1;
5767         }
5768
5769         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5770    });
5771 }
5772
5773 sub vm_suspend {
5774     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5775
5776     my $conf;
5777     my $path;
5778     my $storecfg;
5779     my $vmstate;
5780
5781     PVE::QemuConfig->lock_config($vmid, sub {
5782
5783         $conf = PVE::QemuConfig->load_config($vmid);
5784
5785         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5786         PVE::QemuConfig->check_lock($conf)
5787             if !($skiplock || $is_backing_up);
5788
5789         die "cannot suspend to disk during backup\n"
5790             if $is_backing_up && $includestate;
5791
5792         if ($includestate) {
5793             $conf->{lock} = 'suspending';
5794             my $date = strftime("%Y-%m-%d", localtime(time()));
5795             $storecfg = PVE::Storage::config();
5796             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5797             $path = PVE::Storage::path($storecfg, $vmstate);
5798             PVE::QemuConfig->write_config($vmid, $conf);
5799         } else {
5800             vm_mon_cmd($vmid, "stop");
5801         }
5802     });
5803
5804     if ($includestate) {
5805         # save vm state
5806         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5807
5808         eval {
5809             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5810             for(;;) {
5811                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5812                 if (!$state->{status}) {
5813                     die "savevm not active\n";
5814                 } elsif ($state->{status} eq 'active') {
5815                     sleep(1);
5816                     next;
5817                 } elsif ($state->{status} eq 'completed') {
5818                     print "State saved, quitting\n";
5819                     last;
5820                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5821                     die "query-savevm failed with error '$state->{error}'\n"
5822                 } else {
5823                     die "query-savevm returned status '$state->{status}'\n";
5824                 }
5825             }
5826         };
5827         my $err = $@;
5828
5829         PVE::QemuConfig->lock_config($vmid, sub {
5830             $conf = PVE::QemuConfig->load_config($vmid);
5831             if ($err) {
5832                 # cleanup, but leave suspending lock, to indicate something went wrong
5833                 eval {
5834                     vm_mon_cmd($vmid, "savevm-end");
5835                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5836                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5837                     delete $conf->@{qw(vmstate runningmachine)};
5838                     PVE::QemuConfig->write_config($vmid, $conf);
5839                 };
5840                 warn $@ if $@;
5841                 die $err;
5842             }
5843
5844             die "lock changed unexpectedly\n"
5845                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5846
5847             vm_qmp_command($vmid, { execute => "quit" });
5848             $conf->{lock} = 'suspended';
5849             PVE::QemuConfig->write_config($vmid, $conf);
5850         });
5851     }
5852 }
5853
5854 sub vm_resume {
5855     my ($vmid, $skiplock, $nocheck) = @_;
5856
5857     PVE::QemuConfig->lock_config($vmid, sub {
5858         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5859         my $res = $vm_mon_cmd->($vmid, 'query-status');
5860         my $resume_cmd = 'cont';
5861
5862         if ($res->{status} && $res->{status} eq 'suspended') {
5863             $resume_cmd = 'system_wakeup';
5864         }
5865
5866         if (!$nocheck) {
5867
5868             my $conf = PVE::QemuConfig->load_config($vmid);
5869
5870             PVE::QemuConfig->check_lock($conf)
5871                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5872         }
5873
5874         $vm_mon_cmd->($vmid, $resume_cmd);
5875     });
5876 }
5877
5878 sub vm_sendkey {
5879     my ($vmid, $skiplock, $key) = @_;
5880
5881     PVE::QemuConfig->lock_config($vmid, sub {
5882
5883         my $conf = PVE::QemuConfig->load_config($vmid);
5884
5885         # there is no qmp command, so we use the human monitor command
5886         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5887         die $res if $res ne '';
5888     });
5889 }
5890
5891 sub vm_destroy {
5892     my ($storecfg, $vmid, $skiplock) = @_;
5893
5894     PVE::QemuConfig->lock_config($vmid, sub {
5895
5896         my $conf = PVE::QemuConfig->load_config($vmid);
5897
5898         if (!check_running($vmid)) {
5899             destroy_vm($storecfg, $vmid, undef, $skiplock);
5900         } else {
5901             die "VM $vmid is running - destroy failed\n";
5902         }
5903     });
5904 }
5905
5906 # vzdump restore implementaion
5907
5908 sub tar_archive_read_firstfile {
5909     my $archive = shift;
5910
5911     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5912
5913     # try to detect archive type first
5914     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5915         die "unable to open file '$archive'\n";
5916     my $firstfile = <$fh>;
5917     kill 15, $pid;
5918     close $fh;
5919
5920     die "ERROR: archive contaions no data\n" if !$firstfile;
5921     chomp $firstfile;
5922
5923     return $firstfile;
5924 }
5925
5926 sub tar_restore_cleanup {
5927     my ($storecfg, $statfile) = @_;
5928
5929     print STDERR "starting cleanup\n";
5930
5931     if (my $fd = IO::File->new($statfile, "r")) {
5932         while (defined(my $line = <$fd>)) {
5933             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5934                 my $volid = $2;
5935                 eval {
5936                     if ($volid =~ m|^/|) {
5937                         unlink $volid || die 'unlink failed\n';
5938                     } else {
5939                         PVE::Storage::vdisk_free($storecfg, $volid);
5940                     }
5941                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5942                 };
5943                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5944             } else {
5945                 print STDERR "unable to parse line in statfile - $line";
5946             }
5947         }
5948         $fd->close();
5949     }
5950 }
5951
5952 sub restore_archive {
5953     my ($archive, $vmid, $user, $opts) = @_;
5954
5955     my $format = $opts->{format};
5956     my $comp;
5957
5958     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5959         $format = 'tar' if !$format;
5960         $comp = 'gzip';
5961     } elsif ($archive =~ m/\.tar$/) {
5962         $format = 'tar' if !$format;
5963     } elsif ($archive =~ m/.tar.lzo$/) {
5964         $format = 'tar' if !$format;
5965         $comp = 'lzop';
5966     } elsif ($archive =~ m/\.vma$/) {
5967         $format = 'vma' if !$format;
5968     } elsif ($archive =~ m/\.vma\.gz$/) {
5969         $format = 'vma' if !$format;
5970         $comp = 'gzip';
5971     } elsif ($archive =~ m/\.vma\.lzo$/) {
5972         $format = 'vma' if !$format;
5973         $comp = 'lzop';
5974     } else {
5975         $format = 'vma' if !$format; # default
5976     }
5977
5978     # try to detect archive format
5979     if ($format eq 'tar') {
5980         return restore_tar_archive($archive, $vmid, $user, $opts);
5981     } else {
5982         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5983     }
5984 }
5985
5986 sub restore_update_config_line {
5987     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5988
5989     return if $line =~ m/^\#qmdump\#/;
5990     return if $line =~ m/^\#vzdump\#/;
5991     return if $line =~ m/^lock:/;
5992     return if $line =~ m/^unused\d+:/;
5993     return if $line =~ m/^parent:/;
5994
5995     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5996     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5997         # try to convert old 1.X settings
5998         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5999         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6000             my ($model, $macaddr) = split(/\=/, $devconfig);
6001             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6002             my $net = {
6003                 model => $model,
6004                 bridge => "vmbr$ind",
6005                 macaddr => $macaddr,
6006             };
6007             my $netstr = print_net($net);
6008
6009             print $outfd "net$cookie->{netcount}: $netstr\n";
6010             $cookie->{netcount}++;
6011         }
6012     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6013         my ($id, $netstr) = ($1, $2);
6014         my $net = parse_net($netstr);
6015         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6016         $netstr = print_net($net);
6017         print $outfd "$id: $netstr\n";
6018     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6019         my $virtdev = $1;
6020         my $value = $3;
6021         my $di = parse_drive($virtdev, $value);
6022         if (defined($di->{backup}) && !$di->{backup}) {
6023             print $outfd "#$line";
6024         } elsif ($map->{$virtdev}) {
6025             delete $di->{format}; # format can change on restore
6026             $di->{file} = $map->{$virtdev};
6027             $value = print_drive($vmid, $di);
6028             print $outfd "$virtdev: $value\n";
6029         } else {
6030             print $outfd $line;
6031         }
6032     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6033         my $vmgenid = $1;
6034         if ($vmgenid ne '0') {
6035             # always generate a new vmgenid if there was a valid one setup
6036             $vmgenid = generate_uuid();
6037         }
6038         print $outfd "vmgenid: $vmgenid\n";
6039     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6040         my ($uuid, $uuid_str);
6041         UUID::generate($uuid);
6042         UUID::unparse($uuid, $uuid_str);
6043         my $smbios1 = parse_smbios1($2);
6044         $smbios1->{uuid} = $uuid_str;
6045         print $outfd $1.print_smbios1($smbios1)."\n";
6046     } else {
6047         print $outfd $line;
6048     }
6049 }
6050
6051 sub scan_volids {
6052     my ($cfg, $vmid) = @_;
6053
6054     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6055
6056     my $volid_hash = {};
6057     foreach my $storeid (keys %$info) {
6058         foreach my $item (@{$info->{$storeid}}) {
6059             next if !($item->{volid} && $item->{size});
6060             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6061             $volid_hash->{$item->{volid}} = $item;
6062         }
6063     }
6064
6065     return $volid_hash;
6066 }
6067
6068 sub is_volume_in_use {
6069     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6070
6071     my $path = PVE::Storage::path($storecfg, $volid);
6072
6073     my $scan_config = sub {
6074         my ($cref, $snapname) = @_;
6075
6076         foreach my $key (keys %$cref) {
6077             my $value = $cref->{$key};
6078             if (is_valid_drivename($key)) {
6079                 next if $skip_drive && $key eq $skip_drive;
6080                 my $drive = parse_drive($key, $value);
6081                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6082                 return 1 if $volid eq $drive->{file};
6083                 if ($drive->{file} =~ m!^/!) {
6084                     return 1 if $drive->{file} eq $path;
6085                 } else {
6086                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6087                     next if !$storeid;
6088                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6089                     next if !$scfg;
6090                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6091                 }
6092             }
6093         }
6094
6095         return 0;
6096     };
6097
6098     return 1 if &$scan_config($conf);
6099
6100     undef $skip_drive;
6101
6102     foreach my $snapname (keys %{$conf->{snapshots}}) {
6103         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6104     }
6105
6106     return 0;
6107 }
6108
6109 sub update_disksize {
6110     my ($vmid, $conf, $volid_hash) = @_;
6111
6112     my $changes;
6113     my $prefix = "VM $vmid:";
6114
6115     # used and unused disks
6116     my $referenced = {};
6117
6118     # Note: it is allowed to define multiple storages with same path (alias), so
6119     # we need to check both 'volid' and real 'path' (two different volid can point
6120     # to the same path).
6121
6122     my $referencedpath = {};
6123
6124     # update size info
6125     foreach my $opt (keys %$conf) {
6126         if (is_valid_drivename($opt)) {
6127             my $drive = parse_drive($opt, $conf->{$opt});
6128             my $volid = $drive->{file};
6129             next if !$volid;
6130
6131             $referenced->{$volid} = 1;
6132             if ($volid_hash->{$volid} &&
6133                 (my $path = $volid_hash->{$volid}->{path})) {
6134                 $referencedpath->{$path} = 1;
6135             }
6136
6137             next if drive_is_cdrom($drive);
6138             next if !$volid_hash->{$volid};
6139
6140             $drive->{size} = $volid_hash->{$volid}->{size};
6141             my $new = print_drive($vmid, $drive);
6142             if ($new ne $conf->{$opt}) {
6143                 $changes = 1;
6144                 $conf->{$opt} = $new;
6145                 print "$prefix update disk '$opt' information.\n";
6146             }
6147         }
6148     }
6149
6150     # remove 'unusedX' entry if volume is used
6151     foreach my $opt (keys %$conf) {
6152         next if $opt !~ m/^unused\d+$/;
6153         my $volid = $conf->{$opt};
6154         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6155         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6156             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6157             $changes = 1;
6158             delete $conf->{$opt};
6159         }
6160
6161         $referenced->{$volid} = 1;
6162         $referencedpath->{$path} = 1 if $path;
6163     }
6164
6165     foreach my $volid (sort keys %$volid_hash) {
6166         next if $volid =~ m/vm-$vmid-state-/;
6167         next if $referenced->{$volid};
6168         my $path = $volid_hash->{$volid}->{path};
6169         next if !$path; # just to be sure
6170         next if $referencedpath->{$path};
6171         $changes = 1;
6172         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6173         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6174         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6175     }
6176
6177     return $changes;
6178 }
6179
6180 sub rescan {
6181     my ($vmid, $nolock, $dryrun) = @_;
6182
6183     my $cfg = PVE::Storage::config();
6184
6185     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6186     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6187     foreach my $stor (keys %{$cfg->{ids}}) {
6188         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6189     }
6190
6191     print "rescan volumes...\n";
6192     my $volid_hash = scan_volids($cfg, $vmid);
6193
6194     my $updatefn =  sub {
6195         my ($vmid) = @_;
6196
6197         my $conf = PVE::QemuConfig->load_config($vmid);
6198
6199         PVE::QemuConfig->check_lock($conf);
6200
6201         my $vm_volids = {};
6202         foreach my $volid (keys %$volid_hash) {
6203             my $info = $volid_hash->{$volid};
6204             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6205         }
6206
6207         my $changes = update_disksize($vmid, $conf, $vm_volids);
6208
6209         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6210     };
6211
6212     if (defined($vmid)) {
6213         if ($nolock) {
6214             &$updatefn($vmid);
6215         } else {
6216             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6217         }
6218     } else {
6219         my $vmlist = config_list();
6220         foreach my $vmid (keys %$vmlist) {
6221             if ($nolock) {
6222                 &$updatefn($vmid);
6223             } else {
6224                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6225             }
6226         }
6227     }
6228 }
6229
6230 sub restore_vma_archive {
6231     my ($archive, $vmid, $user, $opts, $comp) = @_;
6232
6233     my $readfrom = $archive;
6234
6235     my $cfg = PVE::Storage::config();
6236     my $commands = [];
6237     my $bwlimit = $opts->{bwlimit};
6238
6239     my $dbg_cmdstring = '';
6240     my $add_pipe = sub {
6241         my ($cmd) = @_;
6242         push @$commands, $cmd;
6243         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6244         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6245         $readfrom = '-';
6246     };
6247
6248     my $input = undef;
6249     if ($archive eq '-') {
6250         $input = '<&STDIN';
6251     } else {
6252         # If we use a backup from a PVE defined storage we also consider that
6253         # storage's rate limit:
6254         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6255         if (defined($volid)) {
6256             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6257             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6258             if ($readlimit) {
6259                 print STDERR "applying read rate limit: $readlimit\n";
6260                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6261                 $add_pipe->($cstream);
6262             }
6263         }
6264     }
6265
6266     if ($comp) {
6267         my $cmd;
6268         if ($comp eq 'gzip') {
6269             $cmd = ['zcat', $readfrom];
6270         } elsif ($comp eq 'lzop') {
6271             $cmd = ['lzop', '-d', '-c', $readfrom];
6272         } else {
6273             die "unknown compression method '$comp'\n";
6274         }
6275         $add_pipe->($cmd);
6276     }
6277
6278     my $tmpdir = "/var/tmp/vzdumptmp$$";
6279     rmtree $tmpdir;
6280
6281     # disable interrupts (always do cleanups)
6282     local $SIG{INT} =
6283         local $SIG{TERM} =
6284         local $SIG{QUIT} =
6285         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6286
6287     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6288     POSIX::mkfifo($mapfifo, 0600);
6289     my $fifofh;
6290
6291     my $openfifo = sub {
6292         open($fifofh, '>', $mapfifo) || die $!;
6293     };
6294
6295     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6296
6297     my $oldtimeout;
6298     my $timeout = 5;
6299
6300     my $devinfo = {};
6301
6302     my $rpcenv = PVE::RPCEnvironment::get();
6303
6304     my $conffile = PVE::QemuConfig->config_file($vmid);
6305     my $tmpfn = "$conffile.$$.tmp";
6306
6307     # Note: $oldconf is undef if VM does not exists
6308     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6309     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6310
6311     my %storage_limits;
6312
6313     my $print_devmap = sub {
6314         my $virtdev_hash = {};
6315
6316         my $cfgfn = "$tmpdir/qemu-server.conf";
6317
6318         # we can read the config - that is already extracted
6319         my $fh = IO::File->new($cfgfn, "r") ||
6320             "unable to read qemu-server.conf - $!\n";
6321
6322         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6323         if (-f $fwcfgfn) {
6324             my $pve_firewall_dir = '/etc/pve/firewall';
6325             mkdir $pve_firewall_dir; # make sure the dir exists
6326             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6327         }
6328
6329         while (defined(my $line = <$fh>)) {
6330             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6331                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6332                 die "archive does not contain data for drive '$virtdev'\n"
6333                     if !$devinfo->{$devname};
6334                 if (defined($opts->{storage})) {
6335                     $storeid = $opts->{storage} || 'local';
6336                 } elsif (!$storeid) {
6337                     $storeid = 'local';
6338                 }
6339                 $format = 'raw' if !$format;
6340                 $devinfo->{$devname}->{devname} = $devname;
6341                 $devinfo->{$devname}->{virtdev} = $virtdev;
6342                 $devinfo->{$devname}->{format} = $format;
6343                 $devinfo->{$devname}->{storeid} = $storeid;
6344
6345                 # check permission on storage
6346                 my $pool = $opts->{pool}; # todo: do we need that?
6347                 if ($user ne 'root@pam') {
6348                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6349                 }
6350
6351                 $storage_limits{$storeid} = $bwlimit;
6352
6353                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6354             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6355                 my $virtdev = $1;
6356                 my $drive = parse_drive($virtdev, $2);
6357                 if (drive_is_cloudinit($drive)) {
6358                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6359                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6360                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6361
6362                     my $d = {
6363                         format => $format,
6364                         storeid => $opts->{storage} // $storeid,
6365                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6366                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6367                         name => "vm-$vmid-cloudinit",
6368                         is_cloudinit => 1,
6369                     };
6370                     $virtdev_hash->{$virtdev} = $d;
6371                 }
6372             }
6373         }
6374
6375         foreach my $key (keys %storage_limits) {
6376             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6377             next if !$limit;
6378             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6379             $storage_limits{$key} = $limit * 1024;
6380         }
6381
6382         foreach my $devname (keys %$devinfo) {
6383             die "found no device mapping information for device '$devname'\n"
6384                 if !$devinfo->{$devname}->{virtdev};
6385         }
6386
6387         # create empty/temp config
6388         if ($oldconf) {
6389             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6390             foreach_drive($oldconf, sub {
6391                 my ($ds, $drive) = @_;
6392
6393                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6394
6395                 my $volid = $drive->{file};
6396                 return if !$volid || $volid =~ m|^/|;
6397
6398                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6399                 return if !$path || !$owner || ($owner != $vmid);
6400
6401                 # Note: only delete disk we want to restore
6402                 # other volumes will become unused
6403                 if ($virtdev_hash->{$ds}) {
6404                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6405                     if (my $err = $@) {
6406                         warn $err;
6407                     }
6408                 }
6409             });
6410
6411             # delete vmstate files, after the restore we have no snapshots anymore
6412             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6413                 my $snap = $oldconf->{snapshots}->{$snapname};
6414                 if ($snap->{vmstate}) {
6415                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6416                     if (my $err = $@) {
6417                         warn $err;
6418                     }
6419                 }
6420             }
6421         }
6422
6423         my $map = {};
6424         foreach my $virtdev (sort keys %$virtdev_hash) {
6425             my $d = $virtdev_hash->{$virtdev};
6426             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6427             my $storeid = $d->{storeid};
6428             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6429
6430             my $map_opts = '';
6431             if (my $limit = $storage_limits{$storeid}) {
6432                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6433             }
6434
6435             # test if requested format is supported
6436             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6437             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6438             $d->{format} = $defFormat if !$supported;
6439
6440             my $name;
6441             if ($d->{is_cloudinit}) {
6442                 $name = $d->{name};
6443                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6444             }
6445
6446             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6447             print STDERR "new volume ID is '$volid'\n";
6448             $d->{volid} = $volid;
6449
6450             PVE::Storage::activate_volumes($cfg, [$volid]);
6451
6452             my $write_zeros = 1;
6453             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6454                 $write_zeros = 0;
6455             }
6456
6457             if (!$d->{is_cloudinit}) {
6458                 my $path = PVE::Storage::path($cfg, $volid);
6459
6460                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6461
6462                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6463             }
6464             $map->{$virtdev} = $volid;
6465         }
6466
6467         $fh->seek(0, 0) || die "seek failed - $!\n";
6468
6469         my $outfd = new IO::File ($tmpfn, "w") ||
6470             die "unable to write config for VM $vmid\n";
6471
6472         my $cookie = { netcount => 0 };
6473         while (defined(my $line = <$fh>)) {
6474             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6475         }
6476
6477         $fh->close();
6478         $outfd->close();
6479     };
6480
6481     eval {
6482         # enable interrupts
6483         local $SIG{INT} =
6484             local $SIG{TERM} =
6485             local $SIG{QUIT} =
6486             local $SIG{HUP} =
6487             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6488         local $SIG{ALRM} = sub { die "got timeout\n"; };
6489
6490         $oldtimeout = alarm($timeout);
6491
6492         my $parser = sub {
6493             my $line = shift;
6494
6495             print "$line\n";
6496
6497             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6498                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6499                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6500             } elsif ($line =~ m/^CTIME: /) {
6501                 # we correctly received the vma config, so we can disable
6502                 # the timeout now for disk allocation (set to 10 minutes, so
6503                 # that we always timeout if something goes wrong)
6504                 alarm(600);
6505                 &$print_devmap();
6506                 print $fifofh "done\n";
6507                 my $tmp = $oldtimeout || 0;
6508                 $oldtimeout = undef;
6509                 alarm($tmp);
6510                 close($fifofh);
6511             }
6512         };
6513
6514         print "restore vma archive: $dbg_cmdstring\n";
6515         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6516     };
6517     my $err = $@;
6518
6519     alarm($oldtimeout) if $oldtimeout;
6520
6521     my $vollist = [];
6522     foreach my $devname (keys %$devinfo) {
6523         my $volid = $devinfo->{$devname}->{volid};
6524         push @$vollist, $volid if $volid;
6525     }
6526
6527     PVE::Storage::deactivate_volumes($cfg, $vollist);
6528
6529     unlink $mapfifo;
6530
6531     if ($err) {
6532         rmtree $tmpdir;
6533         unlink $tmpfn;
6534
6535         foreach my $devname (keys %$devinfo) {
6536             my $volid = $devinfo->{$devname}->{volid};
6537             next if !$volid;
6538             eval {
6539                 if ($volid =~ m|^/|) {
6540                     unlink $volid || die 'unlink failed\n';
6541                 } else {
6542                     PVE::Storage::vdisk_free($cfg, $volid);
6543                 }
6544                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6545             };
6546             print STDERR "unable to cleanup '$volid' - $@" if $@;
6547         }
6548         die $err;
6549     }
6550
6551     rmtree $tmpdir;
6552
6553     rename($tmpfn, $conffile) ||
6554         die "unable to commit configuration file '$conffile'\n";
6555
6556     PVE::Cluster::cfs_update(); # make sure we read new file
6557
6558     eval { rescan($vmid, 1); };
6559     warn $@ if $@;
6560 }
6561
6562 sub restore_tar_archive {
6563     my ($archive, $vmid, $user, $opts) = @_;
6564
6565     if ($archive ne '-') {
6566         my $firstfile = tar_archive_read_firstfile($archive);
6567         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6568             if $firstfile ne 'qemu-server.conf';
6569     }
6570
6571     my $storecfg = PVE::Storage::config();
6572
6573     # destroy existing data - keep empty config
6574     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6575     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6576
6577     my $tocmd = "/usr/lib/qemu-server/qmextract";
6578
6579     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6580     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6581     $tocmd .= ' --prealloc' if $opts->{prealloc};
6582     $tocmd .= ' --info' if $opts->{info};
6583
6584     # tar option "xf" does not autodetect compression when read from STDIN,
6585     # so we pipe to zcat
6586     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6587         PVE::Tools::shellquote("--to-command=$tocmd");
6588
6589     my $tmpdir = "/var/tmp/vzdumptmp$$";
6590     mkpath $tmpdir;
6591
6592     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6593     local $ENV{VZDUMP_VMID} = $vmid;
6594     local $ENV{VZDUMP_USER} = $user;
6595
6596     my $conffile = PVE::QemuConfig->config_file($vmid);
6597     my $tmpfn = "$conffile.$$.tmp";
6598
6599     # disable interrupts (always do cleanups)
6600     local $SIG{INT} =
6601         local $SIG{TERM} =
6602         local $SIG{QUIT} =
6603         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6604
6605     eval {
6606         # enable interrupts
6607         local $SIG{INT} =
6608             local $SIG{TERM} =
6609             local $SIG{QUIT} =
6610             local $SIG{HUP} =
6611             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6612
6613         if ($archive eq '-') {
6614             print "extracting archive from STDIN\n";
6615             run_command($cmd, input => "<&STDIN");
6616         } else {
6617             print "extracting archive '$archive'\n";
6618             run_command($cmd);
6619         }
6620
6621         return if $opts->{info};
6622
6623         # read new mapping
6624         my $map = {};
6625         my $statfile = "$tmpdir/qmrestore.stat";
6626         if (my $fd = IO::File->new($statfile, "r")) {
6627             while (defined (my $line = <$fd>)) {
6628                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6629                     $map->{$1} = $2 if $1;
6630                 } else {
6631                     print STDERR "unable to parse line in statfile - $line\n";
6632                 }
6633             }
6634             $fd->close();
6635         }
6636
6637         my $confsrc = "$tmpdir/qemu-server.conf";
6638
6639         my $srcfd = new IO::File($confsrc, "r") ||
6640             die "unable to open file '$confsrc'\n";
6641
6642         my $outfd = new IO::File ($tmpfn, "w") ||
6643             die "unable to write config for VM $vmid\n";
6644
6645         my $cookie = { netcount => 0 };
6646         while (defined (my $line = <$srcfd>)) {
6647             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6648         }
6649
6650         $srcfd->close();
6651         $outfd->close();
6652     };
6653     my $err = $@;
6654
6655     if ($err) {
6656
6657         unlink $tmpfn;
6658
6659         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6660
6661         die $err;
6662     }
6663
6664     rmtree $tmpdir;
6665
6666     rename $tmpfn, $conffile ||
6667         die "unable to commit configuration file '$conffile'\n";
6668
6669     PVE::Cluster::cfs_update(); # make sure we read new file
6670
6671     eval { rescan($vmid, 1); };
6672     warn $@ if $@;
6673 };
6674
6675 sub foreach_storage_used_by_vm {
6676     my ($conf, $func) = @_;
6677
6678     my $sidhash = {};
6679
6680     foreach_drive($conf, sub {
6681         my ($ds, $drive) = @_;
6682         return if drive_is_cdrom($drive);
6683
6684         my $volid = $drive->{file};
6685
6686         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6687         $sidhash->{$sid} = $sid if $sid;
6688     });
6689
6690     foreach my $sid (sort keys %$sidhash) {
6691         &$func($sid);
6692     }
6693 }
6694
6695 sub do_snapshots_with_qemu {
6696     my ($storecfg, $volid) = @_;
6697
6698     my $storage_name = PVE::Storage::parse_volume_id($volid);
6699     my $scfg = $storecfg->{ids}->{$storage_name};
6700
6701     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6702         return 1;
6703     }
6704
6705     if ($volid =~ m/\.(qcow2|qed)$/){
6706         return 1;
6707     }
6708
6709     return undef;
6710 }
6711
6712 sub qga_check_running {
6713     my ($vmid, $nowarn) = @_;
6714
6715     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6716     if ($@) {
6717         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6718         return 0;
6719     }
6720     return 1;
6721 }
6722
6723 sub template_create {
6724     my ($vmid, $conf, $disk) = @_;
6725
6726     my $storecfg = PVE::Storage::config();
6727
6728     foreach_drive($conf, sub {
6729         my ($ds, $drive) = @_;
6730
6731         return if drive_is_cdrom($drive);
6732         return if $disk && $ds ne $disk;
6733
6734         my $volid = $drive->{file};
6735         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6736
6737         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6738         $drive->{file} = $voliddst;
6739         $conf->{$ds} = print_drive($vmid, $drive);
6740         PVE::QemuConfig->write_config($vmid, $conf);
6741     });
6742 }
6743
6744 sub convert_iscsi_path {
6745     my ($path) = @_;
6746
6747     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6748         my $portal = $1;
6749         my $target = $2;
6750         my $lun = $3;
6751
6752         my $initiator_name = get_initiator_name();
6753
6754         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6755                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6756     }
6757
6758     die "cannot convert iscsi path '$path', unkown format\n";
6759 }
6760
6761 sub qemu_img_convert {
6762     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6763
6764     my $storecfg = PVE::Storage::config();
6765     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6766     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6767
6768     if ($src_storeid && $dst_storeid) {
6769
6770         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6771
6772         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6773         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6774
6775         my $src_format = qemu_img_format($src_scfg, $src_volname);
6776         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6777
6778         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6779         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6780
6781         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6782         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6783
6784         my $cmd = [];
6785         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6786         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6787         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6788         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6789
6790         if ($src_is_iscsi) {
6791             push @$cmd, '--image-opts';
6792             $src_path = convert_iscsi_path($src_path);
6793         } else {
6794             push @$cmd, '-f', $src_format;
6795         }
6796
6797         if ($dst_is_iscsi) {
6798             push @$cmd, '--target-image-opts';
6799             $dst_path = convert_iscsi_path($dst_path);
6800         } else {
6801             push @$cmd, '-O', $dst_format;
6802         }
6803
6804         push @$cmd, $src_path;
6805
6806         if (!$dst_is_iscsi && $is_zero_initialized) {
6807             push @$cmd, "zeroinit:$dst_path";
6808         } else {
6809             push @$cmd, $dst_path;
6810         }
6811
6812         my $parser = sub {
6813             my $line = shift;
6814             if($line =~ m/\((\S+)\/100\%\)/){
6815                 my $percent = $1;
6816                 my $transferred = int($size * $percent / 100);
6817                 my $remaining = $size - $transferred;
6818
6819                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6820             }
6821
6822         };
6823
6824         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6825         my $err = $@;
6826         die "copy failed: $err" if $err;
6827     }
6828 }
6829
6830 sub qemu_img_format {
6831     my ($scfg, $volname) = @_;
6832
6833     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6834         return $1;
6835     } else {
6836         return "raw";
6837     }
6838 }
6839
6840 sub qemu_drive_mirror {
6841     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6842
6843     $jobs = {} if !$jobs;
6844
6845     my $qemu_target;
6846     my $format;
6847     $jobs->{"drive-$drive"} = {};
6848
6849     if ($dst_volid =~ /^nbd:/) {
6850         $qemu_target = $dst_volid;
6851         $format = "nbd";
6852     } else {
6853         my $storecfg = PVE::Storage::config();
6854         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6855
6856         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6857
6858         $format = qemu_img_format($dst_scfg, $dst_volname);
6859
6860         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6861
6862         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6863     }
6864
6865     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6866     $opts->{format} = $format if $format;
6867
6868     if (defined($bwlimit)) {
6869         $opts->{speed} = $bwlimit * 1024;
6870         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6871     } else {
6872         print "drive mirror is starting for drive-$drive\n";
6873     }
6874
6875     # if a job already runs for this device we get an error, catch it for cleanup
6876     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6877     if (my $err = $@) {
6878         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6879         warn "$@\n" if $@;
6880         die "mirroring error: $err\n";
6881     }
6882
6883     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6884 }
6885
6886 sub qemu_drive_mirror_monitor {
6887     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6888
6889     eval {
6890         my $err_complete = 0;
6891
6892         while (1) {
6893             die "storage migration timed out\n" if $err_complete > 300;
6894
6895             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6896
6897             my $running_mirror_jobs = {};
6898             foreach my $stat (@$stats) {
6899                 next if $stat->{type} ne 'mirror';
6900                 $running_mirror_jobs->{$stat->{device}} = $stat;
6901             }
6902
6903             my $readycounter = 0;
6904
6905             foreach my $job (keys %$jobs) {
6906
6907                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6908                     print "$job : finished\n";
6909                     delete $jobs->{$job};
6910                     next;
6911                 }
6912
6913                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6914
6915                 my $busy = $running_mirror_jobs->{$job}->{busy};
6916                 my $ready = $running_mirror_jobs->{$job}->{ready};
6917                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6918                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6919                     my $remaining = $total - $transferred;
6920                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6921
6922                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6923                 }
6924
6925                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6926             }
6927
6928             last if scalar(keys %$jobs) == 0;
6929
6930             if ($readycounter == scalar(keys %$jobs)) {
6931                 print "all mirroring jobs are ready \n";
6932                 last if $skipcomplete; #do the complete later
6933
6934                 if ($vmiddst && $vmiddst != $vmid) {
6935                     my $agent_running = $qga && qga_check_running($vmid);
6936                     if ($agent_running) {
6937                         print "freeze filesystem\n";
6938                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6939                     } else {
6940                         print "suspend vm\n";
6941                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6942                     }
6943
6944                     # if we clone a disk for a new target vm, we don't switch the disk
6945                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6946
6947                     if ($agent_running) {
6948                         print "unfreeze filesystem\n";
6949                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6950                     } else {
6951                         print "resume vm\n";
6952                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6953                     }
6954
6955                     last;
6956                 } else {
6957
6958                     foreach my $job (keys %$jobs) {
6959                         # try to switch the disk if source and destination are on the same guest
6960                         print "$job: Completing block job...\n";
6961
6962                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6963                         if ($@ =~ m/cannot be completed/) {
6964                             print "$job: Block job cannot be completed, try again.\n";
6965                             $err_complete++;
6966                         }else {
6967                             print "$job: Completed successfully.\n";
6968                             $jobs->{$job}->{complete} = 1;
6969                         }
6970                     }
6971                 }
6972             }
6973             sleep 1;
6974         }
6975     };
6976     my $err = $@;
6977
6978     if ($err) {
6979         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6980         die "mirroring error: $err";
6981     }
6982
6983 }
6984
6985 sub qemu_blockjobs_cancel {
6986     my ($vmid, $jobs) = @_;
6987
6988     foreach my $job (keys %$jobs) {
6989         print "$job: Cancelling block job\n";
6990         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6991         $jobs->{$job}->{cancel} = 1;
6992     }
6993
6994     while (1) {
6995         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6996
6997         my $running_jobs = {};
6998         foreach my $stat (@$stats) {
6999             $running_jobs->{$stat->{device}} = $stat;
7000         }
7001
7002         foreach my $job (keys %$jobs) {
7003
7004             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7005                 print "$job: Done.\n";
7006                 delete $jobs->{$job};
7007             }
7008         }
7009
7010         last if scalar(keys %$jobs) == 0;
7011
7012         sleep 1;
7013     }
7014 }
7015
7016 sub clone_disk {
7017     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7018         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7019
7020     my $newvolid;
7021
7022     if (!$full) {
7023         print "create linked clone of drive $drivename ($drive->{file})\n";
7024         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7025         push @$newvollist, $newvolid;
7026     } else {
7027
7028         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7029         $storeid = $storage if $storage;
7030
7031         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7032         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7033
7034         print "create full clone of drive $drivename ($drive->{file})\n";
7035         my $name = undef;
7036         if (drive_is_cloudinit($drive)) {
7037             $name = "vm-$newvmid-cloudinit";
7038             $snapname = undef;
7039             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7040             if ($dst_format ne 'raw') {
7041                 $name .= ".$dst_format";
7042             }
7043         }
7044         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7045         push @$newvollist, $newvolid;
7046
7047         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7048
7049         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7050         if (!$running || $snapname) {
7051             # TODO: handle bwlimits
7052             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7053         } else {
7054
7055             my $kvmver = get_running_qemu_version ($vmid);
7056             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7057                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7058                     if $drive->{iothread};
7059             }
7060
7061             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7062         }
7063     }
7064
7065     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7066
7067     my $disk = $drive;
7068     $disk->{format} = undef;
7069     $disk->{file} = $newvolid;
7070     $disk->{size} = $size;
7071
7072     return $disk;
7073 }
7074
7075 # this only works if VM is running
7076 sub get_current_qemu_machine {
7077     my ($vmid) = @_;
7078
7079     my $cmd = { execute => 'query-machines', arguments => {} };
7080     my $res = vm_qmp_command($vmid, $cmd);
7081
7082     my ($current, $default);
7083     foreach my $e (@$res) {
7084         $default = $e->{name} if $e->{'is-default'};
7085         $current = $e->{name} if $e->{'is-current'};
7086     }
7087
7088     # fallback to the default machine if current is not supported by qemu
7089     return $current || $default || 'pc';
7090 }
7091
7092 sub get_running_qemu_version {
7093     my ($vmid) = @_;
7094     my $cmd = { execute => 'query-version', arguments => {} };
7095     my $res = vm_qmp_command($vmid, $cmd);
7096     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7097 }
7098
7099 sub qemu_machine_feature_enabled {
7100     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7101
7102     my $current_major;
7103     my $current_minor;
7104
7105     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7106
7107         $current_major = $3;
7108         $current_minor = $4;
7109
7110     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7111
7112         $current_major = $1;
7113         $current_minor = $2;
7114     }
7115
7116     return 1 if $current_major > $version_major ||
7117                 ($current_major == $version_major &&
7118                  $current_minor >= $version_minor);
7119 }
7120
7121 sub qemu_machine_pxe {
7122     my ($vmid, $conf, $machine) = @_;
7123
7124     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7125
7126     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7127         $machine .= '.pxe';
7128     }
7129
7130     return $machine;
7131 }
7132
7133 sub qemu_use_old_bios_files {
7134     my ($machine_type) = @_;
7135
7136     return if !$machine_type;
7137
7138     my $use_old_bios_files = undef;
7139
7140     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7141         $machine_type = $1;
7142         $use_old_bios_files = 1;
7143     } else {
7144         my $kvmver = kvm_user_version();
7145         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7146         # load new efi bios files on migration. So this hack is required to allow
7147         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7148         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7149         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7150     }
7151
7152     return ($use_old_bios_files, $machine_type);
7153 }
7154
7155 sub create_efidisk($$$$$) {
7156     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7157
7158     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7159     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7160
7161     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7162     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7163     PVE::Storage::activate_volumes($storecfg, [$volid]);
7164
7165     my $path = PVE::Storage::path($storecfg, $volid);
7166     eval {
7167         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7168     };
7169     die "Copying EFI vars image failed: $@" if $@;
7170
7171     return ($volid, $vars_size);
7172 }
7173
7174 sub vm_iothreads_list {
7175     my ($vmid) = @_;
7176
7177     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7178
7179     my $iothreads = {};
7180     foreach my $iothread (@$res) {
7181         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7182     }
7183
7184     return $iothreads;
7185 }
7186
7187 sub scsihw_infos {
7188     my ($conf, $drive) = @_;
7189
7190     my $maxdev = 0;
7191
7192     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7193         $maxdev = 7;
7194     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7195         $maxdev = 1;
7196     } else {
7197         $maxdev = 256;
7198     }
7199
7200     my $controller = int($drive->{index} / $maxdev);
7201     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7202
7203     return ($maxdev, $controller, $controller_prefix);
7204 }
7205
7206 sub add_hyperv_enlightenments {
7207     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7208
7209     return if $winversion < 6;
7210     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7211
7212     if ($gpu_passthrough || defined($hv_vendor_id)) {
7213         $hv_vendor_id //= 'proxmox';
7214         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7215     }
7216
7217     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7218         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7219         push @$cpuFlags , 'hv_vapic';
7220         push @$cpuFlags , 'hv_time';
7221     } else {
7222         push @$cpuFlags , 'hv_spinlocks=0xffff';
7223     }
7224
7225     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7226         push @$cpuFlags , 'hv_reset';
7227         push @$cpuFlags , 'hv_vpindex';
7228         push @$cpuFlags , 'hv_runtime';
7229     }
7230
7231     if ($winversion >= 7) {
7232         push @$cpuFlags , 'hv_relaxed';
7233
7234         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7235             push @$cpuFlags , 'hv_synic';
7236             push @$cpuFlags , 'hv_stimer';
7237         }
7238
7239         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7240             push @$cpuFlags , 'hv_tlbflush';
7241             push @$cpuFlags , 'hv_ipi';
7242             # FIXME: AMD does not supports this currently, only add with special flag??
7243             #push @$cpuFlags , 'hv_evmcs';
7244         }
7245     }
7246 }
7247
7248 sub windows_version {
7249     my ($ostype) = @_;
7250
7251     return 0 if !$ostype;
7252
7253     my $winversion = 0;
7254
7255     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7256         $winversion = 5;
7257     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7258         $winversion = 6;
7259     } elsif ($ostype =~ m/^win(\d+)$/) {
7260         $winversion = $1;
7261     }
7262
7263     return $winversion;
7264 }
7265
7266 sub resolve_dst_disk_format {
7267         my ($storecfg, $storeid, $src_volname, $format) = @_;
7268         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7269
7270         if (!$format) {
7271             # if no target format is specified, use the source disk format as hint
7272             if ($src_volname) {
7273                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7274                 $format = qemu_img_format($scfg, $src_volname);
7275             } else {
7276                 return $defFormat;
7277             }
7278         }
7279
7280         # test if requested format is supported - else use default
7281         my $supported = grep { $_ eq $format } @$validFormats;
7282         $format = $defFormat if !$supported;
7283         return $format;
7284 }
7285
7286 sub resolve_first_disk {
7287     my $conf = shift;
7288     my @disks = PVE::QemuServer::valid_drive_names();
7289     my $firstdisk;
7290     foreach my $ds (reverse @disks) {
7291         next if !$conf->{$ds};
7292         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7293         next if PVE::QemuServer::drive_is_cdrom($disk);
7294         $firstdisk = $ds;
7295     }
7296     return $firstdisk;
7297 }
7298
7299 sub generate_uuid {
7300     my ($uuid, $uuid_str);
7301     UUID::generate($uuid);
7302     UUID::unparse($uuid, $uuid_str);
7303     return $uuid_str;
7304 }
7305
7306 sub generate_smbios1_uuid {
7307     return "uuid=".generate_uuid();
7308 }
7309
7310 sub nbd_stop {
7311     my ($vmid) = @_;
7312
7313     vm_mon_cmd($vmid, 'nbd-server-stop');
7314 }
7315
7316 # bash completion helper
7317
7318 sub complete_backup_archives {
7319     my ($cmdname, $pname, $cvalue) = @_;
7320
7321     my $cfg = PVE::Storage::config();
7322
7323     my $storeid;
7324
7325     if ($cvalue =~ m/^([^:]+):/) {
7326         $storeid = $1;
7327     }
7328
7329     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7330
7331     my $res = [];
7332     foreach my $id (keys %$data) {
7333         foreach my $item (@{$data->{$id}}) {
7334             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7335             push @$res, $item->{volid} if defined($item->{volid});
7336         }
7337     }
7338
7339     return $res;
7340 }
7341
7342 my $complete_vmid_full = sub {
7343     my ($running) = @_;
7344
7345     my $idlist = vmstatus();
7346
7347     my $res = [];
7348
7349     foreach my $id (keys %$idlist) {
7350         my $d = $idlist->{$id};
7351         if (defined($running)) {
7352             next if $d->{template};
7353             next if $running && $d->{status} ne 'running';
7354             next if !$running && $d->{status} eq 'running';
7355         }
7356         push @$res, $id;
7357
7358     }
7359     return $res;
7360 };
7361
7362 sub complete_vmid {
7363     return &$complete_vmid_full();
7364 }
7365
7366 sub complete_vmid_stopped {
7367     return &$complete_vmid_full(0);
7368 }
7369
7370 sub complete_vmid_running {
7371     return &$complete_vmid_full(1);
7372 }
7373
7374 sub complete_storage {
7375
7376     my $cfg = PVE::Storage::config();
7377     my $ids = $cfg->{ids};
7378
7379     my $res = [];
7380     foreach my $sid (keys %$ids) {
7381         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7382         next if !$ids->{$sid}->{content}->{images};
7383         push @$res, $sid;
7384     }
7385
7386     return $res;
7387 }
7388
7389 1;