PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
 171
 172 my $cpu_fmt = {
 173     cputype => {
 174         description => "Emulated CPU type.",
 175         type => 'string',
 176         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 177         default => 'kvm64',
 178         default_key => 1,
 179     },
 180     hidden => {
 181         description => "Do not identify as a KVM virtual machine.",
 182         type => 'boolean',
 183         optional => 1,
 184         default => 0
 185     },
 186     'hv-vendor-id' => {
 187         type => 'string',
 188         pattern => qr/[a-zA-Z0-9]{1,12}/,
 189         format_description => 'vendor-id',
 190         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 191         optional => 1,
 192     },
 193     flags => {
 194         description => "List of additional CPU flags separated by ';'."
 195                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 196                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
 197         format_description => '+FLAG[;-FLAG...]',
 198         type => 'string',
 199         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 200         optional => 1,
 201     },
 202 };
 203
 204 my $watchdog_fmt = {
 205     model => {
 206         default_key => 1,
 207         type => 'string',
 208         enum => [qw(i6300esb ib700)],
 209         description => "Watchdog type to emulate.",
 210         default => 'i6300esb',
 211         optional => 1,
 212     },
 213     action => {
 214         type => 'string',
 215         enum => [qw(reset shutdown poweroff pause debug none)],
 216         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 217         optional => 1,
 218     },
 219 };
 220 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 221
 222 my $agent_fmt = {
 223     enabled => {
 224         description => "Enable/disable Qemu GuestAgent.",
 225         type => 'boolean',
 226         default => 0,
 227         default_key => 1,
 228     },
 229     fstrim_cloned_disks => {
 230         description => "Run fstrim after cloning/moving a disk.",
 231         type => 'boolean',
 232         optional => 1,
 233         default => 0
 234     },
 235 };
 236
 237 my $vga_fmt = {
 238     type => {
 239         description => "Select the VGA type.",
 240         type => 'string',
 241         default => 'std',
 242         optional => 1,
 243         default_key => 1,
 244         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 245     },
 246     memory => {
 247         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 248         type => 'integer',
 249         optional => 1,
 250         minimum => 4,
 251         maximum => 512,
 252     },
 253 };
 254
 255 my $ivshmem_fmt = {
 256     size => {
 257         type => 'integer',
 258         minimum => 1,
 259         description => "The size of the file in MB.",
 260     },
 261     name => {
 262         type => 'string',
 263         pattern => '[a-zA-Z0-9\-]+',
 264         optional => 1,
 265         format_description => 'string',
 266         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 267     },
 268 };
 269
 270 my $confdesc = {
 271     onboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Specifies whether a VM will be started during system bootup.",
 275         default => 0,
 276     },
 277     autostart => {
 278         optional => 1,
 279         type => 'boolean',
 280         description => "Automatic restart after crash (currently ignored).",
 281         default => 0,
 282     },
 283     hotplug => {
 284         optional => 1,
 285         type => 'string', format => 'pve-hotplug-features',
 286         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 287         default => 'network,disk,usb',
 288     },
 289     reboot => {
 290         optional => 1,
 291         type => 'boolean',
 292         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 293         default => 1,
 294     },
 295     lock => {
 296         optional => 1,
 297         type => 'string',
 298         description => "Lock/unlock the VM.",
 299         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 300     },
 301     cpulimit => {
 302         optional => 1,
 303         type => 'number',
 304         description => "Limit of CPU usage.",
 305         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 306         minimum => 0,
 307         maximum => 128,
 308         default => 0,
 309     },
 310     cpuunits => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "CPU weight for a VM.",
 314         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 315         minimum => 2,
 316         maximum => 262144,
 317         default => 1024,
 318     },
 319     memory => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 323         minimum => 16,
 324         default => 512,
 325     },
 326     balloon => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 330         minimum => 0,
 331     },
 332     shares => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 336         minimum => 0,
 337         maximum => 50000,
 338         default => 1000,
 339     },
 340     keyboard => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 344                        "It should not be necessary to set it.",
 345         enum => PVE::Tools::kvmkeymaplist(),
 346         default => undef,
 347     },
 348     name => {
 349         optional => 1,
 350         type => 'string', format => 'dns-name',
 351         description => "Set a name for the VM. Only used on the configuration web interface.",
 352     },
 353     scsihw => {
 354         optional => 1,
 355         type => 'string',
 356         description => "SCSI controller model",
 357         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 358         default => 'lsi',
 359     },
 360     description => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 364     },
 365     ostype => {
 366         optional => 1,
 367         type => 'string',
 368         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 369         description => "Specify guest operating system.",
 370         verbose_description => <<EODESC,
 371 Specify guest operating system. This is used to enable special
 372 optimization/features for specific operating systems:
 373
 374 [horizontal]
 375 other;; unspecified OS
 376 wxp;; Microsoft Windows XP
 377 w2k;; Microsoft Windows 2000
 378 w2k3;; Microsoft Windows 2003
 379 w2k8;; Microsoft Windows 2008
 380 wvista;; Microsoft Windows Vista
 381 win7;; Microsoft Windows 7
 382 win8;; Microsoft Windows 8/2012/2012r2
 383 win10;; Microsoft Windows 10/2016
 384 l24;; Linux 2.4 Kernel
 385 l26;; Linux 2.6/3.X Kernel
 386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 387 EODESC
 388     },
 389     boot => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 393         pattern => '[acdn]{1,4}',
 394         default => 'cdn',
 395     },
 396     bootdisk => {
 397         optional => 1,
 398         type => 'string', format => 'pve-qm-bootdisk',
 399         description => "Enable booting from specified disk.",
 400         pattern => '(ide|sata|scsi|virtio)\d+',
 401     },
 402     smp => {
 403         optional => 1,
 404         type => 'integer',
 405         description => "The number of CPUs. Please use option -sockets instead.",
 406         minimum => 1,
 407         default => 1,
 408     },
 409     sockets => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "The number of CPU sockets.",
 413         minimum => 1,
 414         default => 1,
 415     },
 416     cores => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "The number of cores per socket.",
 420         minimum => 1,
 421         default => 1,
 422     },
 423     numa => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable NUMA.",
 427         default => 0,
 428     },
 429     hugepages => {
 430         optional => 1,
 431         type => 'string',
 432         description => "Enable/disable hugepages memory.",
 433         enum => [qw(any 2 1024)],
 434     },
 435     vcpus => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Number of hotplugged vcpus.",
 439         minimum => 1,
 440         default => 0,
 441     },
 442     acpi => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable ACPI.",
 446         default => 1,
 447     },
 448     agent => {
 449         optional => 1,
 450         description => "Enable/disable Qemu GuestAgent and its properties.",
 451         type => 'string',
 452         format => $agent_fmt,
 453     },
 454     kvm => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Enable/disable KVM hardware virtualization.",
 458         default => 1,
 459     },
 460     tdf => {
 461         optional => 1,
 462         type => 'boolean',
 463         description => "Enable/disable time drift fix.",
 464         default => 0,
 465     },
 466     localtime => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 470     },
 471     freeze => {
 472         optional => 1,
 473         type => 'boolean',
 474         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 475     },
 476     vga => {
 477         optional => 1,
 478         type => 'string', format => $vga_fmt,
 479         description => "Configure the VGA hardware.",
 480         verbose_description => "Configure the VGA Hardware. If you want to use ".
 481             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 482             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 483             "is 'std' for all OS types besides some Windows versions (XP and " .
 484             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 485             "display server. For win* OS you can select how many independent " .
 486             "displays you want, Linux guests can add displays them self.\n".
 487             "You can also run without any graphic card, using a serial device as terminal.",
 488     },
 489     watchdog => {
 490         optional => 1,
 491         type => 'string', format => 'pve-qm-watchdog',
 492         description => "Create a virtual hardware watchdog device.",
 493         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 494             " (by a guest action), the watchdog must be periodically polled " .
 495             "by an agent inside the guest or else the watchdog will reset " .
 496             "the guest (or execute the respective action specified)",
 497     },
 498     startdate => {
 499         optional => 1,
 500         type => 'string',
 501         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 502         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 503         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 504         default => 'now',
 505     },
 506     startup =>  get_standard_option('pve-startup-order'),
 507     template => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable Template.",
 511         default => 0,
 512     },
 513     args => {
 514         optional => 1,
 515         type => 'string',
 516         description => "Arbitrary arguments passed to kvm.",
 517         verbose_description => <<EODESCR,
 518 Arbitrary arguments passed to kvm, for example:
 519
 520 args: -no-reboot -no-hpet
 521
 522 NOTE: this option is for experts only.
 523 EODESCR
 524     },
 525     tablet => {
 526         optional => 1,
 527         type => 'boolean',
 528         default => 1,
 529         description => "Enable/disable the USB tablet device.",
 530         verbose_description => "Enable/disable the USB tablet device. This device is " .
 531             "usually needed to allow absolute mouse positioning with VNC. " .
 532             "Else the mouse runs out of sync with normal VNC clients. " .
 533             "If you're running lots of console-only guests on one host, " .
 534             "you may consider disabling this to save some context switches. " .
 535             "This is turned off by default if you use spice (-vga=qxl).",
 536     },
 537     migrate_speed => {
 538         optional => 1,
 539         type => 'integer',
 540         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 541         minimum => 0,
 542         default => 0,
 543     },
 544     migrate_downtime => {
 545         optional => 1,
 546         type => 'number',
 547         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 548         minimum => 0,
 549         default => 0.1,
 550     },
 551     cdrom => {
 552         optional => 1,
 553         type => 'string', format => 'pve-qm-ide',
 554         typetext => '<volume>',
 555         description => "This is an alias for option -ide2",
 556     },
 557     cpu => {
 558         optional => 1,
 559         description => "Emulated CPU type.",
 560         type => 'string',
 561         format => $cpu_fmt,
 562     },
 563     parent => get_standard_option('pve-snapshot-name', {
 564         optional => 1,
 565         description => "Parent snapshot name. This is used internally, and should not be modified.",
 566     }),
 567     snaptime => {
 568         optional => 1,
 569         description => "Timestamp for snapshots.",
 570         type => 'integer',
 571         minimum => 0,
 572     },
 573     vmstate => {
 574         optional => 1,
 575         type => 'string', format => 'pve-volume-id',
 576         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 577     },
 578     vmstatestorage => get_standard_option('pve-storage-id', {
 579         description => "Default storage for VM state volumes/files.",
 580         optional => 1,
 581     }),
 582     runningmachine => get_standard_option('pve-qemu-machine', {
 583         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 584     }),
 585     machine => get_standard_option('pve-qemu-machine'),
 586     arch => {
 587         description => "Virtual processor architecture. Defaults to the host.",
 588         optional => 1,
 589         type => 'string',
 590         enum => [qw(x86_64 aarch64)],
 591     },
 592     smbios1 => {
 593         description => "Specify SMBIOS type 1 fields.",
 594         type => 'string', format => 'pve-qm-smbios1',
 595         maxLength => 512,
 596         optional => 1,
 597     },
 598     protection => {
 599         optional => 1,
 600         type => 'boolean',
 601         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 602         default => 0,
 603     },
 604     bios => {
 605         optional => 1,
 606         type => 'string',
 607         enum => [ qw(seabios ovmf) ],
 608         description => "Select BIOS implementation.",
 609         default => 'seabios',
 610     },
 611     vmgenid => {
 612         type => 'string',
 613         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 614         format_description => 'UUID',
 615         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 616         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 617             " 128-bit integer value identifier to the guest OS. This allows to".
 618             " notify the guest operating system when the virtual machine is".
 619             " executed with a different configuration (e.g. snapshot execution".
 620             " or creation from a template). The guest operating system notices".
 621             " the change, and is then able to react as appropriate by marking".
 622             " its copies of distributed databases as dirty, re-initializing its".
 623             " random number generator, etc.\n".
 624             "Note that auto-creation only works when done throug API/CLI create".
 625             " or update methods, but not when manually editing the config file.",
 626         default => "1 (autogenerated)",
 627         optional => 1,
 628     },
 629     hookscript => {
 630         type => 'string',
 631         format => 'pve-volume-id',
 632         optional => 1,
 633         description => "Script that will be executed during various steps in the vms lifetime.",
 634     },
 635     ivshmem => {
 636         type => 'string',
 637         format => $ivshmem_fmt,
 638         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 639         optional => 1,
 640     }
 641 };
 642
 643 my $cicustom_fmt = {
 644     meta => {
 645         type => 'string',
 646         optional => 1,
 647         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 648         format => 'pve-volume-id',
 649         format_description => 'volume',
 650     },
 651     network => {
 652         type => 'string',
 653         optional => 1,
 654         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 655         format => 'pve-volume-id',
 656         format_description => 'volume',
 657     },
 658     user => {
 659         type => 'string',
 660         optional => 1,
 661         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 662         format => 'pve-volume-id',
 663         format_description => 'volume',
 664     },
 665 };
 666 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 667
 668 my $confdesc_cloudinit = {
 669     citype => {
 670         optional => 1,
 671         type => 'string',
 672         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 673         enum => ['configdrive2', 'nocloud'],
 674     },
 675     ciuser => {
 676         optional => 1,
 677         type => 'string',
 678         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 679     },
 680     cipassword => {
 681         optional => 1,
 682         type => 'string',
 683         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 684     },
 685     cicustom => {
 686         optional => 1,
 687         type => 'string',
 688         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 689         format => 'pve-qm-cicustom',
 690     },
 691     searchdomain => {
 692         optional => 1,
 693         type => 'string',
 694         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 695     },
 696     nameserver => {
 697         optional => 1,
 698         type => 'string', format => 'address-list',
 699         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 700     },
 701     sshkeys => {
 702         optional => 1,
 703         type => 'string',
 704         format => 'urlencoded',
 705         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 706     },
 707 };
 708
 709 # what about other qemu settings ?
 710 #cpu => 'string',
 711 #machine => 'string',
 712 #fda => 'file',
 713 #fdb => 'file',
 714 #mtdblock => 'file',
 715 #sd => 'file',
 716 #pflash => 'file',
 717 #snapshot => 'bool',
 718 #bootp => 'file',
 719 ##tftp => 'dir',
 720 ##smb => 'dir',
 721 #kernel => 'file',
 722 #append => 'string',
 723 #initrd => 'file',
 724 ##soundhw => 'string',
 725
 726 while (my ($k, $v) = each %$confdesc) {
 727     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 728 }
 729
 730 my $MAX_IDE_DISKS = 4;
 731 my $MAX_SCSI_DISKS = 14;
 732 my $MAX_VIRTIO_DISKS = 16;
 733 my $MAX_SATA_DISKS = 6;
 734 my $MAX_USB_DEVICES = 5;
 735 my $MAX_NETS = 32;
 736 my $MAX_UNUSED_DISKS = 256;
 737 my $MAX_HOSTPCI_DEVICES = 4;
 738 my $MAX_SERIAL_PORTS = 4;
 739 my $MAX_PARALLEL_PORTS = 3;
 740 my $MAX_NUMA = 8;
 741
 742 my $numa_fmt = {
 743     cpus => {
 744         type => "string",
 745         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 746         description => "CPUs accessing this NUMA node.",
 747         format_description => "id[-id];...",
 748     },
 749     memory => {
 750         type => "number",
 751         description => "Amount of memory this NUMA node provides.",
 752         optional => 1,
 753     },
 754     hostnodes => {
 755         type => "string",
 756         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 757         description => "Host NUMA nodes to use.",
 758         format_description => "id[-id];...",
 759         optional => 1,
 760     },
 761     policy => {
 762         type => 'string',
 763         enum => [qw(preferred bind interleave)],
 764         description => "NUMA allocation policy.",
 765         optional => 1,
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 769 my $numadesc = {
 770     optional => 1,
 771     type => 'string', format => $numa_fmt,
 772     description => "NUMA topology.",
 773 };
 774 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 775
 776 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 777     $confdesc->{"numa$i"} = $numadesc;
 778 }
 779
 780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 781                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 782                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 784
 785 my $net_fmt_bridge_descr = <<__EOD__;
 786 Bridge to attach the network device to. The Proxmox VE standard bridge
 787 is called 'vmbr0'.
 788
 789 If you do not specify a bridge, we create a kvm user (NATed) network
 790 device, which provides DHCP and DNS services. The following addresses
 791 are used:
 792
 793  10.0.2.2   Gateway
 794  10.0.2.3   DNS Server
 795  10.0.2.4   SMB Server
 796
 797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 798 __EOD__
 799
 800 my $net_fmt = {
 801     macaddr  => get_standard_option('mac-addr', {
 802         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 803     }),
 804     model => {
 805         type => 'string',
 806         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 807         enum => $nic_model_list,
 808         default_key => 1,
 809     },
 810     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 811     bridge => {
 812         type => 'string',
 813         description => $net_fmt_bridge_descr,
 814         format_description => 'bridge',
 815         optional => 1,
 816     },
 817     queues => {
 818         type => 'integer',
 819         minimum => 0, maximum => 16,
 820         description => 'Number of packet queues to be used on the device.',
 821         optional => 1,
 822     },
 823     rate => {
 824         type => 'number',
 825         minimum => 0,
 826         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 827         optional => 1,
 828     },
 829     tag => {
 830         type => 'integer',
 831         minimum => 1, maximum => 4094,
 832         description => 'VLAN tag to apply to packets on this interface.',
 833         optional => 1,
 834     },
 835     trunks => {
 836         type => 'string',
 837         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 838         description => 'VLAN trunks to pass through this interface.',
 839         format_description => 'vlanid[;vlanid...]',
 840         optional => 1,
 841     },
 842     firewall => {
 843         type => 'boolean',
 844         description => 'Whether this interface should be protected by the firewall.',
 845         optional => 1,
 846     },
 847     link_down => {
 848         type => 'boolean',
 849         description => 'Whether this interface should be disconnected (like pulling the plug).',
 850         optional => 1,
 851     },
 852 };
 853
 854 my $netdesc = {
 855     optional => 1,
 856     type => 'string', format => $net_fmt,
 857     description => "Specify network devices.",
 858 };
 859
 860 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 861
 862 my $ipconfig_fmt = {
 863     ip => {
 864         type => 'string',
 865         format => 'pve-ipv4-config',
 866         format_description => 'IPv4Format/CIDR',
 867         description => 'IPv4 address in CIDR format.',
 868         optional => 1,
 869         default => 'dhcp',
 870     },
 871     gw => {
 872         type => 'string',
 873         format => 'ipv4',
 874         format_description => 'GatewayIPv4',
 875         description => 'Default gateway for IPv4 traffic.',
 876         optional => 1,
 877         requires => 'ip',
 878     },
 879     ip6 => {
 880         type => 'string',
 881         format => 'pve-ipv6-config',
 882         format_description => 'IPv6Format/CIDR',
 883         description => 'IPv6 address in CIDR format.',
 884         optional => 1,
 885         default => 'dhcp',
 886     },
 887     gw6 => {
 888         type => 'string',
 889         format => 'ipv6',
 890         format_description => 'GatewayIPv6',
 891         description => 'Default gateway for IPv6 traffic.',
 892         optional => 1,
 893         requires => 'ip6',
 894     },
 895 };
 896 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 897 my $ipconfigdesc = {
 898     optional => 1,
 899     type => 'string', format => 'pve-qm-ipconfig',
 900     description => <<'EODESCR',
 901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 902
 903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 904
 905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 907
 908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 909 EODESCR
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 912
 913 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 914     $confdesc->{"net$i"} = $netdesc;
 915     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 916 }
 917
 918 foreach my $key (keys %$confdesc_cloudinit) {
 919     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 920 }
 921
 922 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 923 sub verify_volume_id_or_qm_path {
 924     my ($volid, $noerr) = @_;
 925
 926     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 927         return $volid;
 928     }
 929
 930     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 931     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 932     if ($@) {
 933         return undef if $noerr;
 934         die $@;
 935     }
 936     return $volid;
 937 }
 938
 939 my $drivename_hash;
 940
 941 my %drivedesc_base = (
 942     volume => { alias => 'file' },
 943     file => {
 944         type => 'string',
 945         format => 'pve-volume-id-or-qm-path',
 946         default_key => 1,
 947         format_description => 'volume',
 948         description => "The drive's backing volume.",
 949     },
 950     media => {
 951         type => 'string',
 952         enum => [qw(cdrom disk)],
 953         description => "The drive's media type.",
 954         default => 'disk',
 955         optional => 1
 956     },
 957     cyls => {
 958         type => 'integer',
 959         description => "Force the drive's physical geometry to have a specific cylinder count.",
 960         optional => 1
 961     },
 962     heads => {
 963         type => 'integer',
 964         description => "Force the drive's physical geometry to have a specific head count.",
 965         optional => 1
 966     },
 967     secs => {
 968         type => 'integer',
 969         description => "Force the drive's physical geometry to have a specific sector count.",
 970         optional => 1
 971     },
 972     trans => {
 973         type => 'string',
 974         enum => [qw(none lba auto)],
 975         description => "Force disk geometry bios translation mode.",
 976         optional => 1,
 977     },
 978     snapshot => {
 979         type => 'boolean',
 980         description => "Controls qemu's snapshot mode feature."
 981             . " If activated, changes made to the disk are temporary and will"
 982             . " be discarded when the VM is shutdown.",
 983         optional => 1,
 984     },
 985     cache => {
 986         type => 'string',
 987         enum => [qw(none writethrough writeback unsafe directsync)],
 988         description => "The drive's cache mode",
 989         optional => 1,
 990     },
 991     format => get_standard_option('pve-qm-image-format'),
 992     size => {
 993         type => 'string',
 994         format => 'disk-size',
 995         format_description => 'DiskSize',
 996         description => "Disk size. This is purely informational and has no effect.",
 997         optional => 1,
 998     },
 999     backup => {
1000         type => 'boolean',
1001         description => "Whether the drive should be included when making backups.",
1002         optional => 1,
1003     },
1004     replicate => {
1005         type => 'boolean',
1006         description => 'Whether the drive should considered for replication jobs.',
1007         optional => 1,
1008         default => 1,
1009     },
1010     rerror => {
1011         type => 'string',
1012         enum => [qw(ignore report stop)],
1013         description => 'Read error action.',
1014         optional => 1,
1015     },
1016     werror => {
1017         type => 'string',
1018         enum => [qw(enospc ignore report stop)],
1019         description => 'Write error action.',
1020         optional => 1,
1021     },
1022     aio => {
1023         type => 'string',
1024         enum => [qw(native threads)],
1025         description => 'AIO type to use.',
1026         optional => 1,
1027     },
1028     discard => {
1029         type => 'string',
1030         enum => [qw(ignore on)],
1031         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1032         optional => 1,
1033     },
1034     detect_zeroes => {
1035         type => 'boolean',
1036         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1037         optional => 1,
1038     },
1039     serial => {
1040         type => 'string',
1041         format => 'urlencoded',
1042         format_description => 'serial',
1043         maxLength => 20*3, # *3 since it's %xx url enoded
1044         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1045         optional => 1,
1046     },
1047     shared => {
1048         type => 'boolean',
1049         description => 'Mark this locally-managed volume as available on all nodes',
1050         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1051         optional => 1,
1052         default => 0,
1053     }
1054 );
1055
1056 my %iothread_fmt = ( iothread => {
1057         type => 'boolean',
1058         description => "Whether to use iothreads for this drive",
1059         optional => 1,
1060 });
1061
1062 my %model_fmt = (
1063     model => {
1064         type => 'string',
1065         format => 'urlencoded',
1066         format_description => 'model',
1067         maxLength => 40*3, # *3 since it's %xx url enoded
1068         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1069         optional => 1,
1070     },
1071 );
1072
1073 my %queues_fmt = (
1074     queues => {
1075         type => 'integer',
1076         description => "Number of queues.",
1077         minimum => 2,
1078         optional => 1
1079     }
1080 );
1081
1082 my %scsiblock_fmt = (
1083     scsiblock => {
1084         type => 'boolean',
1085         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1086         optional => 1,
1087         default => 0,
1088     },
1089 );
1090
1091 my %ssd_fmt = (
1092     ssd => {
1093         type => 'boolean',
1094         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1095         optional => 1,
1096     },
1097 );
1098
1099 my %wwn_fmt = (
1100     wwn => {
1101         type => 'string',
1102         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1103         format_description => 'wwn',
1104         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1105         optional => 1,
1106     },
1107 );
1108
1109 my $add_throttle_desc = sub {
1110     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1111     my $d = {
1112         type => $type,
1113         format_description => $unit,
1114         description => "Maximum $what in $longunit.",
1115         optional => 1,
1116     };
1117     $d->{minimum} = $minimum if defined($minimum);
1118     $drivedesc_base{$key} = $d;
1119 };
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1122 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1123 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1124 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1130
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1138
1139 # burst lengths
1140 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1146
1147 # legacy support
1148 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1152
1153 my $ide_fmt = {
1154     %drivedesc_base,
1155     %model_fmt,
1156     %ssd_fmt,
1157     %wwn_fmt,
1158 };
1159 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1160
1161 my $idedesc = {
1162     optional => 1,
1163     type => 'string', format => $ide_fmt,
1164     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1165 };
1166 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1167
1168 my $scsi_fmt = {
1169     %drivedesc_base,
1170     %iothread_fmt,
1171     %queues_fmt,
1172     %scsiblock_fmt,
1173     %ssd_fmt,
1174     %wwn_fmt,
1175 };
1176 my $scsidesc = {
1177     optional => 1,
1178     type => 'string', format => $scsi_fmt,
1179     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1182
1183 my $sata_fmt = {
1184     %drivedesc_base,
1185     %ssd_fmt,
1186     %wwn_fmt,
1187 };
1188 my $satadesc = {
1189     optional => 1,
1190     type => 'string', format => $sata_fmt,
1191     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1192 };
1193 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1194
1195 my $virtio_fmt = {
1196     %drivedesc_base,
1197     %iothread_fmt,
1198 };
1199 my $virtiodesc = {
1200     optional => 1,
1201     type => 'string', format => $virtio_fmt,
1202     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1205
1206 my $alldrive_fmt = {
1207     %drivedesc_base,
1208     %iothread_fmt,
1209     %model_fmt,
1210     %queues_fmt,
1211     %scsiblock_fmt,
1212     %ssd_fmt,
1213     %wwn_fmt,
1214 };
1215
1216 my $efidisk_fmt = {
1217     volume => { alias => 'file' },
1218     file => {
1219         type => 'string',
1220         format => 'pve-volume-id-or-qm-path',
1221         default_key => 1,
1222         format_description => 'volume',
1223         description => "The drive's backing volume.",
1224     },
1225     format => get_standard_option('pve-qm-image-format'),
1226     size => {
1227         type => 'string',
1228         format => 'disk-size',
1229         format_description => 'DiskSize',
1230         description => "Disk size. This is purely informational and has no effect.",
1231         optional => 1,
1232     },
1233 };
1234
1235 my $efidisk_desc = {
1236     optional => 1,
1237     type => 'string', format => $efidisk_fmt,
1238     description => "Configure a Disk for storing EFI vars",
1239 };
1240
1241 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1242
1243 my $usb_fmt = {
1244     host => {
1245         default_key => 1,
1246         type => 'string', format => 'pve-qm-usb-device',
1247         format_description => 'HOSTUSBDEVICE|spice',
1248         description => <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1250
1251  'bus-port(.port)*' (decimal numbers) or
1252  'vendor_id:product_id' (hexadeciaml numbers) or
1253  'spice'
1254
1255 You can use the 'lsusb -t' command to list existing usb devices.
1256
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1258
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1260 EODESCR
1261     },
1262     usb3 => {
1263         optional => 1,
1264         type => 'boolean',
1265         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1266         default => 0,
1267     },
1268 };
1269
1270 my $usbdesc = {
1271     optional => 1,
1272     type => 'string', format => $usb_fmt,
1273     description => "Configure an USB device (n is 0 to 4).",
1274 };
1275 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1276
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1278 my $hostpci_fmt = {
1279     host => {
1280         default_key => 1,
1281         type => 'string',
1282         pattern => qr/$PCIRE(;$PCIRE)*/,
1283         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1284         description => <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1287
1288 'bus:dev.func' (hexadecimal numbers)
1289
1290 You can us the 'lspci' command to list existing PCI devices.
1291 EODESCR
1292     },
1293     rombar => {
1294         type => 'boolean',
1295         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1296         optional => 1,
1297         default => 1,
1298     },
1299     romfile => {
1300         type => 'string',
1301         pattern => '[^,;]+',
1302         format_description => 'string',
1303         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1304         optional => 1,
1305     },
1306     pcie => {
1307         type => 'boolean',
1308         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1309         optional => 1,
1310         default => 0,
1311     },
1312     'x-vga' => {
1313         type => 'boolean',
1314         description =>  "Enable vfio-vga device support.",
1315         optional => 1,
1316         default => 0,
1317     },
1318     'mdev' => {
1319         type => 'string',
1320         format_description => 'string',
1321         pattern => '[^/\.:]+',
1322         optional => 1,
1323         description => <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1327 EODESCR
1328     }
1329 };
1330 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1331
1332 my $hostpcidesc = {
1333         optional => 1,
1334         type => 'string', format => 'pve-qm-hostpci',
1335         description => "Map host PCI devices into guest.",
1336         verbose_description =>  <<EODESCR,
1337 Map host PCI devices into guest.
1338
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1341
1342 CAUTION: Experimental! User reported problems with this option.
1343 EODESCR
1344 };
1345 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1346
1347 my $serialdesc = {
1348         optional => 1,
1349         type => 'string',
1350         pattern => '(/dev/.+|socket)',
1351         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1352         verbose_description =>  <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1356
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1358
1359 CAUTION: Experimental! User reported problems with this option.
1360 EODESCR
1361 };
1362
1363 my $paralleldesc= {
1364         optional => 1,
1365         type => 'string',
1366         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1367         description =>  "Map host parallel devices (n is 0 to 2).",
1368         verbose_description =>  <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1370
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1372
1373 CAUTION: Experimental! User reported problems with this option.
1374 EODESCR
1375 };
1376
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1378     $confdesc->{"parallel$i"} = $paralleldesc;
1379 }
1380
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1382     $confdesc->{"serial$i"} = $serialdesc;
1383 }
1384
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1386     $confdesc->{"hostpci$i"} = $hostpcidesc;
1387 }
1388
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1390     $drivename_hash->{"ide$i"} = 1;
1391     $confdesc->{"ide$i"} = $idedesc;
1392 }
1393
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1395     $drivename_hash->{"sata$i"} = 1;
1396     $confdesc->{"sata$i"} = $satadesc;
1397 }
1398
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1400     $drivename_hash->{"scsi$i"} = 1;
1401     $confdesc->{"scsi$i"} = $scsidesc ;
1402 }
1403
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1405     $drivename_hash->{"virtio$i"} = 1;
1406     $confdesc->{"virtio$i"} = $virtiodesc;
1407 }
1408
1409 $drivename_hash->{efidisk0} = 1;
1410 $confdesc->{efidisk0} = $efidisk_desc;
1411
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1413     $confdesc->{"usb$i"} = $usbdesc;
1414 }
1415
1416 my $unuseddesc = {
1417     optional => 1,
1418     type => 'string', format => 'pve-volume-id',
1419     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1420 };
1421
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1423     $confdesc->{"unused$i"} = $unuseddesc;
1424 }
1425
1426 my $kvm_api_version = 0;
1427
1428 sub kvm_version {
1429     return $kvm_api_version if $kvm_api_version;
1430
1431     open my $fh, '<', '/dev/kvm'
1432         or return undef;
1433
1434     # 0xae00 => KVM_GET_API_VERSION
1435     $kvm_api_version = ioctl($fh, 0xae00, 0);
1436
1437     return $kvm_api_version;
1438 }
1439
1440 my $kvm_user_version;
1441
1442 sub kvm_user_version {
1443
1444     return $kvm_user_version if $kvm_user_version;
1445
1446     $kvm_user_version = 'unknown';
1447
1448     my $code = sub {
1449         my $line = shift;
1450         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451             $kvm_user_version = $2;
1452         }
1453     };
1454
1455     eval { run_command("kvm -version", outfunc => $code); };
1456     warn $@ if $@;
1457
1458     return $kvm_user_version;
1459
1460 }
1461
1462 sub kernel_has_vhost_net {
1463     return -c '/dev/vhost-net';
1464 }
1465
1466 sub valid_drive_names {
1467     # order is important - used to autoselect boot disk
1468     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1472             'efidisk0');
1473 }
1474
1475 sub is_valid_drivename {
1476     my $dev = shift;
1477
1478     return defined($drivename_hash->{$dev});
1479 }
1480
1481 sub option_exists {
1482     my $key = shift;
1483     return defined($confdesc->{$key});
1484 }
1485
1486 sub nic_models {
1487     return $nic_model_list;
1488 }
1489
1490 sub os_list_description {
1491
1492     return {
1493         other => 'Other',
1494         wxp => 'Windows XP',
1495         w2k => 'Windows 2000',
1496         w2k3 =>, 'Windows 2003',
1497         w2k8 => 'Windows 2008',
1498         wvista => 'Windows Vista',
1499         win7 => 'Windows 7',
1500         win8 => 'Windows 8/2012',
1501         win10 => 'Windows 10/2016',
1502         l24 => 'Linux 2.4',
1503         l26 => 'Linux 2.6',
1504     };
1505 }
1506
1507 my $cdrom_path;
1508
1509 sub get_cdrom_path {
1510
1511     return  $cdrom_path if $cdrom_path;
1512
1513     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1514     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1515     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1516 }
1517
1518 sub get_iso_path {
1519     my ($storecfg, $vmid, $cdrom) = @_;
1520
1521     if ($cdrom eq 'cdrom') {
1522         return get_cdrom_path();
1523     } elsif ($cdrom eq 'none') {
1524         return '';
1525     } elsif ($cdrom =~ m|^/|) {
1526         return $cdrom;
1527     } else {
1528         return PVE::Storage::path($storecfg, $cdrom);
1529     }
1530 }
1531
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id {
1534     my ($vmid, $file, $media) = @_;
1535
1536      if (!($file eq 'none' || $file eq 'cdrom' ||
1537           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1538
1539         return undef if $file =~ m|/|;
1540
1541         if ($media && $media eq 'cdrom') {
1542             $file = "local:iso/$file";
1543         } else {
1544             $file = "local:$vmid/$file";
1545         }
1546     }
1547
1548     return $file;
1549 }
1550
1551 sub verify_media_type {
1552     my ($opt, $vtype, $media) = @_;
1553
1554     return if !$media;
1555
1556     my $etype;
1557     if ($media eq 'disk') {
1558         $etype = 'images';
1559     } elsif ($media eq 'cdrom') {
1560         $etype = 'iso';
1561     } else {
1562         die "internal error";
1563     }
1564
1565     return if ($vtype eq $etype);
1566
1567     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1568 }
1569
1570 sub cleanup_drive_path {
1571     my ($opt, $storecfg, $drive) = @_;
1572
1573     # try to convert filesystem paths to volume IDs
1574
1575     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1576         ($drive->{file} !~ m|^/dev/.+|) &&
1577         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1578         ($drive->{file} !~ m/^\d+$/)) {
1579         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1580         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1582         verify_media_type($opt, $vtype, $drive->{media});
1583         $drive->{file} = $volid;
1584     }
1585
1586     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1587 }
1588
1589 sub parse_hotplug_features {
1590     my ($data) = @_;
1591
1592     my $res = {};
1593
1594     return $res if $data eq '0';
1595
1596     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1597
1598     foreach my $feature (PVE::Tools::split_list($data)) {
1599         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1600             $res->{$1} = 1;
1601         } else {
1602             die "invalid hotplug feature '$feature'\n";
1603         }
1604     }
1605     return $res;
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1609 sub pve_verify_hotplug_features {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_hotplug_features($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse hotplug option\n";
1617 }
1618
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 #        [,iothread=on][,serial=serial][,model=model]
1624
1625 sub parse_drive {
1626     my ($key, $data) = @_;
1627
1628     my ($interface, $index);
1629
1630     if ($key =~ m/^([^\d]+)(\d+)$/) {
1631         $interface = $1;
1632         $index = $2;
1633     } else {
1634         return undef;
1635     }
1636
1637     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1638                                      : $confdesc->{$key}->{format};
1639     if (!$desc) {
1640         warn "invalid drive key: $key\n";
1641         return undef;
1642     }
1643     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1644     return undef if !$res;
1645     $res->{interface} = $interface;
1646     $res->{index} = $index;
1647
1648     my $error = 0;
1649     foreach my $opt (qw(bps bps_rd bps_wr)) {
1650         if (my $bps = defined(delete $res->{$opt})) {
1651             if (defined($res->{"m$opt"})) {
1652                 warn "both $opt and m$opt specified\n";
1653                 ++$error;
1654                 next;
1655             }
1656             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1657         }
1658     }
1659
1660     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661     for my $requirement (
1662         [mbps_max => 'mbps'],
1663         [mbps_rd_max => 'mbps_rd'],
1664         [mbps_wr_max => 'mbps_wr'],
1665         [miops_max => 'miops'],
1666         [miops_rd_max => 'miops_rd'],
1667         [miops_wr_max => 'miops_wr'],
1668         [bps_max_length => 'mbps_max'],
1669         [bps_rd_max_length => 'mbps_rd_max'],
1670         [bps_wr_max_length => 'mbps_wr_max'],
1671         [iops_max_length => 'iops_max'],
1672         [iops_rd_max_length => 'iops_rd_max'],
1673         [iops_wr_max_length => 'iops_wr_max']) {
1674         my ($option, $requires) = @$requirement;
1675         if ($res->{$option} && !$res->{$requires}) {
1676             warn "$option requires $requires\n";
1677             ++$error;
1678         }
1679     }
1680
1681     return undef if $error;
1682
1683     return undef if $res->{mbps_rd} && $res->{mbps};
1684     return undef if $res->{mbps_wr} && $res->{mbps};
1685     return undef if $res->{iops_rd} && $res->{iops};
1686     return undef if $res->{iops_wr} && $res->{iops};
1687
1688     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1689         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1690         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1691         return undef if $res->{interface} eq 'virtio';
1692     }
1693
1694     if (my $size = $res->{size}) {
1695         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1696     }
1697
1698     return $res;
1699 }
1700
1701 sub print_drive {
1702     my ($vmid, $drive) = @_;
1703     my $data = { %$drive };
1704     delete $data->{$_} for qw(index interface);
1705     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1706 }
1707
1708 sub scsi_inquiry {
1709     my($fh, $noerr) = @_;
1710
1711     my $SG_IO = 0x2285;
1712     my $SG_GET_VERSION_NUM = 0x2282;
1713
1714     my $versionbuf = "\x00" x 8;
1715     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1716     if (!$ret) {
1717         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1718         return undef;
1719     }
1720     my $version = unpack("I", $versionbuf);
1721     if ($version < 30000) {
1722         die "scsi generic interface too old\n"  if !$noerr;
1723         return undef;
1724     }
1725
1726     my $buf = "\x00" x 36;
1727     my $sensebuf = "\x00" x 8;
1728     my $cmd = pack("C x3 C x1", 0x12, 36);
1729
1730     # see /usr/include/scsi/sg.h
1731     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1732
1733     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734                       length($sensebuf), 0, length($buf), $buf,
1735                       $cmd, $sensebuf, 6000);
1736
1737     $ret = ioctl($fh, $SG_IO, $packet);
1738     if (!$ret) {
1739         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1740         return undef;
1741     }
1742
1743     my @res = unpack($sg_io_hdr_t, $packet);
1744     if ($res[17] || $res[18]) {
1745         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1746         return undef;
1747     }
1748
1749     my $res = {};
1750     (my $byte0, my $byte1, $res->{vendor},
1751      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1752
1753     $res->{removable} = $byte1 & 128 ? 1 : 0;
1754     $res->{type} = $byte0 & 31;
1755
1756     return $res;
1757 }
1758
1759 sub path_is_scsi {
1760     my ($path) = @_;
1761
1762     my $fh = IO::File->new("+<$path") || return undef;
1763     my $res = scsi_inquiry($fh, 1);
1764     close($fh);
1765
1766     return $res;
1767 }
1768
1769 sub machine_type_is_q35 {
1770     my ($conf) = @_;
1771
1772     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1773 }
1774
1775 sub print_tabletdevice_full {
1776     my ($conf, $arch) = @_;
1777
1778     my $q35 = machine_type_is_q35($conf);
1779
1780     # we use uhci for old VMs because tablet driver was buggy in older qemu
1781     my $usbbus;
1782     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1783         $usbbus = 'ehci';
1784     } else {
1785         $usbbus = 'uhci';
1786     }
1787
1788     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1789 }
1790
1791 sub print_keyboarddevice_full {
1792     my ($conf, $arch, $machine) = @_;
1793
1794     return undef if $arch ne 'aarch64';
1795
1796     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1797 }
1798
1799 sub print_drivedevice_full {
1800     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1801
1802     my $device = '';
1803     my $maxdev = 0;
1804
1805     if ($drive->{interface} eq 'virtio') {
1806         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1809     } elsif ($drive->{interface} eq 'scsi') {
1810
1811         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1812         my $unit = $drive->{index} % $maxdev;
1813         my $devicetype = 'hd';
1814         my $path = '';
1815         if (drive_is_cdrom($drive)) {
1816             $devicetype = 'cd';
1817         } else {
1818             if ($drive->{file} =~ m|^/|) {
1819                 $path = $drive->{file};
1820                 if (my $info = path_is_scsi($path)) {
1821                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1822                         $devicetype = 'block';
1823                     } elsif ($info->{type} == 1) { # tape
1824                         $devicetype = 'generic';
1825                     }
1826                 }
1827             } else {
1828                  $path = PVE::Storage::path($storecfg, $drive->{file});
1829             }
1830
1831             if($path =~ m/^iscsi\:\/\//){
1832                 $devicetype = 'generic';
1833             }
1834         }
1835
1836         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1837             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1838         } else {
1839             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1840         }
1841
1842         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843             $device .= ",rotation_rate=1";
1844         }
1845         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1846
1847     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1848         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1849         my $controller = int($drive->{index} / $maxdev);
1850         my $unit = $drive->{index} % $maxdev;
1851         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1852
1853         $device = "ide-$devicetype";
1854         if ($drive->{interface} eq 'ide') {
1855             $device .= ",bus=ide.$controller,unit=$unit";
1856         } else {
1857             $device .= ",bus=ahci$controller.$unit";
1858         }
1859         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1860
1861         if ($devicetype eq 'hd') {
1862             if (my $model = $drive->{model}) {
1863                 $model = URI::Escape::uri_unescape($model);
1864                 $device .= ",model=$model";
1865             }
1866             if ($drive->{ssd}) {
1867                 $device .= ",rotation_rate=1";
1868             }
1869         }
1870         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1871     } elsif ($drive->{interface} eq 'usb') {
1872         die "implement me";
1873         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1874     } else {
1875         die "unsupported interface type";
1876     }
1877
1878     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1879
1880     if (my $serial = $drive->{serial}) {
1881         $serial = URI::Escape::uri_unescape($serial);
1882         $device .= ",serial=$serial";
1883     }
1884
1885
1886     return $device;
1887 }
1888
1889 sub get_initiator_name {
1890     my $initiator;
1891
1892     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893     while (defined(my $line = <$fh>)) {
1894         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1895         $initiator = $1;
1896         last;
1897     }
1898     $fh->close();
1899
1900     return $initiator;
1901 }
1902
1903 sub print_drive_full {
1904     my ($storecfg, $vmid, $drive) = @_;
1905
1906     my $path;
1907     my $volid = $drive->{file};
1908     my $format;
1909
1910     if (drive_is_cdrom($drive)) {
1911         $path = get_iso_path($storecfg, $vmid, $volid);
1912     } else {
1913         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1914         if ($storeid) {
1915             $path = PVE::Storage::path($storecfg, $volid);
1916             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1917             $format = qemu_img_format($scfg, $volname);
1918         } else {
1919             $path = $volid;
1920             $format = "raw";
1921         }
1922    }
1923
1924     my $opts = '';
1925     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926     foreach my $o (@qemu_drive_options) {
1927         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1928     }
1929
1930     # snapshot only accepts on|off
1931     if (defined($drive->{snapshot})) {
1932         my $v = $drive->{snapshot} ? 'on' : 'off';
1933         $opts .= ",snapshot=$v";
1934     }
1935
1936     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1937         my ($dir, $qmpname) = @$type;
1938         if (my $v = $drive->{"mbps$dir"}) {
1939             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1940         }
1941         if (my $v = $drive->{"mbps${dir}_max"}) {
1942             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1943         }
1944         if (my $v = $drive->{"bps${dir}_max_length"}) {
1945             $opts .= ",throttling.bps$qmpname-max-length=$v";
1946         }
1947         if (my $v = $drive->{"iops${dir}"}) {
1948             $opts .= ",throttling.iops$qmpname=$v";
1949         }
1950         if (my $v = $drive->{"iops${dir}_max"}) {
1951             $opts .= ",throttling.iops$qmpname-max=$v";
1952         }
1953         if (my $v = $drive->{"iops${dir}_max_length"}) {
1954             $opts .= ",throttling.iops$qmpname-max-length=$v";
1955         }
1956     }
1957
1958     $opts .= ",format=$format" if $format && !$drive->{format};
1959
1960     my $cache_direct = 0;
1961
1962     if (my $cache = $drive->{cache}) {
1963         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964     } elsif (!drive_is_cdrom($drive)) {
1965         $opts .= ",cache=none";
1966         $cache_direct = 1;
1967     }
1968
1969     # aio native works only with O_DIRECT
1970     if (!$drive->{aio}) {
1971         if($cache_direct) {
1972             $opts .= ",aio=native";
1973         } else {
1974             $opts .= ",aio=threads";
1975         }
1976     }
1977
1978     if (!drive_is_cdrom($drive)) {
1979         my $detectzeroes;
1980         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1981             $detectzeroes = 'off';
1982         } elsif ($drive->{discard}) {
1983             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1984         } else {
1985             # This used to be our default with discard not being specified:
1986             $detectzeroes = 'on';
1987         }
1988         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1989     }
1990
1991     my $pathinfo = $path ? "file=$path," : '';
1992
1993     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1994 }
1995
1996 sub print_netdevice_full {
1997     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1998
1999     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2000
2001     my $device = $net->{model};
2002     if ($net->{model} eq 'virtio') {
2003          $device = 'virtio-net-pci';
2004      };
2005
2006     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2007     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2009         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010         my $vectors = $net->{queues} * 2 + 2;
2011         $tmpstr .= ",vectors=$vectors,mq=on";
2012     }
2013     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2014
2015     if ($use_old_bios_files) {
2016         my $romfile;
2017         if ($device eq 'virtio-net-pci') {
2018             $romfile = 'pxe-virtio.rom';
2019         } elsif ($device eq 'e1000') {
2020             $romfile = 'pxe-e1000.rom';
2021         } elsif ($device eq 'ne2k') {
2022             $romfile = 'pxe-ne2k_pci.rom';
2023         } elsif ($device eq 'pcnet') {
2024             $romfile = 'pxe-pcnet.rom';
2025         } elsif ($device eq 'rtl8139') {
2026             $romfile = 'pxe-rtl8139.rom';
2027         }
2028         $tmpstr .= ",romfile=$romfile" if $romfile;
2029     }
2030
2031     return $tmpstr;
2032 }
2033
2034 sub print_netdev_full {
2035     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2036
2037     my $i = '';
2038     if ($netid =~ m/^net(\d+)$/) {
2039         $i = int($1);
2040     }
2041
2042     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2043
2044     my $ifname = "tap${vmid}i$i";
2045
2046     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047     die "interface name '$ifname' is too long (max 15 character)\n"
2048         if length($ifname) >= 16;
2049
2050     my $vhostparam = '';
2051     if (is_native($arch)) {
2052         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2053     }
2054
2055     my $vmname = $conf->{name} || "vm$vmid";
2056
2057     my $netdev = "";
2058     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2059
2060     if ($net->{bridge}) {
2061         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2062     } else {
2063         $netdev = "type=user,id=$netid,hostname=$vmname";
2064     }
2065
2066     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2067
2068     return $netdev;
2069 }
2070
2071
2072 sub print_cpu_device {
2073     my ($conf, $id) = @_;
2074
2075     my $kvm = $conf->{kvm} // 1;
2076     my $cpu = $kvm ? "kvm64" : "qemu64";
2077     if (my $cputype = $conf->{cpu}) {
2078         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2079             or die "Cannot parse cpu description: $cputype\n";
2080         $cpu = $cpuconf->{cputype};
2081     }
2082
2083     my $cores = $conf->{cores} || 1;
2084
2085     my $current_core = ($id - 1) % $cores;
2086     my $current_socket = int(($id - 1 - $current_core)/$cores);
2087
2088     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2089 }
2090
2091 my $vga_map = {
2092     'cirrus' => 'cirrus-vga',
2093     'std' => 'VGA',
2094     'vmware' => 'vmware-svga',
2095     'virtio' => 'virtio-vga',
2096 };
2097
2098 sub print_vga_device {
2099     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2100
2101     my $type = $vga_map->{$vga->{type}};
2102     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103         $type = 'virtio-gpu';
2104     }
2105     my $vgamem_mb = $vga->{memory};
2106     if ($qxlnum) {
2107         $type = $id ? 'qxl' : 'qxl-vga';
2108     }
2109     die "no devicetype for $vga->{type}\n" if !$type;
2110
2111     my $memory = "";
2112     if ($vgamem_mb) {
2113         if ($vga->{type} eq 'virtio') {
2114             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2115             $memory = ",max_hostmem=$bytes";
2116         } elsif ($qxlnum) {
2117             # from https://www.spice-space.org/multiple-monitors.html
2118             $memory = ",vgamem_mb=$vga->{memory}";
2119             my $ram = $vgamem_mb * 4;
2120             my $vram = $vgamem_mb * 2;
2121             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2122         } else {
2123             $memory = ",vgamem_mb=$vga->{memory}";
2124         }
2125     } elsif ($qxlnum && $id) {
2126         $memory = ",ram_size=67108864,vram_size=33554432";
2127     }
2128
2129     my $q35 = machine_type_is_q35($conf);
2130     my $vgaid = "vga" . ($id // '');
2131     my $pciaddr;
2132
2133     if ($q35 && $vgaid eq 'vga') {
2134         # the first display uses pcie.0 bus on q35 machines
2135         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2136     } else {
2137         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2138     }
2139
2140     return "$type,id=${vgaid}${memory}${pciaddr}";
2141 }
2142
2143 sub drive_is_cloudinit {
2144     my ($drive) = @_;
2145     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2146 }
2147
2148 sub drive_is_cdrom {
2149     my ($drive, $exclude_cloudinit) = @_;
2150
2151     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2152
2153     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2154
2155 }
2156
2157 sub parse_number_sets {
2158     my ($set) = @_;
2159     my $res = [];
2160     foreach my $part (split(/;/, $set)) {
2161         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163             push @$res, [ $1, $2 ];
2164         } else {
2165             die "invalid range: $part\n";
2166         }
2167     }
2168     return $res;
2169 }
2170
2171 sub parse_numa {
2172     my ($data) = @_;
2173
2174     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2175     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2176     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2177     return $res;
2178 }
2179
2180 sub parse_hostpci {
2181     my ($value) = @_;
2182
2183     return undef if !$value;
2184
2185     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2186
2187     my @idlist = split(/;/, $res->{host});
2188     delete $res->{host};
2189     foreach my $id (@idlist) {
2190         if ($id =~ m/\./) { # full id 00:00.1
2191             push @{$res->{pciid}}, {
2192                 id => $id,
2193             };
2194         } else { # partial id 00:00
2195             $res->{pciid} = PVE::SysFSTools::lspci($id);
2196         }
2197     }
2198     return $res;
2199 }
2200
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2202 sub parse_net {
2203     my ($data) = @_;
2204
2205     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2206     if ($@) {
2207         warn $@;
2208         return undef;
2209     }
2210     if (!defined($res->{macaddr})) {
2211         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2212         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2213     }
2214     return $res;
2215 }
2216
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig {
2219     my ($data) = @_;
2220
2221     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2222     if ($@) {
2223         warn $@;
2224         return undef;
2225     }
2226
2227     if ($res->{gw} && !$res->{ip}) {
2228         warn 'gateway specified without specifying an IP address';
2229         return undef;
2230     }
2231     if ($res->{gw6} && !$res->{ip6}) {
2232         warn 'IPv6 gateway specified without specifying an IPv6 address';
2233         return undef;
2234     }
2235     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2236         warn 'gateway specified together with DHCP';
2237         return undef;
2238     }
2239     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2240         # gw6 + auto/dhcp
2241         warn "IPv6 gateway specified together with $res->{ip6} address";
2242         return undef;
2243     }
2244
2245     if (!$res->{ip} && !$res->{ip6}) {
2246         return { ip => 'dhcp', ip6 => 'dhcp' };
2247     }
2248
2249     return $res;
2250 }
2251
2252 sub print_net {
2253     my $net = shift;
2254
2255     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2256 }
2257
2258 sub add_random_macs {
2259     my ($settings) = @_;
2260
2261     foreach my $opt (keys %$settings) {
2262         next if $opt !~ m/^net(\d+)$/;
2263         my $net = parse_net($settings->{$opt});
2264         next if !$net;
2265         $settings->{$opt} = print_net($net);
2266     }
2267 }
2268
2269 sub vm_is_volid_owner {
2270     my ($storecfg, $vmid, $volid) = @_;
2271
2272     if ($volid !~  m|^/|) {
2273         my ($path, $owner);
2274         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2275         if ($owner && ($owner == $vmid)) {
2276             return 1;
2277         }
2278     }
2279
2280     return undef;
2281 }
2282
2283 sub split_flagged_list {
2284     my $text = shift || '';
2285     $text =~ s/[,;]/ /g;
2286     $text =~ s/^\s+//;
2287     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2288 }
2289
2290 sub join_flagged_list {
2291     my ($how, $lst) = @_;
2292     join $how, map { $lst->{$_} . $_ } keys %$lst;
2293 }
2294
2295 sub vmconfig_delete_pending_option {
2296     my ($conf, $key, $force) = @_;
2297
2298     delete $conf->{pending}->{$key};
2299     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2300     $pending_delete_hash->{$key} = $force ? '!' : '';
2301     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2302 }
2303
2304 sub vmconfig_undelete_pending_option {
2305     my ($conf, $key) = @_;
2306
2307     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2308     delete $pending_delete_hash->{$key};
2309
2310     if (%$pending_delete_hash) {
2311         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2312     } else {
2313         delete $conf->{pending}->{delete};
2314     }
2315 }
2316
2317 sub vmconfig_register_unused_drive {
2318     my ($storecfg, $vmid, $conf, $drive) = @_;
2319
2320     if (drive_is_cloudinit($drive)) {
2321         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2322         warn $@ if $@;
2323     } elsif (!drive_is_cdrom($drive)) {
2324         my $volid = $drive->{file};
2325         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2326             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2327         }
2328     }
2329 }
2330
2331 sub vmconfig_cleanup_pending {
2332     my ($conf) = @_;
2333
2334     # remove pending changes when nothing changed
2335     my $changes;
2336     foreach my $opt (keys %{$conf->{pending}}) {
2337         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2338             $changes = 1;
2339             delete $conf->{pending}->{$opt};
2340         }
2341     }
2342
2343     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2344     my $pending_delete_hash = {};
2345     while (my ($opt, $force) = each %$current_delete_hash) {
2346         if (defined($conf->{$opt})) {
2347             $pending_delete_hash->{$opt} = $force;
2348         } else {
2349             $changes = 1;
2350         }
2351     }
2352
2353     if (%$pending_delete_hash) {
2354         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2355     } else {
2356         delete $conf->{pending}->{delete};
2357     }
2358
2359     return $changes;
2360 }
2361
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2363 my $smbios1_fmt = {
2364     uuid => {
2365         type => 'string',
2366         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367         format_description => 'UUID',
2368         description => "Set SMBIOS1 UUID.",
2369         optional => 1,
2370     },
2371     version => {
2372         type => 'string',
2373         pattern => '[A-Za-z0-9+\/]+={0,2}',
2374         format_description => 'Base64 encoded string',
2375         description => "Set SMBIOS1 version.",
2376         optional => 1,
2377     },
2378     serial => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 serial number.",
2383         optional => 1,
2384     },
2385     manufacturer => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 manufacturer.",
2390         optional => 1,
2391     },
2392     product => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 product ID.",
2397         optional => 1,
2398     },
2399     sku => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 SKU string.",
2404         optional => 1,
2405     },
2406     family => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 family string.",
2411         optional => 1,
2412     },
2413     base64 => {
2414         type => 'boolean',
2415         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2416         optional => 1,
2417     },
2418 };
2419
2420 sub parse_smbios1 {
2421     my ($data) = @_;
2422
2423     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2424     warn $@ if $@;
2425     return $res;
2426 }
2427
2428 sub print_smbios1 {
2429     my ($smbios1) = @_;
2430     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2431 }
2432
2433 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2434
2435 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2436 sub verify_bootdisk {
2437     my ($value, $noerr) = @_;
2438
2439     return $value if is_valid_drivename($value);
2440
2441     return undef if $noerr;
2442
2443     die "invalid boot disk '$value'\n";
2444 }
2445
2446 sub parse_watchdog {
2447     my ($value) = @_;
2448
2449     return undef if !$value;
2450
2451     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2452     warn $@ if $@;
2453     return $res;
2454 }
2455
2456 sub parse_guest_agent {
2457     my ($value) = @_;
2458
2459     return {} if !defined($value->{agent});
2460
2461     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2462     warn $@ if $@;
2463
2464     # if the agent is disabled ignore the other potentially set properties
2465     return {} if !$res->{enabled};
2466     return $res;
2467 }
2468
2469 sub parse_vga {
2470     my ($value) = @_;
2471
2472     return {} if !$value;
2473     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2474     warn $@ if $@;
2475     return $res;
2476 }
2477
2478 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2479 sub verify_usb_device {
2480     my ($value, $noerr) = @_;
2481
2482     return $value if parse_usb_device($value);
2483
2484     return undef if $noerr;
2485
2486     die "unable to parse usb device\n";
2487 }
2488
2489 # add JSON properties for create and set function
2490 sub json_config_properties {
2491     my $prop = shift;
2492
2493     foreach my $opt (keys %$confdesc) {
2494         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495         $prop->{$opt} = $confdesc->{$opt};
2496     }
2497
2498     return $prop;
2499 }
2500
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties {
2503
2504     return dclone($confdesc_cloudinit);
2505 }
2506
2507 sub check_type {
2508     my ($key, $value) = @_;
2509
2510     die "unknown setting '$key'\n" if !$confdesc->{$key};
2511
2512     my $type = $confdesc->{$key}->{type};
2513
2514     if (!defined($value)) {
2515         die "got undefined value\n";
2516     }
2517
2518     if ($value =~ m/[\n\r]/) {
2519         die "property contains a line feed\n";
2520     }
2521
2522     if ($type eq 'boolean') {
2523         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525         die "type check ('boolean') failed - got '$value'\n";
2526     } elsif ($type eq 'integer') {
2527         return int($1) if $value =~ m/^(\d+)$/;
2528         die "type check ('integer') failed - got '$value'\n";
2529     } elsif ($type eq 'number') {
2530         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531         die "type check ('number') failed - got '$value'\n";
2532     } elsif ($type eq 'string') {
2533         if (my $fmt = $confdesc->{$key}->{format}) {
2534             PVE::JSONSchema::check_format($fmt, $value);
2535             return $value;
2536         }
2537         $value =~ s/^\"(.*)\"$/$1/;
2538         return $value;
2539     } else {
2540         die "internal error"
2541     }
2542 }
2543
2544 sub touch_config {
2545     my ($vmid) = @_;
2546
2547     my $conf = PVE::QemuConfig->config_file($vmid);
2548     utime undef, undef, $conf;
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conffile = PVE::QemuConfig->config_file($vmid);
2555
2556     my $conf = PVE::QemuConfig->load_config($vmid);
2557
2558     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2559
2560     if ($conf->{template}) {
2561         # check if any base image is still used by a linked clone
2562         foreach_drive($conf, sub {
2563                 my ($ds, $drive) = @_;
2564
2565                 return if drive_is_cdrom($drive);
2566
2567                 my $volid = $drive->{file};
2568
2569                 return if !$volid || $volid =~ m|^/|;
2570
2571                 die "base volume '$volid' is still in use by linked cloned\n"
2572                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2573
2574         });
2575     }
2576
2577     # only remove disks owned by this VM
2578     foreach_drive($conf, sub {
2579         my ($ds, $drive) = @_;
2580
2581         return if drive_is_cdrom($drive, 1);
2582
2583         my $volid = $drive->{file};
2584
2585         return if !$volid || $volid =~ m|^/|;
2586
2587         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2588         return if !$path || !$owner || ($owner != $vmid);
2589
2590         eval {
2591             PVE::Storage::vdisk_free($storecfg, $volid);
2592         };
2593         warn "Could not remove disk '$volid', check manually: $@" if $@;
2594
2595     });
2596
2597     if ($keep_empty_config) {
2598         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2599     } else {
2600         unlink $conffile;
2601     }
2602
2603     # also remove unused disk
2604     eval {
2605         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2606
2607         eval {
2608             PVE::Storage::foreach_volid($dl, sub {
2609                 my ($volid, $sid, $volname, $d) = @_;
2610                 PVE::Storage::vdisk_free($storecfg, $volid);
2611             });
2612         };
2613         warn $@ if $@;
2614
2615     };
2616     warn $@ if $@;
2617 }
2618
2619 sub parse_vm_config {
2620     my ($filename, $raw) = @_;
2621
2622     return undef if !defined($raw);
2623
2624     my $res = {
2625         digest => Digest::SHA::sha1_hex($raw),
2626         snapshots => {},
2627         pending => {},
2628     };
2629
2630     $filename =~ m|/qemu-server/(\d+)\.conf$|
2631         || die "got strange filename '$filename'";
2632
2633     my $vmid = $1;
2634
2635     my $conf = $res;
2636     my $descr;
2637     my $section = '';
2638
2639     my @lines = split(/\n/, $raw);
2640     foreach my $line (@lines) {
2641         next if $line =~ m/^\s*$/;
2642
2643         if ($line =~ m/^\[PENDING\]\s*$/i) {
2644             $section = 'pending';
2645             if (defined($descr)) {
2646                 $descr =~ s/\s+$//;
2647                 $conf->{description} = $descr;
2648             }
2649             $descr = undef;
2650             $conf = $res->{$section} = {};
2651             next;
2652
2653         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2654             $section = $1;
2655             if (defined($descr)) {
2656                 $descr =~ s/\s+$//;
2657                 $conf->{description} = $descr;
2658             }
2659             $descr = undef;
2660             $conf = $res->{snapshots}->{$section} = {};
2661             next;
2662         }
2663
2664         if ($line =~ m/^\#(.*)\s*$/) {
2665             $descr = '' if !defined($descr);
2666             $descr .= PVE::Tools::decode_text($1) . "\n";
2667             next;
2668         }
2669
2670         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671             $descr = '' if !defined($descr);
2672             $descr .= PVE::Tools::decode_text($2);
2673         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674             $conf->{snapstate} = $1;
2675         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2676             my $key = $1;
2677             my $value = $2;
2678             $conf->{$key} = $value;
2679         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2680             my $value = $1;
2681             if ($section eq 'pending') {
2682                 $conf->{delete} = $value; # we parse this later
2683             } else {
2684                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2685             }
2686         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2687             my $key = $1;
2688             my $value = $2;
2689             eval { $value = check_type($key, $value); };
2690             if ($@) {
2691                 warn "vm $vmid - unable to parse value of '$key' - $@";
2692             } else {
2693                 $key = 'ide2' if $key eq 'cdrom';
2694                 my $fmt = $confdesc->{$key}->{format};
2695                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696                     my $v = parse_drive($key, $value);
2697                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2698                         $v->{file} = $volid;
2699                         $value = print_drive($vmid, $v);
2700                     } else {
2701                         warn "vm $vmid - unable to parse value of '$key'\n";
2702                         next;
2703                     }
2704                 }
2705
2706                 $conf->{$key} = $value;
2707             }
2708         }
2709     }
2710
2711     if (defined($descr)) {
2712         $descr =~ s/\s+$//;
2713         $conf->{description} = $descr;
2714     }
2715     delete $res->{snapstate}; # just to be sure
2716
2717     return $res;
2718 }
2719
2720 sub write_vm_config {
2721     my ($filename, $conf) = @_;
2722
2723     delete $conf->{snapstate}; # just to be sure
2724
2725     if ($conf->{cdrom}) {
2726         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2727         $conf->{ide2} = $conf->{cdrom};
2728         delete $conf->{cdrom};
2729     }
2730
2731     # we do not use 'smp' any longer
2732     if ($conf->{sockets}) {
2733         delete $conf->{smp};
2734     } elsif ($conf->{smp}) {
2735         $conf->{sockets} = $conf->{smp};
2736         delete $conf->{cores};
2737         delete $conf->{smp};
2738     }
2739
2740     my $used_volids = {};
2741
2742     my $cleanup_config = sub {
2743         my ($cref, $pending, $snapname) = @_;
2744
2745         foreach my $key (keys %$cref) {
2746             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747                 $key eq 'snapstate' || $key eq 'pending';
2748             my $value = $cref->{$key};
2749             if ($key eq 'delete') {
2750                 die "propertry 'delete' is only allowed in [PENDING]\n"
2751                     if !$pending;
2752                 # fixme: check syntax?
2753                 next;
2754             }
2755             eval { $value = check_type($key, $value); };
2756             die "unable to parse value of '$key' - $@" if $@;
2757
2758             $cref->{$key} = $value;
2759
2760             if (!$snapname && is_valid_drivename($key)) {
2761                 my $drive = parse_drive($key, $value);
2762                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2763             }
2764         }
2765     };
2766
2767     &$cleanup_config($conf);
2768
2769     &$cleanup_config($conf->{pending}, 1);
2770
2771     foreach my $snapname (keys %{$conf->{snapshots}}) {
2772         die "internal error" if $snapname eq 'pending';
2773         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2774     }
2775
2776     # remove 'unusedX' settings if we re-add a volume
2777     foreach my $key (keys %$conf) {
2778         my $value = $conf->{$key};
2779         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780             delete $conf->{$key};
2781         }
2782     }
2783
2784     my $generate_raw_config = sub {
2785         my ($conf, $pending) = @_;
2786
2787         my $raw = '';
2788
2789         # add description as comment to top of file
2790         if (defined(my $descr = $conf->{description})) {
2791             if ($descr) {
2792                 foreach my $cl (split(/\n/, $descr)) {
2793                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2794                 }
2795             } else {
2796                 $raw .= "#\n" if $pending;
2797             }
2798         }
2799
2800         foreach my $key (sort keys %$conf) {
2801             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802             $raw .= "$key: $conf->{$key}\n";
2803         }
2804         return $raw;
2805     };
2806
2807     my $raw = &$generate_raw_config($conf);
2808
2809     if (scalar(keys %{$conf->{pending}})){
2810         $raw .= "\n[PENDING]\n";
2811         $raw .= &$generate_raw_config($conf->{pending}, 1);
2812     }
2813
2814     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2815         $raw .= "\n[$snapname]\n";
2816         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2817     }
2818
2819     return $raw;
2820 }
2821
2822 sub load_defaults {
2823
2824     my $res = {};
2825
2826     # we use static defaults from our JSON schema configuration
2827     foreach my $key (keys %$confdesc) {
2828         if (defined(my $default = $confdesc->{$key}->{default})) {
2829             $res->{$key} = $default;
2830         }
2831     }
2832
2833     return $res;
2834 }
2835
2836 sub config_list {
2837     my $vmlist = PVE::Cluster::get_vmlist();
2838     my $res = {};
2839     return $res if !$vmlist || !$vmlist->{ids};
2840     my $ids = $vmlist->{ids};
2841
2842     foreach my $vmid (keys %$ids) {
2843         my $d = $ids->{$vmid};
2844         next if !$d->{node} || $d->{node} ne $nodename;
2845         next if !$d->{type} || $d->{type} ne 'qemu';
2846         $res->{$vmid}->{exists} = 1;
2847     }
2848     return $res;
2849 }
2850
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources {
2853     my ($conf, $noerr) = @_;
2854
2855     my @loc_res = ();
2856
2857     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2858     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2859
2860     push @loc_res, "ivshmem" if $conf->{ivshmem};
2861
2862     foreach my $k (keys %$conf) {
2863         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864         # sockets are safe: they will recreated be on the target side post-migrate
2865         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2867     }
2868
2869     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2870
2871     return \@loc_res;
2872 }
2873
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability {
2876     my ($storecfg, $conf, $node) = @_;
2877
2878     foreach_drive($conf, sub {
2879         my ($ds, $drive) = @_;
2880
2881         my $volid = $drive->{file};
2882         return if !$volid;
2883
2884         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2885         return if !$sid;
2886
2887         # check if storage is available on both nodes
2888         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2889         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2890    });
2891 }
2892
2893 # list nodes where all VM images are available (used by has_feature API)
2894 sub shared_nodes {
2895     my ($conf, $storecfg) = @_;
2896
2897     my $nodelist = PVE::Cluster::get_nodelist();
2898     my $nodehash = { map { $_ => 1 } @$nodelist };
2899     my $nodename = PVE::INotify::nodename();
2900
2901     foreach_drive($conf, sub {
2902         my ($ds, $drive) = @_;
2903
2904         my $volid = $drive->{file};
2905         return if !$volid;
2906
2907         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2908         if ($storeid) {
2909             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2910             if ($scfg->{disable}) {
2911                 $nodehash = {};
2912             } elsif (my $avail = $scfg->{nodes}) {
2913                 foreach my $node (keys %$nodehash) {
2914                     delete $nodehash->{$node} if !$avail->{$node};
2915                 }
2916             } elsif (!$scfg->{shared}) {
2917                 foreach my $node (keys %$nodehash) {
2918                     delete $nodehash->{$node} if $node ne $nodename
2919                 }
2920             }
2921         }
2922     });
2923
2924     return $nodehash
2925 }
2926
2927 sub check_local_storage_availability {
2928     my ($conf, $storecfg) = @_;
2929
2930     my $nodelist = PVE::Cluster::get_nodelist();
2931     my $nodehash = { map { $_ => {} } @$nodelist };
2932
2933     foreach_drive($conf, sub {
2934         my ($ds, $drive) = @_;
2935
2936         my $volid = $drive->{file};
2937         return if !$volid;
2938
2939         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2940         if ($storeid) {
2941             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2942
2943             if ($scfg->{disable}) {
2944                 foreach my $node (keys %$nodehash) {
2945                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2946                 }
2947             } elsif (my $avail = $scfg->{nodes}) {
2948                 foreach my $node (keys %$nodehash) {
2949                     if (!$avail->{$node}) {
2950                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2951                     }
2952                 }
2953             }
2954         }
2955     });
2956
2957     foreach my $node (values %$nodehash) {
2958         if (my $unavail = $node->{unavailable_storages}) {
2959             $node->{unavailable_storages} = [ sort keys %$unavail ];
2960         }
2961     }
2962
2963     return $nodehash
2964 }
2965
2966 sub check_cmdline {
2967     my ($pidfile, $pid) = @_;
2968
2969     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2970     if (defined($fh)) {
2971         my $line = <$fh>;
2972         $fh->close;
2973         return undef if !$line;
2974         my @param = split(/\0/, $line);
2975
2976         my $cmd = $param[0];
2977         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2978
2979         for (my $i = 0; $i < scalar (@param); $i++) {
2980             my $p = $param[$i];
2981             next if !$p;
2982             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2983                 my $p = $param[$i+1];
2984                 return 1 if $p && ($p eq $pidfile);
2985                 return undef;
2986             }
2987         }
2988     }
2989     return undef;
2990 }
2991
2992 sub check_running {
2993     my ($vmid, $nocheck, $node) = @_;
2994
2995     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2996
2997     die "unable to find configuration file for VM $vmid - no such machine\n"
2998         if !$nocheck && ! -f $filename;
2999
3000     my $pidfile = pidfile_name($vmid);
3001
3002     if (my $fd = IO::File->new("<$pidfile")) {
3003         my $st = stat($fd);
3004         my $line = <$fd>;
3005         close($fd);
3006
3007         my $mtime = $st->mtime;
3008         if ($mtime > time()) {
3009             warn "file '$filename' modified in future\n";
3010         }
3011
3012         if ($line =~ m/^(\d+)$/) {
3013             my $pid = $1;
3014             if (check_cmdline($pidfile, $pid)) {
3015                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3016                     return $pid;
3017                 }
3018             }
3019         }
3020     }
3021
3022     return undef;
3023 }
3024
3025 sub vzlist {
3026
3027     my $vzlist = config_list();
3028
3029     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3030
3031     while (defined(my $de = $fd->read)) {
3032         next if $de !~ m/^(\d+)\.pid$/;
3033         my $vmid = $1;
3034         next if !defined($vzlist->{$vmid});
3035         if (my $pid = check_running($vmid)) {
3036             $vzlist->{$vmid}->{pid} = $pid;
3037         }
3038     }
3039
3040     return $vzlist;
3041 }
3042
3043 sub disksize {
3044     my ($storecfg, $conf) = @_;
3045
3046     my $bootdisk = $conf->{bootdisk};
3047     return undef if !$bootdisk;
3048     return undef if !is_valid_drivename($bootdisk);
3049
3050     return undef if !$conf->{$bootdisk};
3051
3052     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3053     return undef if !defined($drive);
3054
3055     return undef if drive_is_cdrom($drive);
3056
3057     my $volid = $drive->{file};
3058     return undef if !$volid;
3059
3060     return $drive->{size};
3061 }
3062
3063 our $vmstatus_return_properties = {
3064     vmid => get_standard_option('pve-vmid'),
3065     status => {
3066         description => "Qemu process status.",
3067         type => 'string',
3068         enum => ['stopped', 'running'],
3069     },
3070     maxmem => {
3071         description => "Maximum memory in bytes.",
3072         type => 'integer',
3073         optional => 1,
3074         renderer => 'bytes',
3075     },
3076     maxdisk => {
3077         description => "Root disk size in bytes.",
3078         type => 'integer',
3079         optional => 1,
3080         renderer => 'bytes',
3081     },
3082     name => {
3083         description => "VM name.",
3084         type => 'string',
3085         optional => 1,
3086     },
3087     qmpstatus => {
3088         description => "Qemu QMP agent status.",
3089         type => 'string',
3090         optional => 1,
3091     },
3092     pid => {
3093         description => "PID of running qemu process.",
3094         type => 'integer',
3095         optional => 1,
3096     },
3097     uptime => {
3098         description => "Uptime.",
3099         type => 'integer',
3100         optional => 1,
3101         renderer => 'duration',
3102     },
3103     cpus => {
3104         description => "Maximum usable CPUs.",
3105         type => 'number',
3106         optional => 1,
3107     },
3108     lock => {
3109         description => "The current config lock, if any.",
3110         type => 'string',
3111         optional => 1,
3112     }
3113 };
3114
3115 my $last_proc_pid_stat;
3116
3117 # get VM status information
3118 # This must be fast and should not block ($full == false)
3119 # We only query KVM using QMP if $full == true (this can be slow)
3120 sub vmstatus {
3121     my ($opt_vmid, $full) = @_;
3122
3123     my $res = {};
3124
3125     my $storecfg = PVE::Storage::config();
3126
3127     my $list = vzlist();
3128     my $defaults = load_defaults();
3129
3130     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3131
3132     my $cpucount = $cpuinfo->{cpus} || 1;
3133
3134     foreach my $vmid (keys %$list) {
3135         next if $opt_vmid && ($vmid ne $opt_vmid);
3136
3137         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3138         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3139
3140         my $d = { vmid => $vmid };
3141         $d->{pid} = $list->{$vmid}->{pid};
3142
3143         # fixme: better status?
3144         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3145
3146         my $size = disksize($storecfg, $conf);
3147         if (defined($size)) {
3148             $d->{disk} = 0; # no info available
3149             $d->{maxdisk} = $size;
3150         } else {
3151             $d->{disk} = 0;
3152             $d->{maxdisk} = 0;
3153         }
3154
3155         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3156             * ($conf->{cores} || $defaults->{cores});
3157         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3158         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3159
3160         $d->{name} = $conf->{name} || "VM $vmid";
3161         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3162             : $defaults->{memory}*(1024*1024);
3163
3164         if ($conf->{balloon}) {
3165             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3166             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3167                 : $defaults->{shares};
3168         }
3169
3170         $d->{uptime} = 0;
3171         $d->{cpu} = 0;
3172         $d->{mem} = 0;
3173
3174         $d->{netout} = 0;
3175         $d->{netin} = 0;
3176
3177         $d->{diskread} = 0;
3178         $d->{diskwrite} = 0;
3179
3180         $d->{template} = PVE::QemuConfig->is_template($conf);
3181
3182         $d->{serial} = 1 if conf_has_serial($conf);
3183         $d->{lock} = $conf->{lock} if $conf->{lock};
3184
3185         $res->{$vmid} = $d;
3186     }
3187
3188     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3189     foreach my $dev (keys %$netdev) {
3190         next if $dev !~ m/^tap([1-9]\d*)i/;
3191         my $vmid = $1;
3192         my $d = $res->{$vmid};
3193         next if !$d;
3194
3195         $d->{netout} += $netdev->{$dev}->{receive};
3196         $d->{netin} += $netdev->{$dev}->{transmit};
3197
3198         if ($full) {
3199             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3200             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3201         }
3202
3203     }
3204
3205     my $ctime = gettimeofday;
3206
3207     foreach my $vmid (keys %$list) {
3208
3209         my $d = $res->{$vmid};
3210         my $pid = $d->{pid};
3211         next if !$pid;
3212
3213         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3214         next if !$pstat; # not running
3215
3216         my $used = $pstat->{utime} + $pstat->{stime};
3217
3218         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3219
3220         if ($pstat->{vsize}) {
3221             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3222         }
3223
3224         my $old = $last_proc_pid_stat->{$pid};
3225         if (!$old) {
3226             $last_proc_pid_stat->{$pid} = {
3227                 time => $ctime,
3228                 used => $used,
3229                 cpu => 0,
3230             };
3231             next;
3232         }
3233
3234         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3235
3236         if ($dtime > 1000) {
3237             my $dutime = $used -  $old->{used};
3238
3239             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3240             $last_proc_pid_stat->{$pid} = {
3241                 time => $ctime,
3242                 used => $used,
3243                 cpu => $d->{cpu},
3244             };
3245         } else {
3246             $d->{cpu} = $old->{cpu};
3247         }
3248     }
3249
3250     return $res if !$full;
3251
3252     my $qmpclient = PVE::QMPClient->new();
3253
3254     my $ballooncb = sub {
3255         my ($vmid, $resp) = @_;
3256
3257         my $info = $resp->{'return'};
3258         return if !$info->{max_mem};
3259
3260         my $d = $res->{$vmid};
3261
3262         # use memory assigned to VM
3263         $d->{maxmem} = $info->{max_mem};
3264         $d->{balloon} = $info->{actual};
3265
3266         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3267             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3268             $d->{freemem} = $info->{free_mem};
3269         }
3270
3271         $d->{ballooninfo} = $info;
3272     };
3273
3274     my $blockstatscb = sub {
3275         my ($vmid, $resp) = @_;
3276         my $data = $resp->{'return'} || [];
3277         my $totalrdbytes = 0;
3278         my $totalwrbytes = 0;
3279
3280         for my $blockstat (@$data) {
3281             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3282             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3283
3284             $blockstat->{device} =~ s/drive-//;
3285             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3286         }
3287         $res->{$vmid}->{diskread} = $totalrdbytes;
3288         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3289     };
3290
3291     my $statuscb = sub {
3292         my ($vmid, $resp) = @_;
3293
3294         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3295         # this fails if ballon driver is not loaded, so this must be
3296         # the last commnand (following command are aborted if this fails).
3297         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3298
3299         my $status = 'unknown';
3300         if (!defined($status = $resp->{'return'}->{status})) {
3301             warn "unable to get VM status\n";
3302             return;
3303         }
3304
3305         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3306     };
3307
3308     foreach my $vmid (keys %$list) {
3309         next if $opt_vmid && ($vmid ne $opt_vmid);
3310         next if !$res->{$vmid}->{pid}; # not running
3311         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3312     }
3313
3314     $qmpclient->queue_execute(undef, 2);
3315
3316     foreach my $vmid (keys %$list) {
3317         next if $opt_vmid && ($vmid ne $opt_vmid);
3318         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3319     }
3320
3321     return $res;
3322 }
3323
3324 sub foreach_drive {
3325     my ($conf, $func, @param) = @_;
3326
3327     foreach my $ds (valid_drive_names()) {
3328         next if !defined($conf->{$ds});
3329
3330         my $drive = parse_drive($ds, $conf->{$ds});
3331         next if !$drive;
3332
3333         &$func($ds, $drive, @param);
3334     }
3335 }
3336
3337 sub foreach_volid {
3338     my ($conf, $func, @param) = @_;
3339
3340     my $volhash = {};
3341
3342     my $test_volid = sub {
3343         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3344
3345         return if !$volid;
3346
3347         $volhash->{$volid}->{cdrom} //= 1;
3348         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3349
3350         $volhash->{$volid}->{replicate} //= 0;
3351         $volhash->{$volid}->{replicate} = 1 if $replicate;
3352
3353         $volhash->{$volid}->{shared} //= 0;
3354         $volhash->{$volid}->{shared} = 1 if $shared;
3355
3356         $volhash->{$volid}->{referenced_in_config} //= 0;
3357         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3358
3359         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3360             if defined($snapname);
3361         $volhash->{$volid}->{size} = $size if $size;
3362     };
3363
3364     foreach_drive($conf, sub {
3365         my ($ds, $drive) = @_;
3366         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3367     });
3368
3369     foreach my $snapname (keys %{$conf->{snapshots}}) {
3370         my $snap = $conf->{snapshots}->{$snapname};
3371         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3372         foreach_drive($snap, sub {
3373             my ($ds, $drive) = @_;
3374             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3375         });
3376     }
3377
3378     foreach my $volid (keys %$volhash) {
3379         &$func($volid, $volhash->{$volid}, @param);
3380     }
3381 }
3382
3383 sub conf_has_serial {
3384     my ($conf) = @_;
3385
3386     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3387         if ($conf->{"serial$i"}) {
3388             return 1;
3389         }
3390     }
3391
3392     return 0;
3393 }
3394
3395 sub vga_conf_has_spice {
3396     my ($vga) = @_;
3397
3398     my $vgaconf = parse_vga($vga);
3399     my $vgatype = $vgaconf->{type};
3400     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3401
3402     return $1 || 1;
3403 }
3404
3405 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3406 sub get_host_arch() {
3407     $host_arch = (POSIX::uname())[4] if !$host_arch;
3408     return $host_arch;
3409 }
3410
3411 sub is_native($) {
3412     my ($arch) = @_;
3413     return get_host_arch() eq $arch;
3414 }
3415
3416 my $default_machines = {
3417     x86_64 => 'pc',
3418     aarch64 => 'virt',
3419 };
3420
3421 sub get_basic_machine_info {
3422     my ($conf, $forcemachine) = @_;
3423
3424     my $arch = $conf->{arch} // get_host_arch();
3425     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3426     return ($arch, $machine);
3427 }
3428
3429 sub get_ovmf_files($) {
3430     my ($arch) = @_;
3431
3432     my $ovmf = $OVMF->{$arch}
3433         or die "no OVMF images known for architecture '$arch'\n";
3434
3435     return @$ovmf;
3436 }
3437
3438 my $Arch2Qemu = {
3439     aarch64 => '/usr/bin/qemu-system-aarch64',
3440     x86_64 => '/usr/bin/qemu-system-x86_64',
3441 };
3442 sub get_command_for_arch($) {
3443     my ($arch) = @_;
3444     return '/usr/bin/kvm' if is_native($arch);
3445
3446     my $cmd = $Arch2Qemu->{$arch}
3447         or die "don't know how to emulate architecture '$arch'\n";
3448     return $cmd;
3449 }
3450
3451 sub get_cpu_options {
3452     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3453
3454     my $cpuFlags = [];
3455     my $ostype = $conf->{ostype};
3456
3457     my $cpu = $kvm ? "kvm64" : "qemu64";
3458     if ($arch eq 'aarch64') {
3459         $cpu = 'cortex-a57';
3460     }
3461     my $hv_vendor_id;
3462     if (my $cputype = $conf->{cpu}) {
3463         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3464             or die "Cannot parse cpu description: $cputype\n";
3465         $cpu = $cpuconf->{cputype};
3466         $kvm_off = 1 if $cpuconf->{hidden};
3467         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3468
3469         if (defined(my $flags = $cpuconf->{flags})) {
3470             push @$cpuFlags, split(";", $flags);
3471         }
3472     }
3473
3474     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3475
3476     push @$cpuFlags , '-x2apic'
3477         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3478
3479     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3480
3481     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3482
3483     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3484
3485         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3486         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3487     }
3488
3489     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3490
3491     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3492
3493     push @$cpuFlags, 'kvm=off' if $kvm_off;
3494
3495     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3496         push @$cpuFlags, "vendor=${cpu_vendor}"
3497             if $cpu_vendor ne 'default';
3498     } elsif ($arch ne 'aarch64') {
3499         die "internal error"; # should not happen
3500     }
3501
3502     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3503
3504     return ('-cpu', $cpu);
3505 }
3506
3507 sub config_to_command {
3508     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3509
3510     my $cmd = [];
3511     my $globalFlags = [];
3512     my $machineFlags = [];
3513     my $rtcFlags = [];
3514     my $devices = [];
3515     my $pciaddr = '';
3516     my $bridges = {};
3517     my $kvmver = kvm_user_version();
3518     my $vernum = 0; # unknown
3519     my $ostype = $conf->{ostype};
3520     my $winversion = windows_version($ostype);
3521     my $kvm = $conf->{kvm};
3522
3523     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3524     $kvm //= 1 if is_native($arch);
3525
3526     if ($kvm) {
3527         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3528             if !defined kvm_version();
3529     }
3530
3531     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3532         $vernum = $1*1000000+$2*1000;
3533     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3534         $vernum = $1*1000000+$2*1000+$3;
3535     }
3536
3537     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3538
3539     my $have_ovz = -f '/proc/vz/vestat';
3540
3541     my $q35 = machine_type_is_q35($conf);
3542     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3543     my $use_old_bios_files = undef;
3544     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3545
3546     my $cpuunits = defined($conf->{cpuunits}) ?
3547             $conf->{cpuunits} : $defaults->{cpuunits};
3548
3549     push @$cmd, get_command_for_arch($arch);
3550
3551     push @$cmd, '-id', $vmid;
3552
3553     my $vmname = $conf->{name} || "vm$vmid";
3554
3555     push @$cmd, '-name', $vmname;
3556
3557     my $use_virtio = 0;
3558
3559     my $qmpsocket = qmp_socket($vmid);
3560     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3561     push @$cmd, '-mon', "chardev=qmp,mode=control";
3562
3563     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3564         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3565         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3566     }
3567
3568     push @$cmd, '-pidfile' , pidfile_name($vmid);
3569
3570     push @$cmd, '-daemonize';
3571
3572     if ($conf->{smbios1}) {
3573         my $smbios_conf = parse_smbios1($conf->{smbios1});
3574         if ($smbios_conf->{base64}) {
3575             # Do not pass base64 flag to qemu
3576             delete $smbios_conf->{base64};
3577             my $smbios_string = "";
3578             foreach my $key (keys %$smbios_conf) {
3579                 my $value;
3580                 if ($key eq "uuid") {
3581                     $value = $smbios_conf->{uuid}
3582                 } else {
3583                     $value = decode_base64($smbios_conf->{$key});
3584                 }
3585                 # qemu accepts any binary data, only commas need escaping by double comma
3586                 $value =~ s/,/,,/g;
3587                 $smbios_string .= "," . $key . "=" . $value if $value;
3588             }
3589             push @$cmd, '-smbios', "type=1" . $smbios_string;
3590         } else {
3591             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3592         }
3593     }
3594
3595     if ($conf->{vmgenid}) {
3596         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3597     }
3598
3599     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3600     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3601         die "uefi base image not found\n" if ! -f $ovmf_code;
3602
3603         my $path;
3604         my $format;
3605         if (my $efidisk = $conf->{efidisk0}) {
3606             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3607             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3608             $format = $d->{format};
3609             if ($storeid) {
3610                 $path = PVE::Storage::path($storecfg, $d->{file});
3611                 if (!defined($format)) {
3612                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3613                     $format = qemu_img_format($scfg, $volname);
3614                 }
3615             } else {
3616                 $path = $d->{file};
3617                 die "efidisk format must be specified\n"
3618                     if !defined($format);
3619             }
3620         } else {
3621             warn "no efidisk configured! Using temporary efivars disk.\n";
3622             $path = "/tmp/$vmid-ovmf.fd";
3623             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3624             $format = 'raw';
3625         }
3626
3627         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3628         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3629     }
3630
3631     # load q35 config
3632     if ($q35) {
3633         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3634         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3635             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3636         } else {
3637             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3638         }
3639     }
3640
3641     # add usb controllers
3642     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3643     push @$devices, @usbcontrollers if @usbcontrollers;
3644     my $vga = parse_vga($conf->{vga});
3645
3646     my $qxlnum = vga_conf_has_spice($conf->{vga});
3647     $vga->{type} = 'qxl' if $qxlnum;
3648
3649     if (!$vga->{type}) {
3650         if ($arch eq 'aarch64') {
3651             $vga->{type} = 'virtio';
3652         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3653             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3654         } else {
3655             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3656         }
3657     }
3658
3659     # enable absolute mouse coordinates (needed by vnc)
3660     my $tablet;
3661     if (defined($conf->{tablet})) {
3662         $tablet = $conf->{tablet};
3663     } else {
3664         $tablet = $defaults->{tablet};
3665         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3666         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3667     }
3668
3669     if ($tablet) {
3670         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3671         my $kbd = print_keyboarddevice_full($conf, $arch);
3672         push @$devices, '-device', $kbd if defined($kbd);
3673     }
3674
3675     my $kvm_off = 0;
3676     my $gpu_passthrough;
3677
3678     # host pci devices
3679     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3680         my $d = parse_hostpci($conf->{"hostpci$i"});
3681         next if !$d;
3682
3683         my $pcie = $d->{pcie};
3684         if ($pcie) {
3685             die "q35 machine model is not enabled" if !$q35;
3686             # win7 wants to have the pcie devices directly on the pcie bus
3687             # instead of in the root port
3688             if ($winversion == 7) {
3689                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3690             } else {
3691                 $pciaddr = print_pcie_addr("hostpci$i");
3692             }
3693         } else {
3694             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3695         }
3696
3697         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3698         my $romfile = $d->{romfile};
3699
3700         my $xvga = '';
3701         if ($d->{'x-vga'}) {
3702             $xvga = ',x-vga=on';
3703             $kvm_off = 1;
3704             $vga->{type} = 'none' if !defined($conf->{vga});
3705             $gpu_passthrough = 1;
3706
3707             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3708                 $xvga = "";
3709             }
3710         }
3711         my $pcidevices = $d->{pciid};
3712         my $multifunction = 1 if @$pcidevices > 1;
3713         my $sysfspath;
3714         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3715             my $id = $pcidevices->[0]->{id};
3716             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3717             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3718         } elsif ($d->{mdev}) {
3719             warn "ignoring mediated device with multifunction device\n";
3720         }
3721
3722         my $j=0;
3723         foreach my $pcidevice (@$pcidevices) {
3724
3725             my $id = "hostpci$i";
3726             $id .= ".$j" if $multifunction;
3727             my $addr = $pciaddr;
3728             $addr .= ".$j" if $multifunction;
3729             my $devicestr = "vfio-pci";
3730             if ($sysfspath) {
3731                 $devicestr .= ",sysfsdev=$sysfspath";
3732             } else {
3733                 $devicestr .= ",host=$pcidevice->{id}";
3734             }
3735             $devicestr .= ",id=$id$addr";
3736
3737             if($j == 0){
3738                 $devicestr .= "$rombar$xvga";
3739                 $devicestr .= ",multifunction=on" if $multifunction;
3740                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3741             }
3742
3743             push @$devices, '-device', $devicestr;
3744             $j++;
3745         }
3746     }
3747
3748     # usb devices
3749     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3750     push @$devices, @usbdevices if @usbdevices;
3751     # serial devices
3752     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3753         if (my $path = $conf->{"serial$i"}) {
3754             if ($path eq 'socket') {
3755                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3756                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3757                 # On aarch64, serial0 is the UART device. Qemu only allows
3758                 # connecting UART devices via the '-serial' command line, as
3759                 # the device has a fixed slot on the hardware...
3760                 if ($arch eq 'aarch64' && $i == 0) {
3761                     push @$devices, '-serial', "chardev:serial$i";
3762                 } else {
3763                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3764                 }
3765             } else {
3766                 die "no such serial device\n" if ! -c $path;
3767                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3768                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3769             }
3770         }
3771     }
3772
3773     # parallel devices
3774     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3775         if (my $path = $conf->{"parallel$i"}) {
3776             die "no such parallel device\n" if ! -c $path;
3777             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3778             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3779             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3780         }
3781     }
3782
3783
3784     my $sockets = 1;
3785     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3786     $sockets = $conf->{sockets} if  $conf->{sockets};
3787
3788     my $cores = $conf->{cores} || 1;
3789
3790     my $maxcpus = $sockets * $cores;
3791
3792     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3793
3794     my $allowed_vcpus = $cpuinfo->{cpus};
3795
3796     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3797         if ($allowed_vcpus < $maxcpus);
3798
3799     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3800
3801         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3802         for (my $i = 2; $i <= $vcpus; $i++)  {
3803             my $cpustr = print_cpu_device($conf,$i);
3804             push @$cmd, '-device', $cpustr;
3805         }
3806
3807     } else {
3808
3809         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3810     }
3811     push @$cmd, '-nodefaults';
3812
3813     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3814
3815     my $bootindex_hash = {};
3816     my $i = 1;
3817     foreach my $o (split(//, $bootorder)) {
3818         $bootindex_hash->{$o} = $i*100;
3819         $i++;
3820     }
3821
3822     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3823
3824     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3825
3826     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3827
3828     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3829         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3830         my $socket = vnc_socket($vmid);
3831         push @$cmd,  '-vnc', "unix:$socket,password";
3832     } else {
3833         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3834         push @$cmd, '-nographic';
3835     }
3836
3837     # time drift fix
3838     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3839
3840     my $useLocaltime = $conf->{localtime};
3841
3842     if ($winversion >= 5) { # windows
3843         $useLocaltime = 1 if !defined($conf->{localtime});
3844
3845         # use time drift fix when acpi is enabled
3846         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3847             $tdf = 1 if !defined($conf->{tdf});
3848         }
3849     }
3850
3851     if ($winversion >= 6) {
3852         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3853         push @$cmd, '-no-hpet';
3854     }
3855
3856     push @$rtcFlags, 'driftfix=slew' if $tdf;
3857
3858     if (!$kvm) {
3859         push @$machineFlags, 'accel=tcg';
3860     }
3861
3862     if ($machine_type) {
3863         push @$machineFlags, "type=${machine_type}";
3864     }
3865
3866     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3867         push @$rtcFlags, "base=$conf->{startdate}";
3868     } elsif ($useLocaltime) {
3869         push @$rtcFlags, 'base=localtime';
3870     }
3871
3872     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3873
3874     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3875
3876     push @$cmd, '-S' if $conf->{freeze};
3877
3878     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3879
3880     # enable sound
3881     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3882     #push @$cmd, '-soundhw', 'es1370';
3883     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3884
3885     if (parse_guest_agent($conf)->{enabled}) {
3886         my $qgasocket = qmp_socket($vmid, 1);
3887         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3888         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3889         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3890         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3891     }
3892
3893     my $spice_port;
3894
3895     if ($qxlnum) {
3896         if ($qxlnum > 1) {
3897             if ($winversion){
3898                 for(my $i = 1; $i < $qxlnum; $i++){
3899                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3900                 }
3901             } else {
3902                 # assume other OS works like Linux
3903                 my ($ram, $vram) = ("134217728", "67108864");
3904                 if ($vga->{memory}) {
3905                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3906                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3907                 }
3908                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3909                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3910             }
3911         }
3912
3913         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3914
3915         my $nodename = PVE::INotify::nodename();
3916         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3917         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3918         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3919         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3920         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3921
3922         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3923
3924         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3925         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3926         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3927     }
3928
3929     # enable balloon by default, unless explicitly disabled
3930     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3931         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3932         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3933     }
3934
3935     if ($conf->{watchdog}) {
3936         my $wdopts = parse_watchdog($conf->{watchdog});
3937         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3938         my $watchdog = $wdopts->{model} || 'i6300esb';
3939         push @$devices, '-device', "$watchdog$pciaddr";
3940         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3941     }
3942
3943     my $vollist = [];
3944     my $scsicontroller = {};
3945     my $ahcicontroller = {};
3946     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3947
3948     # Add iscsi initiator name if available
3949     if (my $initiator = get_initiator_name()) {
3950         push @$devices, '-iscsi', "initiator-name=$initiator";
3951     }
3952
3953     foreach_drive($conf, sub {
3954         my ($ds, $drive) = @_;
3955
3956         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3957             push @$vollist, $drive->{file};
3958         }
3959
3960         # ignore efidisk here, already added in bios/fw handling code above
3961         return if $drive->{interface} eq 'efidisk';
3962
3963         $use_virtio = 1 if $ds =~ m/^virtio/;
3964
3965         if (drive_is_cdrom ($drive)) {
3966             if ($bootindex_hash->{d}) {
3967                 $drive->{bootindex} = $bootindex_hash->{d};
3968                 $bootindex_hash->{d} += 1;
3969             }
3970         } else {
3971             if ($bootindex_hash->{c}) {
3972                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3973                 $bootindex_hash->{c} += 1;
3974             }
3975         }
3976
3977         if($drive->{interface} eq 'virtio'){
3978            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3979         }
3980
3981         if ($drive->{interface} eq 'scsi') {
3982
3983             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3984
3985             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3986             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3987
3988             my $iothread = '';
3989             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3990                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3991                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3992             } elsif ($drive->{iothread}) {
3993                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3994             }
3995
3996             my $queues = '';
3997             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3998                 $queues = ",num_queues=$drive->{queues}";
3999             }
4000
4001             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4002             $scsicontroller->{$controller}=1;
4003         }
4004
4005         if ($drive->{interface} eq 'sata') {
4006            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4007            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4008            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4009            $ahcicontroller->{$controller}=1;
4010         }
4011
4012         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4013         push @$devices, '-drive',$drive_cmd;
4014         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4015     });
4016
4017     for (my $i = 0; $i < $MAX_NETS; $i++) {
4018          next if !$conf->{"net$i"};
4019          my $d = parse_net($conf->{"net$i"});
4020          next if !$d;
4021
4022          $use_virtio = 1 if $d->{model} eq 'virtio';
4023
4024          if ($bootindex_hash->{n}) {
4025             $d->{bootindex} = $bootindex_hash->{n};
4026             $bootindex_hash->{n} += 1;
4027          }
4028
4029          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4030          push @$devices, '-netdev', $netdevfull;
4031
4032          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4033          push @$devices, '-device', $netdevicefull;
4034     }
4035
4036     if ($conf->{ivshmem}) {
4037         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4038
4039         my $bus;
4040         if ($q35) {
4041             $bus = print_pcie_addr("ivshmem");
4042         } else {
4043             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4044         }
4045
4046         my $ivshmem_name = $ivshmem->{name} // $vmid;
4047         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4048
4049         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4050         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4051     }
4052
4053     if (!$q35) {
4054         # add pci bridges
4055         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4056            $bridges->{1} = 1;
4057            $bridges->{2} = 1;
4058         }
4059
4060         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4061
4062         while (my ($k, $v) = each %$bridges) {
4063             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4064             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4065         }
4066     }
4067
4068     push @$cmd, @$devices;
4069     push @$cmd, '-rtc', join(',', @$rtcFlags)
4070         if scalar(@$rtcFlags);
4071     push @$cmd, '-machine', join(',', @$machineFlags)
4072         if scalar(@$machineFlags);
4073     push @$cmd, '-global', join(',', @$globalFlags)
4074         if scalar(@$globalFlags);
4075
4076     if (my $vmstate = $conf->{vmstate}) {
4077         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4078         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4079         push @$cmd, '-loadstate', $statepath;
4080     }
4081
4082     # add custom args
4083     if ($conf->{args}) {
4084         my $aa = PVE::Tools::split_args($conf->{args});
4085         push @$cmd, @$aa;
4086     }
4087
4088     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4089 }
4090
4091 sub vnc_socket {
4092     my ($vmid) = @_;
4093     return "${var_run_tmpdir}/$vmid.vnc";
4094 }
4095
4096 sub spice_port {
4097     my ($vmid) = @_;
4098
4099     my $res = vm_mon_cmd($vmid, 'query-spice');
4100
4101     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4102 }
4103
4104 sub qmp_socket {
4105     my ($vmid, $qga, $name) = @_;
4106     my $sockettype = $qga ? 'qga' : 'qmp';
4107     my $ext = $name ? '-'.$name : '';
4108     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4109 }
4110
4111 sub pidfile_name {
4112     my ($vmid) = @_;
4113     return "${var_run_tmpdir}/$vmid.pid";
4114 }
4115
4116 sub vm_devices_list {
4117     my ($vmid) = @_;
4118
4119     my $res = vm_mon_cmd($vmid, 'query-pci');
4120     my $devices_to_check = [];
4121     my $devices = {};
4122     foreach my $pcibus (@$res) {
4123         push @$devices_to_check, @{$pcibus->{devices}},
4124     }
4125
4126     while (@$devices_to_check) {
4127         my $to_check = [];
4128         for my $d (@$devices_to_check) {
4129             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4130             next if !$d->{'pci_bridge'};
4131
4132             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4133             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4134         }
4135         $devices_to_check = $to_check;
4136     }
4137
4138     my $resblock = vm_mon_cmd($vmid, 'query-block');
4139     foreach my $block (@$resblock) {
4140         if($block->{device} =~ m/^drive-(\S+)/){
4141                 $devices->{$1} = 1;
4142         }
4143     }
4144
4145     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4146     foreach my $mice (@$resmice) {
4147         if ($mice->{name} eq 'QEMU HID Tablet') {
4148             $devices->{tablet} = 1;
4149             last;
4150         }
4151     }
4152
4153     # for usb devices there is no query-usb
4154     # but we can iterate over the entries in
4155     # qom-list path=/machine/peripheral
4156     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4157     foreach my $per (@$resperipheral) {
4158         if ($per->{name} =~ m/^usb\d+$/) {
4159             $devices->{$per->{name}} = 1;
4160         }
4161     }
4162
4163     return $devices;
4164 }
4165
4166 sub vm_deviceplug {
4167     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4168
4169     my $q35 = machine_type_is_q35($conf);
4170
4171     my $devices_list = vm_devices_list($vmid);
4172     return 1 if defined($devices_list->{$deviceid});
4173
4174     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4175
4176     if ($deviceid eq 'tablet') {
4177
4178         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4179
4180     } elsif ($deviceid eq 'keyboard') {
4181
4182         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4183
4184     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4185
4186         die "usb hotplug currently not reliable\n";
4187         # since we can't reliably hot unplug all added usb devices
4188         # and usb passthrough disables live migration
4189         # we disable usb hotplugging for now
4190         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4191
4192     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4193
4194         qemu_iothread_add($vmid, $deviceid, $device);
4195
4196         qemu_driveadd($storecfg, $vmid, $device);
4197         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4198
4199         qemu_deviceadd($vmid, $devicefull);
4200         eval { qemu_deviceaddverify($vmid, $deviceid); };
4201         if (my $err = $@) {
4202             eval { qemu_drivedel($vmid, $deviceid); };
4203             warn $@ if $@;
4204             die $err;
4205         }
4206
4207     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4208
4209
4210         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4211         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4212         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4213
4214         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4215
4216         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4217             qemu_iothread_add($vmid, $deviceid, $device);
4218             $devicefull .= ",iothread=iothread-$deviceid";
4219         }
4220
4221         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4222             $devicefull .= ",num_queues=$device->{queues}";
4223         }
4224
4225         qemu_deviceadd($vmid, $devicefull);
4226         qemu_deviceaddverify($vmid, $deviceid);
4227
4228     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4229
4230         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4231         qemu_driveadd($storecfg, $vmid, $device);
4232
4233         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4234         eval { qemu_deviceadd($vmid, $devicefull); };
4235         if (my $err = $@) {
4236             eval { qemu_drivedel($vmid, $deviceid); };
4237             warn $@ if $@;
4238             die $err;
4239         }
4240
4241     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4242
4243         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4244
4245         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4246         my $use_old_bios_files = undef;
4247         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4248
4249         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4250         qemu_deviceadd($vmid, $netdevicefull);
4251         eval {
4252             qemu_deviceaddverify($vmid, $deviceid);
4253             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4254         };
4255         if (my $err = $@) {
4256             eval { qemu_netdevdel($vmid, $deviceid); };
4257             warn $@ if $@;
4258             die $err;
4259         }
4260
4261     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4262
4263         my $bridgeid = $2;
4264         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4265         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4266
4267         qemu_deviceadd($vmid, $devicefull);
4268         qemu_deviceaddverify($vmid, $deviceid);
4269
4270     } else {
4271         die "can't hotplug device '$deviceid'\n";
4272     }
4273
4274     return 1;
4275 }
4276
4277 # fixme: this should raise exceptions on error!
4278 sub vm_deviceunplug {
4279     my ($vmid, $conf, $deviceid) = @_;
4280
4281     my $devices_list = vm_devices_list($vmid);
4282     return 1 if !defined($devices_list->{$deviceid});
4283
4284     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4285
4286     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4287
4288         qemu_devicedel($vmid, $deviceid);
4289
4290     } elsif ($deviceid =~ m/^usb\d+$/) {
4291
4292         die "usb hotplug currently not reliable\n";
4293         # when unplugging usb devices this way,
4294         # there may be remaining usb controllers/hubs
4295         # so we disable it for now
4296         qemu_devicedel($vmid, $deviceid);
4297         qemu_devicedelverify($vmid, $deviceid);
4298
4299     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4300
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303         qemu_drivedel($vmid, $deviceid);
4304         qemu_iothread_del($conf, $vmid, $deviceid);
4305
4306     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4307
4308         qemu_devicedel($vmid, $deviceid);
4309         qemu_devicedelverify($vmid, $deviceid);
4310         qemu_iothread_del($conf, $vmid, $deviceid);
4311
4312     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4313
4314         qemu_devicedel($vmid, $deviceid);
4315         qemu_drivedel($vmid, $deviceid);
4316         qemu_deletescsihw($conf, $vmid, $deviceid);
4317
4318     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4319
4320         qemu_devicedel($vmid, $deviceid);
4321         qemu_devicedelverify($vmid, $deviceid);
4322         qemu_netdevdel($vmid, $deviceid);
4323
4324     } else {
4325         die "can't unplug device '$deviceid'\n";
4326     }
4327
4328     return 1;
4329 }
4330
4331 sub qemu_deviceadd {
4332     my ($vmid, $devicefull) = @_;
4333
4334     $devicefull = "driver=".$devicefull;
4335     my %options =  split(/[=,]/, $devicefull);
4336
4337     vm_mon_cmd($vmid, "device_add" , %options);
4338 }
4339
4340 sub qemu_devicedel {
4341     my ($vmid, $deviceid) = @_;
4342
4343     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4344 }
4345
4346 sub qemu_iothread_add {
4347     my($vmid, $deviceid, $device) = @_;
4348
4349     if ($device->{iothread}) {
4350         my $iothreads = vm_iothreads_list($vmid);
4351         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4352     }
4353 }
4354
4355 sub qemu_iothread_del {
4356     my($conf, $vmid, $deviceid) = @_;
4357
4358     my $confid = $deviceid;
4359     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4360         $confid = 'scsi' . $1;
4361     }
4362     my $device = parse_drive($confid, $conf->{$confid});
4363     if ($device->{iothread}) {
4364         my $iothreads = vm_iothreads_list($vmid);
4365         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4366     }
4367 }
4368
4369 sub qemu_objectadd {
4370     my($vmid, $objectid, $qomtype) = @_;
4371
4372     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4373
4374     return 1;
4375 }
4376
4377 sub qemu_objectdel {
4378     my($vmid, $objectid) = @_;
4379
4380     vm_mon_cmd($vmid, "object-del", id => $objectid);
4381
4382     return 1;
4383 }
4384
4385 sub qemu_driveadd {
4386     my ($storecfg, $vmid, $device) = @_;
4387
4388     my $drive = print_drive_full($storecfg, $vmid, $device);
4389     $drive =~ s/\\/\\\\/g;
4390     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4391
4392     # If the command succeeds qemu prints: "OK"
4393     return 1 if $ret =~ m/OK/s;
4394
4395     die "adding drive failed: $ret\n";
4396 }
4397
4398 sub qemu_drivedel {
4399     my($vmid, $deviceid) = @_;
4400
4401     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4402     $ret =~ s/^\s+//;
4403
4404     return 1 if $ret eq "";
4405
4406     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4407     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4408
4409     die "deleting drive $deviceid failed : $ret\n";
4410 }
4411
4412 sub qemu_deviceaddverify {
4413     my ($vmid, $deviceid) = @_;
4414
4415     for (my $i = 0; $i <= 5; $i++) {
4416          my $devices_list = vm_devices_list($vmid);
4417          return 1 if defined($devices_list->{$deviceid});
4418          sleep 1;
4419     }
4420
4421     die "error on hotplug device '$deviceid'\n";
4422 }
4423
4424
4425 sub qemu_devicedelverify {
4426     my ($vmid, $deviceid) = @_;
4427
4428     # need to verify that the device is correctly removed as device_del
4429     # is async and empty return is not reliable
4430
4431     for (my $i = 0; $i <= 5; $i++) {
4432          my $devices_list = vm_devices_list($vmid);
4433          return 1 if !defined($devices_list->{$deviceid});
4434          sleep 1;
4435     }
4436
4437     die "error on hot-unplugging device '$deviceid'\n";
4438 }
4439
4440 sub qemu_findorcreatescsihw {
4441     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4442
4443     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4444
4445     my $scsihwid="$controller_prefix$controller";
4446     my $devices_list = vm_devices_list($vmid);
4447
4448     if(!defined($devices_list->{$scsihwid})) {
4449         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4450     }
4451
4452     return 1;
4453 }
4454
4455 sub qemu_deletescsihw {
4456     my ($conf, $vmid, $opt) = @_;
4457
4458     my $device = parse_drive($opt, $conf->{$opt});
4459
4460     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4461         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4462         return 1;
4463     }
4464
4465     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4466
4467     my $devices_list = vm_devices_list($vmid);
4468     foreach my $opt (keys %{$devices_list}) {
4469         if (PVE::QemuServer::is_valid_drivename($opt)) {
4470             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4471             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4472                 return 1;
4473             }
4474         }
4475     }
4476
4477     my $scsihwid="scsihw$controller";
4478
4479     vm_deviceunplug($vmid, $conf, $scsihwid);
4480
4481     return 1;
4482 }
4483
4484 sub qemu_add_pci_bridge {
4485     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4486
4487     my $bridges = {};
4488
4489     my $bridgeid;
4490
4491     print_pci_addr($device, $bridges, $arch, $machine_type);
4492
4493     while (my ($k, $v) = each %$bridges) {
4494         $bridgeid = $k;
4495     }
4496     return 1 if !defined($bridgeid) || $bridgeid < 1;
4497
4498     my $bridge = "pci.$bridgeid";
4499     my $devices_list = vm_devices_list($vmid);
4500
4501     if (!defined($devices_list->{$bridge})) {
4502         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4503     }
4504
4505     return 1;
4506 }
4507
4508 sub qemu_set_link_status {
4509     my ($vmid, $device, $up) = @_;
4510
4511     vm_mon_cmd($vmid, "set_link", name => $device,
4512                up => $up ? JSON::true : JSON::false);
4513 }
4514
4515 sub qemu_netdevadd {
4516     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4517
4518     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4519     my %options =  split(/[=,]/, $netdev);
4520
4521     vm_mon_cmd($vmid, "netdev_add",  %options);
4522     return 1;
4523 }
4524
4525 sub qemu_netdevdel {
4526     my ($vmid, $deviceid) = @_;
4527
4528     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4529 }
4530
4531 sub qemu_usb_hotplug {
4532     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4533
4534     return if !$device;
4535
4536     # remove the old one first
4537     vm_deviceunplug($vmid, $conf, $deviceid);
4538
4539     # check if xhci controller is necessary and available
4540     if ($device->{usb3}) {
4541
4542         my $devicelist = vm_devices_list($vmid);
4543
4544         if (!$devicelist->{xhci}) {
4545             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4546             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4547         }
4548     }
4549     my $d = parse_usb_device($device->{host});
4550     $d->{usb3} = $device->{usb3};
4551
4552     # add the new one
4553     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4554 }
4555
4556 sub qemu_cpu_hotplug {
4557     my ($vmid, $conf, $vcpus) = @_;
4558
4559     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4560
4561     my $sockets = 1;
4562     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4563     $sockets = $conf->{sockets} if  $conf->{sockets};
4564     my $cores = $conf->{cores} || 1;
4565     my $maxcpus = $sockets * $cores;
4566
4567     $vcpus = $maxcpus if !$vcpus;
4568
4569     die "you can't add more vcpus than maxcpus\n"
4570         if $vcpus > $maxcpus;
4571
4572     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4573
4574     if ($vcpus < $currentvcpus) {
4575
4576         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4577
4578             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4579                 qemu_devicedel($vmid, "cpu$i");
4580                 my $retry = 0;
4581                 my $currentrunningvcpus = undef;
4582                 while (1) {
4583                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4584                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4585                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4586                     $retry++;
4587                     sleep 1;
4588                 }
4589                 #update conf after each succesfull cpu unplug
4590                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4591                 PVE::QemuConfig->write_config($vmid, $conf);
4592             }
4593         } else {
4594             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4595         }
4596
4597         return;
4598     }
4599
4600     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4601     die "vcpus in running vm does not match its configuration\n"
4602         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4603
4604     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4605
4606         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4607             my $cpustr = print_cpu_device($conf, $i);
4608             qemu_deviceadd($vmid, $cpustr);
4609
4610             my $retry = 0;
4611             my $currentrunningvcpus = undef;
4612             while (1) {
4613                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4614                 last if scalar(@{$currentrunningvcpus}) == $i;
4615                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4616                 sleep 1;
4617                 $retry++;
4618             }
4619             #update conf after each succesfull cpu hotplug
4620             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4621             PVE::QemuConfig->write_config($vmid, $conf);
4622         }
4623     } else {
4624
4625         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4626             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4627         }
4628     }
4629 }
4630
4631 sub qemu_block_set_io_throttle {
4632     my ($vmid, $deviceid,
4633         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4634         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4635         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4636         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4637
4638     return if !check_running($vmid) ;
4639
4640     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4641         bps => int($bps),
4642         bps_rd => int($bps_rd),
4643         bps_wr => int($bps_wr),
4644         iops => int($iops),
4645         iops_rd => int($iops_rd),
4646         iops_wr => int($iops_wr),
4647         bps_max => int($bps_max),
4648         bps_rd_max => int($bps_rd_max),
4649         bps_wr_max => int($bps_wr_max),
4650         iops_max => int($iops_max),
4651         iops_rd_max => int($iops_rd_max),
4652         iops_wr_max => int($iops_wr_max),
4653         bps_max_length => int($bps_max_length),
4654         bps_rd_max_length => int($bps_rd_max_length),
4655         bps_wr_max_length => int($bps_wr_max_length),
4656         iops_max_length => int($iops_max_length),
4657         iops_rd_max_length => int($iops_rd_max_length),
4658         iops_wr_max_length => int($iops_wr_max_length),
4659     );
4660
4661 }
4662
4663 # old code, only used to shutdown old VM after update
4664 sub __read_avail {
4665     my ($fh, $timeout) = @_;
4666
4667     my $sel = new IO::Select;
4668     $sel->add($fh);
4669
4670     my $res = '';
4671     my $buf;
4672
4673     my @ready;
4674     while (scalar (@ready = $sel->can_read($timeout))) {
4675         my $count;
4676         if ($count = $fh->sysread($buf, 8192)) {
4677             if ($buf =~ /^(.*)\(qemu\) $/s) {
4678                 $res .= $1;
4679                 last;
4680             } else {
4681                 $res .= $buf;
4682             }
4683         } else {
4684             if (!defined($count)) {
4685                 die "$!\n";
4686             }
4687             last;
4688         }
4689     }
4690
4691     die "monitor read timeout\n" if !scalar(@ready);
4692
4693     return $res;
4694 }
4695
4696 sub qemu_block_resize {
4697     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4698
4699     my $running = check_running($vmid);
4700
4701     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4702
4703     return if !$running;
4704
4705     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4706
4707 }
4708
4709 sub qemu_volume_snapshot {
4710     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4711
4712     my $running = check_running($vmid);
4713
4714     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4715         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4716     } else {
4717         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4718     }
4719 }
4720
4721 sub qemu_volume_snapshot_delete {
4722     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4723
4724     my $running = check_running($vmid);
4725
4726     if($running) {
4727
4728         $running = undef;
4729         my $conf = PVE::QemuConfig->load_config($vmid);
4730         foreach_drive($conf, sub {
4731             my ($ds, $drive) = @_;
4732             $running = 1 if $drive->{file} eq $volid;
4733         });
4734     }
4735
4736     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4737         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4738     } else {
4739         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4740     }
4741 }
4742
4743 sub set_migration_caps {
4744     my ($vmid) = @_;
4745
4746     my $cap_ref = [];
4747
4748     my $enabled_cap = {
4749         "auto-converge" => 1,
4750         "xbzrle" => 1,
4751         "x-rdma-pin-all" => 0,
4752         "zero-blocks" => 0,
4753         "compress" => 0
4754     };
4755
4756     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4757
4758     for my $supported_capability (@$supported_capabilities) {
4759         push @$cap_ref, {
4760             capability => $supported_capability->{capability},
4761             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4762         };
4763     }
4764
4765     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4766 }
4767
4768 my $fast_plug_option = {
4769     'lock' => 1,
4770     'name' => 1,
4771     'onboot' => 1,
4772     'shares' => 1,
4773     'startup' => 1,
4774     'description' => 1,
4775     'protection' => 1,
4776     'vmstatestorage' => 1,
4777     'hookscript' => 1,
4778 };
4779
4780 # hotplug changes in [PENDING]
4781 # $selection hash can be used to only apply specified options, for
4782 # example: { cores => 1 } (only apply changed 'cores')
4783 # $errors ref is used to return error messages
4784 sub vmconfig_hotplug_pending {
4785     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4786
4787     my $defaults = load_defaults();
4788     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4789
4790     # commit values which do not have any impact on running VM first
4791     # Note: those option cannot raise errors, we we do not care about
4792     # $selection and always apply them.
4793
4794     my $add_error = sub {
4795         my ($opt, $msg) = @_;
4796         $errors->{$opt} = "hotplug problem - $msg";
4797     };
4798
4799     my $changes = 0;
4800     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4801         if ($fast_plug_option->{$opt}) {
4802             $conf->{$opt} = $conf->{pending}->{$opt};
4803             delete $conf->{pending}->{$opt};
4804             $changes = 1;
4805         }
4806     }
4807
4808     if ($changes) {
4809         PVE::QemuConfig->write_config($vmid, $conf);
4810         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4811     }
4812
4813     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4814
4815     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4816     while (my ($opt, $force) = each %$pending_delete_hash) {
4817         next if $selection && !$selection->{$opt};
4818         eval {
4819             if ($opt eq 'hotplug') {
4820                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4821             } elsif ($opt eq 'tablet') {
4822                 die "skip\n" if !$hotplug_features->{usb};
4823                 if ($defaults->{tablet}) {
4824                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4825                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4826                         if $arch eq 'aarch64';
4827                 } else {
4828                     vm_deviceunplug($vmid, $conf, 'tablet');
4829                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4830                 }
4831             } elsif ($opt =~ m/^usb\d+/) {
4832                 die "skip\n";
4833                 # since we cannot reliably hot unplug usb devices
4834                 # we are disabling it
4835                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4836                 vm_deviceunplug($vmid, $conf, $opt);
4837             } elsif ($opt eq 'vcpus') {
4838                 die "skip\n" if !$hotplug_features->{cpu};
4839                 qemu_cpu_hotplug($vmid, $conf, undef);
4840             } elsif ($opt eq 'balloon') {
4841                 # enable balloon device is not hotpluggable
4842                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4843                 # here we reset the ballooning value to memory
4844                 my $balloon = $conf->{memory} || $defaults->{memory};
4845                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4846             } elsif ($fast_plug_option->{$opt}) {
4847                 # do nothing
4848             } elsif ($opt =~ m/^net(\d+)$/) {
4849                 die "skip\n" if !$hotplug_features->{network};
4850                 vm_deviceunplug($vmid, $conf, $opt);
4851             } elsif (is_valid_drivename($opt)) {
4852                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4853                 vm_deviceunplug($vmid, $conf, $opt);
4854                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4855             } elsif ($opt =~ m/^memory$/) {
4856                 die "skip\n" if !$hotplug_features->{memory};
4857                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4858             } elsif ($opt eq 'cpuunits') {
4859                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4860             } elsif ($opt eq 'cpulimit') {
4861                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4862             } else {
4863                 die "skip\n";
4864             }
4865         };
4866         if (my $err = $@) {
4867             &$add_error($opt, $err) if $err ne "skip\n";
4868         } else {
4869             # save new config if hotplug was successful
4870             delete $conf->{$opt};
4871             vmconfig_undelete_pending_option($conf, $opt);
4872             PVE::QemuConfig->write_config($vmid, $conf);
4873             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4874         }
4875     }
4876
4877     my $apply_pending_cloudinit;
4878     $apply_pending_cloudinit = sub {
4879         my ($key, $value) = @_;
4880         $apply_pending_cloudinit = sub {}; # once is enough
4881
4882         my @cloudinit_opts = keys %$confdesc_cloudinit;
4883         foreach my $opt (keys %{$conf->{pending}}) {
4884             next if !grep { $_ eq $opt } @cloudinit_opts;
4885             $conf->{$opt} = delete $conf->{pending}->{$opt};
4886         }
4887
4888         my $new_conf = { %$conf };
4889         $new_conf->{$key} = $value;
4890         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4891     };
4892
4893     foreach my $opt (keys %{$conf->{pending}}) {
4894         next if $selection && !$selection->{$opt};
4895         my $value = $conf->{pending}->{$opt};
4896         eval {
4897             if ($opt eq 'hotplug') {
4898                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4899             } elsif ($opt eq 'tablet') {
4900                 die "skip\n" if !$hotplug_features->{usb};
4901                 if ($value == 1) {
4902                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4903                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4904                         if $arch eq 'aarch64';
4905                 } elsif ($value == 0) {
4906                     vm_deviceunplug($vmid, $conf, 'tablet');
4907                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4908                 }
4909             } elsif ($opt =~ m/^usb\d+$/) {
4910                 die "skip\n";
4911                 # since we cannot reliably hot unplug usb devices
4912                 # we are disabling it
4913                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4914                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4915                 die "skip\n" if !$d;
4916                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4917             } elsif ($opt eq 'vcpus') {
4918                 die "skip\n" if !$hotplug_features->{cpu};
4919                 qemu_cpu_hotplug($vmid, $conf, $value);
4920             } elsif ($opt eq 'balloon') {
4921                 # enable/disable balloning device is not hotpluggable
4922                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4923                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4924                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4925
4926                 # allow manual ballooning if shares is set to zero
4927                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4928                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4929                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4930                 }
4931             } elsif ($opt =~ m/^net(\d+)$/) {
4932                 # some changes can be done without hotplug
4933                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4934                                     $vmid, $opt, $value, $arch, $machine_type);
4935             } elsif (is_valid_drivename($opt)) {
4936                 # some changes can be done without hotplug
4937                 my $drive = parse_drive($opt, $value);
4938                 if (drive_is_cloudinit($drive)) {
4939                     &$apply_pending_cloudinit($opt, $value);
4940                 }
4941                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4942                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4943             } elsif ($opt =~ m/^memory$/) { #dimms
4944                 die "skip\n" if !$hotplug_features->{memory};
4945                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4946             } elsif ($opt eq 'cpuunits') {
4947                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4948             } elsif ($opt eq 'cpulimit') {
4949                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4950                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4951             } else {
4952                 die "skip\n";  # skip non-hot-pluggable options
4953             }
4954         };
4955         if (my $err = $@) {
4956             &$add_error($opt, $err) if $err ne "skip\n";
4957         } else {
4958             # save new config if hotplug was successful
4959             $conf->{$opt} = $value;
4960             delete $conf->{pending}->{$opt};
4961             PVE::QemuConfig->write_config($vmid, $conf);
4962             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4963         }
4964     }
4965 }
4966
4967 sub try_deallocate_drive {
4968     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4969
4970     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4971         my $volid = $drive->{file};
4972         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4973             my $sid = PVE::Storage::parse_volume_id($volid);
4974             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4975
4976             # check if the disk is really unused
4977             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4978                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4979             PVE::Storage::vdisk_free($storecfg, $volid);
4980             return 1;
4981         } else {
4982             # If vm is not owner of this disk remove from config
4983             return 1;
4984         }
4985     }
4986
4987     return undef;
4988 }
4989
4990 sub vmconfig_delete_or_detach_drive {
4991     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4992
4993     my $drive = parse_drive($opt, $conf->{$opt});
4994
4995     my $rpcenv = PVE::RPCEnvironment::get();
4996     my $authuser = $rpcenv->get_user();
4997
4998     if ($force) {
4999         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5000         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5001     } else {
5002         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5003     }
5004 }
5005
5006 sub vmconfig_apply_pending {
5007     my ($vmid, $conf, $storecfg) = @_;
5008
5009     # cold plug
5010
5011     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5012     while (my ($opt, $force) = each %$pending_delete_hash) {
5013         die "internal error" if $opt =~ m/^unused/;
5014         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5015         if (!defined($conf->{$opt})) {
5016             vmconfig_undelete_pending_option($conf, $opt);
5017             PVE::QemuConfig->write_config($vmid, $conf);
5018         } elsif (is_valid_drivename($opt)) {
5019             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5020             vmconfig_undelete_pending_option($conf, $opt);
5021             delete $conf->{$opt};
5022             PVE::QemuConfig->write_config($vmid, $conf);
5023         } else {
5024             vmconfig_undelete_pending_option($conf, $opt);
5025             delete $conf->{$opt};
5026             PVE::QemuConfig->write_config($vmid, $conf);
5027         }
5028     }
5029
5030     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5031
5032     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5033         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5034
5035         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5036             # skip if nothing changed
5037         } elsif (is_valid_drivename($opt)) {
5038             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5039                 if defined($conf->{$opt});
5040             $conf->{$opt} = $conf->{pending}->{$opt};
5041         } else {
5042             $conf->{$opt} = $conf->{pending}->{$opt};
5043         }
5044
5045         delete $conf->{pending}->{$opt};
5046         PVE::QemuConfig->write_config($vmid, $conf);
5047     }
5048 }
5049
5050 my $safe_num_ne = sub {
5051     my ($a, $b) = @_;
5052
5053     return 0 if !defined($a) && !defined($b);
5054     return 1 if !defined($a);
5055     return 1 if !defined($b);
5056
5057     return $a != $b;
5058 };
5059
5060 my $safe_string_ne = sub {
5061     my ($a, $b) = @_;
5062
5063     return 0 if !defined($a) && !defined($b);
5064     return 1 if !defined($a);
5065     return 1 if !defined($b);
5066
5067     return $a ne $b;
5068 };
5069
5070 sub vmconfig_update_net {
5071     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5072
5073     my $newnet = parse_net($value);
5074
5075     if ($conf->{$opt}) {
5076         my $oldnet = parse_net($conf->{$opt});
5077
5078         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5079             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5080             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5081             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5082
5083             # for non online change, we try to hot-unplug
5084             die "skip\n" if !$hotplug;
5085             vm_deviceunplug($vmid, $conf, $opt);
5086         } else {
5087
5088             die "internal error" if $opt !~ m/net(\d+)/;
5089             my $iface = "tap${vmid}i$1";
5090
5091             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5092                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5093                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5094                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5095                 PVE::Network::tap_unplug($iface);
5096                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5097             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5098                 # Rate can be applied on its own but any change above needs to
5099                 # include the rate in tap_plug since OVS resets everything.
5100                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5101             }
5102
5103             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5104                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5105             }
5106
5107             return 1;
5108         }
5109     }
5110
5111     if ($hotplug) {
5112         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5113     } else {
5114         die "skip\n";
5115     }
5116 }
5117
5118 sub vmconfig_update_disk {
5119     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5120
5121     # fixme: do we need force?
5122
5123     my $drive = parse_drive($opt, $value);
5124
5125     if ($conf->{$opt}) {
5126
5127         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5128
5129             my $media = $drive->{media} || 'disk';
5130             my $oldmedia = $old_drive->{media} || 'disk';
5131             die "unable to change media type\n" if $media ne $oldmedia;
5132
5133             if (!drive_is_cdrom($old_drive)) {
5134
5135                 if ($drive->{file} ne $old_drive->{file}) {
5136
5137                     die "skip\n" if !$hotplug;
5138
5139                     # unplug and register as unused
5140                     vm_deviceunplug($vmid, $conf, $opt);
5141                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5142
5143                 } else {
5144                     # update existing disk
5145
5146                     # skip non hotpluggable value
5147                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5148                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5149                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5150                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5151                         die "skip\n";
5152                     }
5153
5154                     # apply throttle
5155                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5156                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5157                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5158                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5159                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5160                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5161                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5162                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5163                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5164                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5165                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5166                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5167                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5168                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5169                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5170                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5171                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5172                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5173
5174                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5175                                                    ($drive->{mbps} || 0)*1024*1024,
5176                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5177                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5178                                                    $drive->{iops} || 0,
5179                                                    $drive->{iops_rd} || 0,
5180                                                    $drive->{iops_wr} || 0,
5181                                                    ($drive->{mbps_max} || 0)*1024*1024,
5182                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5183                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5184                                                    $drive->{iops_max} || 0,
5185                                                    $drive->{iops_rd_max} || 0,
5186                                                    $drive->{iops_wr_max} || 0,
5187                                                    $drive->{bps_max_length} || 1,
5188                                                    $drive->{bps_rd_max_length} || 1,
5189                                                    $drive->{bps_wr_max_length} || 1,
5190                                                    $drive->{iops_max_length} || 1,
5191                                                    $drive->{iops_rd_max_length} || 1,
5192                                                    $drive->{iops_wr_max_length} || 1);
5193
5194                     }
5195
5196                     return 1;
5197                 }
5198
5199             } else { # cdrom
5200
5201                 if ($drive->{file} eq 'none') {
5202                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5203                     if (drive_is_cloudinit($old_drive)) {
5204                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5205                     }
5206                 } else {
5207                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5208                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5209                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5210                 }
5211
5212                 return 1;
5213             }
5214         }
5215     }
5216
5217     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5218     # hotplug new disks
5219     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5220     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5221 }
5222
5223 sub vm_start {
5224     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5225         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5226
5227     PVE::QemuConfig->lock_config($vmid, sub {
5228         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5229
5230         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5231
5232         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5233
5234         PVE::QemuConfig->check_lock($conf)
5235             if !($skiplock || $is_suspended);
5236
5237         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5238
5239         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5240             vmconfig_apply_pending($vmid, $conf, $storecfg);
5241             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5242         }
5243
5244         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5245
5246         my $defaults = load_defaults();
5247
5248         # set environment variable useful inside network script
5249         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5250
5251         my $local_volumes = {};
5252
5253         if ($targetstorage) {
5254             foreach_drive($conf, sub {
5255                 my ($ds, $drive) = @_;
5256
5257                 return if drive_is_cdrom($drive);
5258
5259                 my $volid = $drive->{file};
5260
5261                 return if !$volid;
5262
5263                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5264
5265                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5266                 return if $scfg->{shared};
5267                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5268             });
5269
5270             my $format = undef;
5271
5272             foreach my $opt (sort keys %$local_volumes) {
5273
5274                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5275                 my $drive = parse_drive($opt, $conf->{$opt});
5276
5277                 #if remote storage is specified, use default format
5278                 if ($targetstorage && $targetstorage ne "1") {
5279                     $storeid = $targetstorage;
5280                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5281                     $format = $defFormat;
5282                 } else {
5283                     #else we use same format than original
5284                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5285                     $format = qemu_img_format($scfg, $volid);
5286                 }
5287
5288                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5289                 my $newdrive = $drive;
5290                 $newdrive->{format} = $format;
5291                 $newdrive->{file} = $newvolid;
5292                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5293                 $local_volumes->{$opt} = $drivestr;
5294                 #pass drive to conf for command line
5295                 $conf->{$opt} = $drivestr;
5296             }
5297         }
5298
5299         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5300
5301         if ($is_suspended) {
5302             # enforce machine type on suspended vm to ensure HW compatibility
5303             $forcemachine = $conf->{runningmachine};
5304             print "Resuming suspended VM\n";
5305         }
5306
5307         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5308
5309         my $migrate_port = 0;
5310         my $migrate_uri;
5311         if ($statefile) {
5312             if ($statefile eq 'tcp') {
5313                 my $localip = "localhost";
5314                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5315                 my $nodename = PVE::INotify::nodename();
5316
5317                 if (!defined($migration_type)) {
5318                     if (defined($datacenterconf->{migration}->{type})) {
5319                         $migration_type = $datacenterconf->{migration}->{type};
5320                     } else {
5321                         $migration_type = 'secure';
5322                     }
5323                 }
5324
5325                 if ($migration_type eq 'insecure') {
5326                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5327                     if ($migrate_network_addr) {
5328                         $localip = $migrate_network_addr;
5329                     } else {
5330                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5331                     }
5332
5333                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5334                 }
5335
5336                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5337                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5338                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5339                 push @$cmd, '-incoming', $migrate_uri;
5340                 push @$cmd, '-S';
5341
5342             } elsif ($statefile eq 'unix') {
5343                 # should be default for secure migrations as a ssh TCP forward
5344                 # tunnel is not deterministic reliable ready and fails regurarly
5345                 # to set up in time, so use UNIX socket forwards
5346                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5347                 unlink $socket_addr;
5348
5349                 $migrate_uri = "unix:$socket_addr";
5350
5351                 push @$cmd, '-incoming', $migrate_uri;
5352                 push @$cmd, '-S';
5353
5354             } else {
5355                 push @$cmd, '-loadstate', $statefile;
5356             }
5357         } elsif ($paused) {
5358             push @$cmd, '-S';
5359         }
5360
5361         # host pci devices
5362         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5363           my $d = parse_hostpci($conf->{"hostpci$i"});
5364           next if !$d;
5365           my $pcidevices = $d->{pciid};
5366           foreach my $pcidevice (@$pcidevices) {
5367                 my $pciid = $pcidevice->{id};
5368
5369                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5370                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5371                 die "no pci device info for device '$pciid'\n" if !$info;
5372
5373                 if ($d->{mdev}) {
5374                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5375                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5376                 } else {
5377                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5378                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5379                     die "can't reset pci device '$pciid'\n"
5380                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5381                 }
5382           }
5383         }
5384
5385         PVE::Storage::activate_volumes($storecfg, $vollist);
5386
5387         eval {
5388             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5389                 outfunc => sub {}, errfunc => sub {});
5390         };
5391         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5392         # timeout should be more than enough here...
5393         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5394
5395         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5396                                                   : $defaults->{cpuunits};
5397
5398         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5399         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5400
5401         my %properties = (
5402             Slice => 'qemu.slice',
5403             KillMode => 'none',
5404             CPUShares => $cpuunits
5405         );
5406
5407         if (my $cpulimit = $conf->{cpulimit}) {
5408             $properties{CPUQuota} = int($cpulimit * 100);
5409         }
5410         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5411
5412         my $run_qemu = sub {
5413             PVE::Tools::run_fork sub {
5414                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5415                 run_command($cmd, %run_params);
5416             };
5417         };
5418
5419         if ($conf->{hugepages}) {
5420
5421             my $code = sub {
5422                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5423                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5424
5425                 PVE::QemuServer::Memory::hugepages_mount();
5426                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5427
5428                 eval { $run_qemu->() };
5429                 if (my $err = $@) {
5430                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5431                     die $err;
5432                 }
5433
5434                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5435             };
5436             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5437
5438         } else {
5439             eval { $run_qemu->() };
5440         }
5441
5442         if (my $err = $@) {
5443             # deactivate volumes if start fails
5444             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5445             die "start failed: $err";
5446         }
5447
5448         print "migration listens on $migrate_uri\n" if $migrate_uri;
5449
5450         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5451             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5452             warn $@ if $@;
5453         }
5454
5455         #start nbd server for storage migration
5456         if ($targetstorage) {
5457             my $nodename = PVE::INotify::nodename();
5458             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5459             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5460             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5461             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5462
5463             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5464
5465             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5466
5467             foreach my $opt (sort keys %$local_volumes) {
5468                 my $volid = $local_volumes->{$opt};
5469                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5470                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5471                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5472             }
5473         }
5474
5475         if ($migratedfrom) {
5476             eval {
5477                 set_migration_caps($vmid);
5478             };
5479             warn $@ if $@;
5480
5481             if ($spice_port) {
5482                 print "spice listens on port $spice_port\n";
5483                 if ($spice_ticket) {
5484                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5485                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5486                 }
5487             }
5488
5489         } else {
5490             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5491                 if !$statefile && $conf->{balloon};
5492
5493             foreach my $opt (keys %$conf) {
5494                 next if $opt !~  m/^net\d+$/;
5495                 my $nicconf = parse_net($conf->{$opt});
5496                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5497             }
5498         }
5499
5500         vm_mon_cmd_nocheck($vmid, 'qom-set',
5501                     path => "machine/peripheral/balloon0",
5502                     property => "guest-stats-polling-interval",
5503                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5504
5505         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5506             print "Resumed VM, removing state\n";
5507             delete $conf->@{qw(lock vmstate runningmachine)};
5508             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5509             PVE::Storage::vdisk_free($storecfg, $vmstate);
5510             PVE::QemuConfig->write_config($vmid, $conf);
5511         }
5512
5513         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5514     });
5515 }
5516
5517 sub vm_mon_cmd {
5518     my ($vmid, $execute, %params) = @_;
5519
5520     my $cmd = { execute => $execute, arguments => \%params };
5521     vm_qmp_command($vmid, $cmd);
5522 }
5523
5524 sub vm_mon_cmd_nocheck {
5525     my ($vmid, $execute, %params) = @_;
5526
5527     my $cmd = { execute => $execute, arguments => \%params };
5528     vm_qmp_command($vmid, $cmd, 1);
5529 }
5530
5531 sub vm_qmp_command {
5532     my ($vmid, $cmd, $nocheck) = @_;
5533
5534     my $res;
5535
5536     my $timeout;
5537     if ($cmd->{arguments}) {
5538         $timeout = delete $cmd->{arguments}->{timeout};
5539     }
5540
5541     eval {
5542         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5543         my $sname = qmp_socket($vmid);
5544         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5545             my $qmpclient = PVE::QMPClient->new();
5546
5547             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5548         } else {
5549             die "unable to open monitor socket\n";
5550         }
5551     };
5552     if (my $err = $@) {
5553         syslog("err", "VM $vmid qmp command failed - $err");
5554         die $err;
5555     }
5556
5557     return $res;
5558 }
5559
5560 sub vm_human_monitor_command {
5561     my ($vmid, $cmdline) = @_;
5562
5563     my $res;
5564
5565     my $cmd = {
5566         execute => 'human-monitor-command',
5567         arguments => { 'command-line' => $cmdline},
5568     };
5569
5570     return vm_qmp_command($vmid, $cmd);
5571 }
5572
5573 sub vm_commandline {
5574     my ($storecfg, $vmid, $snapname) = @_;
5575
5576     my $conf = PVE::QemuConfig->load_config($vmid);
5577
5578     if ($snapname) {
5579         my $snapshot = $conf->{snapshots}->{$snapname};
5580         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5581
5582         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5583
5584         $conf = $snapshot;
5585     }
5586
5587     my $defaults = load_defaults();
5588
5589     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5590
5591     return PVE::Tools::cmd2string($cmd);
5592 }
5593
5594 sub vm_reset {
5595     my ($vmid, $skiplock) = @_;
5596
5597     PVE::QemuConfig->lock_config($vmid, sub {
5598
5599         my $conf = PVE::QemuConfig->load_config($vmid);
5600
5601         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5602
5603         vm_mon_cmd($vmid, "system_reset");
5604     });
5605 }
5606
5607 sub get_vm_volumes {
5608     my ($conf) = @_;
5609
5610     my $vollist = [];
5611     foreach_volid($conf, sub {
5612         my ($volid, $attr) = @_;
5613
5614         return if $volid =~ m|^/|;
5615
5616         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5617         return if !$sid;
5618
5619         push @$vollist, $volid;
5620     });
5621
5622     return $vollist;
5623 }
5624
5625 sub vm_stop_cleanup {
5626     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5627
5628     eval {
5629
5630         if (!$keepActive) {
5631             my $vollist = get_vm_volumes($conf);
5632             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5633         }
5634
5635         foreach my $ext (qw(mon qmp pid vnc qga)) {
5636             unlink "/var/run/qemu-server/${vmid}.$ext";
5637         }
5638
5639         if ($conf->{ivshmem}) {
5640             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5641             # just delete it for now, VMs which have this already open do not
5642             # are affected, but new VMs will get a separated one. If this
5643             # becomes an issue we either add some sort of ref-counting or just
5644             # add a "don't delete on stop" flag to the ivshmem format.
5645             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5646         }
5647
5648         foreach my $key (keys %$conf) {
5649             next if $key !~ m/^hostpci(\d+)$/;
5650             my $hostpciindex = $1;
5651             my $d = parse_hostpci($conf->{$key});
5652             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5653
5654             foreach my $pci (@{$d->{pciid}}) {
5655                 my $pciid = $pci->{id};
5656                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5657             }
5658         }
5659
5660         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5661     };
5662     warn $@ if $@; # avoid errors - just warn
5663 }
5664
5665 # Note: use $nockeck to skip tests if VM configuration file exists.
5666 # We need that when migration VMs to other nodes (files already moved)
5667 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5668 sub vm_stop {
5669     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5670
5671     $force = 1 if !defined($force) && !$shutdown;
5672
5673     if ($migratedfrom){
5674         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5675         kill 15, $pid if $pid;
5676         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5677         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5678         return;
5679     }
5680
5681     PVE::QemuConfig->lock_config($vmid, sub {
5682
5683         my $pid = check_running($vmid, $nocheck);
5684         return if !$pid;
5685
5686         my $conf;
5687         if (!$nocheck) {
5688             $conf = PVE::QemuConfig->load_config($vmid);
5689             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5690             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5691                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5692                 $timeout = $opts->{down} if $opts->{down};
5693             }
5694             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5695         }
5696
5697         eval {
5698             if ($shutdown) {
5699                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5700                     vm_qmp_command($vmid, {
5701                         execute => "guest-shutdown",
5702                         arguments => { timeout => $timeout }
5703                     }, $nocheck);
5704                 } else {
5705                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5706                 }
5707             } else {
5708                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5709             }
5710         };
5711         my $err = $@;
5712
5713         if (!$err) {
5714             $timeout = 60 if !defined($timeout);
5715
5716             my $count = 0;
5717             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5718                 $count++;
5719                 sleep 1;
5720             }
5721
5722             if ($count >= $timeout) {
5723                 if ($force) {
5724                     warn "VM still running - terminating now with SIGTERM\n";
5725                     kill 15, $pid;
5726                 } else {
5727                     die "VM quit/powerdown failed - got timeout\n";
5728                 }
5729             } else {
5730                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5731                 return;
5732             }
5733         } else {
5734             if ($force) {
5735                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5736                 kill 15, $pid;
5737             } else {
5738                 die "VM quit/powerdown failed\n";
5739             }
5740         }
5741
5742         # wait again
5743         $timeout = 10;
5744
5745         my $count = 0;
5746         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5747             $count++;
5748             sleep 1;
5749         }
5750
5751         if ($count >= $timeout) {
5752             warn "VM still running - terminating now with SIGKILL\n";
5753             kill 9, $pid;
5754             sleep 1;
5755         }
5756
5757         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5758    });
5759 }
5760
5761 sub vm_suspend {
5762     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5763
5764     my $conf;
5765     my $path;
5766     my $storecfg;
5767     my $vmstate;
5768
5769     PVE::QemuConfig->lock_config($vmid, sub {
5770
5771         $conf = PVE::QemuConfig->load_config($vmid);
5772
5773         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5774         PVE::QemuConfig->check_lock($conf)
5775             if !($skiplock || $is_backing_up);
5776
5777         die "cannot suspend to disk during backup\n"
5778             if $is_backing_up && $includestate;
5779
5780         if ($includestate) {
5781             $conf->{lock} = 'suspending';
5782             my $date = strftime("%Y-%m-%d", localtime(time()));
5783             $storecfg = PVE::Storage::config();
5784             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5785             $path = PVE::Storage::path($storecfg, $vmstate);
5786             PVE::QemuConfig->write_config($vmid, $conf);
5787         } else {
5788             vm_mon_cmd($vmid, "stop");
5789         }
5790     });
5791
5792     if ($includestate) {
5793         # save vm state
5794         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5795
5796         eval {
5797             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5798             for(;;) {
5799                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5800                 if (!$state->{status}) {
5801                     die "savevm not active\n";
5802                 } elsif ($state->{status} eq 'active') {
5803                     sleep(1);
5804                     next;
5805                 } elsif ($state->{status} eq 'completed') {
5806                     print "State saved, quitting\n";
5807                     last;
5808                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5809                     die "query-savevm failed with error '$state->{error}'\n"
5810                 } else {
5811                     die "query-savevm returned status '$state->{status}'\n";
5812                 }
5813             }
5814         };
5815         my $err = $@;
5816
5817         PVE::QemuConfig->lock_config($vmid, sub {
5818             $conf = PVE::QemuConfig->load_config($vmid);
5819             if ($err) {
5820                 # cleanup, but leave suspending lock, to indicate something went wrong
5821                 eval {
5822                     vm_mon_cmd($vmid, "savevm-end");
5823                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5824                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5825                     delete $conf->@{qw(vmstate runningmachine)};
5826                     PVE::QemuConfig->write_config($vmid, $conf);
5827                 };
5828                 warn $@ if $@;
5829                 die $err;
5830             }
5831
5832             die "lock changed unexpectedly\n"
5833                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5834
5835             vm_qmp_command($vmid, { execute => "quit" });
5836             $conf->{lock} = 'suspended';
5837             PVE::QemuConfig->write_config($vmid, $conf);
5838         });
5839     }
5840 }
5841
5842 sub vm_resume {
5843     my ($vmid, $skiplock, $nocheck) = @_;
5844
5845     PVE::QemuConfig->lock_config($vmid, sub {
5846         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5847         my $res = $vm_mon_cmd->($vmid, 'query-status');
5848         my $resume_cmd = 'cont';
5849
5850         if ($res->{status} && $res->{status} eq 'suspended') {
5851             $resume_cmd = 'system_wakeup';
5852         }
5853
5854         if (!$nocheck) {
5855
5856             my $conf = PVE::QemuConfig->load_config($vmid);
5857
5858             PVE::QemuConfig->check_lock($conf)
5859                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5860         }
5861
5862         $vm_mon_cmd->($vmid, $resume_cmd);
5863     });
5864 }
5865
5866 sub vm_sendkey {
5867     my ($vmid, $skiplock, $key) = @_;
5868
5869     PVE::QemuConfig->lock_config($vmid, sub {
5870
5871         my $conf = PVE::QemuConfig->load_config($vmid);
5872
5873         # there is no qmp command, so we use the human monitor command
5874         vm_human_monitor_command($vmid, "sendkey $key");
5875     });
5876 }
5877
5878 sub vm_destroy {
5879     my ($storecfg, $vmid, $skiplock) = @_;
5880
5881     PVE::QemuConfig->lock_config($vmid, sub {
5882
5883         my $conf = PVE::QemuConfig->load_config($vmid);
5884
5885         if (!check_running($vmid)) {
5886             destroy_vm($storecfg, $vmid, undef, $skiplock);
5887         } else {
5888             die "VM $vmid is running - destroy failed\n";
5889         }
5890     });
5891 }
5892
5893 # vzdump restore implementaion
5894
5895 sub tar_archive_read_firstfile {
5896     my $archive = shift;
5897
5898     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5899
5900     # try to detect archive type first
5901     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5902         die "unable to open file '$archive'\n";
5903     my $firstfile = <$fh>;
5904     kill 15, $pid;
5905     close $fh;
5906
5907     die "ERROR: archive contaions no data\n" if !$firstfile;
5908     chomp $firstfile;
5909
5910     return $firstfile;
5911 }
5912
5913 sub tar_restore_cleanup {
5914     my ($storecfg, $statfile) = @_;
5915
5916     print STDERR "starting cleanup\n";
5917
5918     if (my $fd = IO::File->new($statfile, "r")) {
5919         while (defined(my $line = <$fd>)) {
5920             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5921                 my $volid = $2;
5922                 eval {
5923                     if ($volid =~ m|^/|) {
5924                         unlink $volid || die 'unlink failed\n';
5925                     } else {
5926                         PVE::Storage::vdisk_free($storecfg, $volid);
5927                     }
5928                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5929                 };
5930                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5931             } else {
5932                 print STDERR "unable to parse line in statfile - $line";
5933             }
5934         }
5935         $fd->close();
5936     }
5937 }
5938
5939 sub restore_archive {
5940     my ($archive, $vmid, $user, $opts) = @_;
5941
5942     my $format = $opts->{format};
5943     my $comp;
5944
5945     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5946         $format = 'tar' if !$format;
5947         $comp = 'gzip';
5948     } elsif ($archive =~ m/\.tar$/) {
5949         $format = 'tar' if !$format;
5950     } elsif ($archive =~ m/.tar.lzo$/) {
5951         $format = 'tar' if !$format;
5952         $comp = 'lzop';
5953     } elsif ($archive =~ m/\.vma$/) {
5954         $format = 'vma' if !$format;
5955     } elsif ($archive =~ m/\.vma\.gz$/) {
5956         $format = 'vma' if !$format;
5957         $comp = 'gzip';
5958     } elsif ($archive =~ m/\.vma\.lzo$/) {
5959         $format = 'vma' if !$format;
5960         $comp = 'lzop';
5961     } else {
5962         $format = 'vma' if !$format; # default
5963     }
5964
5965     # try to detect archive format
5966     if ($format eq 'tar') {
5967         return restore_tar_archive($archive, $vmid, $user, $opts);
5968     } else {
5969         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5970     }
5971 }
5972
5973 sub restore_update_config_line {
5974     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5975
5976     return if $line =~ m/^\#qmdump\#/;
5977     return if $line =~ m/^\#vzdump\#/;
5978     return if $line =~ m/^lock:/;
5979     return if $line =~ m/^unused\d+:/;
5980     return if $line =~ m/^parent:/;
5981
5982     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5983     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5984         # try to convert old 1.X settings
5985         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5986         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5987             my ($model, $macaddr) = split(/\=/, $devconfig);
5988             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5989             my $net = {
5990                 model => $model,
5991                 bridge => "vmbr$ind",
5992                 macaddr => $macaddr,
5993             };
5994             my $netstr = print_net($net);
5995
5996             print $outfd "net$cookie->{netcount}: $netstr\n";
5997             $cookie->{netcount}++;
5998         }
5999     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6000         my ($id, $netstr) = ($1, $2);
6001         my $net = parse_net($netstr);
6002         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6003         $netstr = print_net($net);
6004         print $outfd "$id: $netstr\n";
6005     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6006         my $virtdev = $1;
6007         my $value = $3;
6008         my $di = parse_drive($virtdev, $value);
6009         if (defined($di->{backup}) && !$di->{backup}) {
6010             print $outfd "#$line";
6011         } elsif ($map->{$virtdev}) {
6012             delete $di->{format}; # format can change on restore
6013             $di->{file} = $map->{$virtdev};
6014             $value = print_drive($vmid, $di);
6015             print $outfd "$virtdev: $value\n";
6016         } else {
6017             print $outfd $line;
6018         }
6019     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6020         my $vmgenid = $1;
6021         if ($vmgenid ne '0') {
6022             # always generate a new vmgenid if there was a valid one setup
6023             $vmgenid = generate_uuid();
6024         }
6025         print $outfd "vmgenid: $vmgenid\n";
6026     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6027         my ($uuid, $uuid_str);
6028         UUID::generate($uuid);
6029         UUID::unparse($uuid, $uuid_str);
6030         my $smbios1 = parse_smbios1($2);
6031         $smbios1->{uuid} = $uuid_str;
6032         print $outfd $1.print_smbios1($smbios1)."\n";
6033     } else {
6034         print $outfd $line;
6035     }
6036 }
6037
6038 sub scan_volids {
6039     my ($cfg, $vmid) = @_;
6040
6041     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6042
6043     my $volid_hash = {};
6044     foreach my $storeid (keys %$info) {
6045         foreach my $item (@{$info->{$storeid}}) {
6046             next if !($item->{volid} && $item->{size});
6047             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6048             $volid_hash->{$item->{volid}} = $item;
6049         }
6050     }
6051
6052     return $volid_hash;
6053 }
6054
6055 sub is_volume_in_use {
6056     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6057
6058     my $path = PVE::Storage::path($storecfg, $volid);
6059
6060     my $scan_config = sub {
6061         my ($cref, $snapname) = @_;
6062
6063         foreach my $key (keys %$cref) {
6064             my $value = $cref->{$key};
6065             if (is_valid_drivename($key)) {
6066                 next if $skip_drive && $key eq $skip_drive;
6067                 my $drive = parse_drive($key, $value);
6068                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6069                 return 1 if $volid eq $drive->{file};
6070                 if ($drive->{file} =~ m!^/!) {
6071                     return 1 if $drive->{file} eq $path;
6072                 } else {
6073                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6074                     next if !$storeid;
6075                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6076                     next if !$scfg;
6077                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6078                 }
6079             }
6080         }
6081
6082         return 0;
6083     };
6084
6085     return 1 if &$scan_config($conf);
6086
6087     undef $skip_drive;
6088
6089     foreach my $snapname (keys %{$conf->{snapshots}}) {
6090         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6091     }
6092
6093     return 0;
6094 }
6095
6096 sub update_disksize {
6097     my ($vmid, $conf, $volid_hash) = @_;
6098
6099     my $changes;
6100     my $prefix = "VM $vmid:";
6101
6102     # used and unused disks
6103     my $referenced = {};
6104
6105     # Note: it is allowed to define multiple storages with same path (alias), so
6106     # we need to check both 'volid' and real 'path' (two different volid can point
6107     # to the same path).
6108
6109     my $referencedpath = {};
6110
6111     # update size info
6112     foreach my $opt (keys %$conf) {
6113         if (is_valid_drivename($opt)) {
6114             my $drive = parse_drive($opt, $conf->{$opt});
6115             my $volid = $drive->{file};
6116             next if !$volid;
6117
6118             $referenced->{$volid} = 1;
6119             if ($volid_hash->{$volid} &&
6120                 (my $path = $volid_hash->{$volid}->{path})) {
6121                 $referencedpath->{$path} = 1;
6122             }
6123
6124             next if drive_is_cdrom($drive);
6125             next if !$volid_hash->{$volid};
6126
6127             $drive->{size} = $volid_hash->{$volid}->{size};
6128             my $new = print_drive($vmid, $drive);
6129             if ($new ne $conf->{$opt}) {
6130                 $changes = 1;
6131                 $conf->{$opt} = $new;
6132                 print "$prefix update disk '$opt' information.\n";
6133             }
6134         }
6135     }
6136
6137     # remove 'unusedX' entry if volume is used
6138     foreach my $opt (keys %$conf) {
6139         next if $opt !~ m/^unused\d+$/;
6140         my $volid = $conf->{$opt};
6141         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6142         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6143             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6144             $changes = 1;
6145             delete $conf->{$opt};
6146         }
6147
6148         $referenced->{$volid} = 1;
6149         $referencedpath->{$path} = 1 if $path;
6150     }
6151
6152     foreach my $volid (sort keys %$volid_hash) {
6153         next if $volid =~ m/vm-$vmid-state-/;
6154         next if $referenced->{$volid};
6155         my $path = $volid_hash->{$volid}->{path};
6156         next if !$path; # just to be sure
6157         next if $referencedpath->{$path};
6158         $changes = 1;
6159         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6160         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6161         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6162     }
6163
6164     return $changes;
6165 }
6166
6167 sub rescan {
6168     my ($vmid, $nolock, $dryrun) = @_;
6169
6170     my $cfg = PVE::Storage::config();
6171
6172     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6173     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6174     foreach my $stor (keys %{$cfg->{ids}}) {
6175         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6176     }
6177
6178     print "rescan volumes...\n";
6179     my $volid_hash = scan_volids($cfg, $vmid);
6180
6181     my $updatefn =  sub {
6182         my ($vmid) = @_;
6183
6184         my $conf = PVE::QemuConfig->load_config($vmid);
6185
6186         PVE::QemuConfig->check_lock($conf);
6187
6188         my $vm_volids = {};
6189         foreach my $volid (keys %$volid_hash) {
6190             my $info = $volid_hash->{$volid};
6191             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6192         }
6193
6194         my $changes = update_disksize($vmid, $conf, $vm_volids);
6195
6196         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6197     };
6198
6199     if (defined($vmid)) {
6200         if ($nolock) {
6201             &$updatefn($vmid);
6202         } else {
6203             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6204         }
6205     } else {
6206         my $vmlist = config_list();
6207         foreach my $vmid (keys %$vmlist) {
6208             if ($nolock) {
6209                 &$updatefn($vmid);
6210             } else {
6211                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6212             }
6213         }
6214     }
6215 }
6216
6217 sub restore_vma_archive {
6218     my ($archive, $vmid, $user, $opts, $comp) = @_;
6219
6220     my $readfrom = $archive;
6221
6222     my $cfg = PVE::Storage::config();
6223     my $commands = [];
6224     my $bwlimit = $opts->{bwlimit};
6225
6226     my $dbg_cmdstring = '';
6227     my $add_pipe = sub {
6228         my ($cmd) = @_;
6229         push @$commands, $cmd;
6230         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6231         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6232         $readfrom = '-';
6233     };
6234
6235     my $input = undef;
6236     if ($archive eq '-') {
6237         $input = '<&STDIN';
6238     } else {
6239         # If we use a backup from a PVE defined storage we also consider that
6240         # storage's rate limit:
6241         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6242         if (defined($volid)) {
6243             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6244             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6245             if ($readlimit) {
6246                 print STDERR "applying read rate limit: $readlimit\n";
6247                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6248                 $add_pipe->($cstream);
6249             }
6250         }
6251     }
6252
6253     if ($comp) {
6254         my $cmd;
6255         if ($comp eq 'gzip') {
6256             $cmd = ['zcat', $readfrom];
6257         } elsif ($comp eq 'lzop') {
6258             $cmd = ['lzop', '-d', '-c', $readfrom];
6259         } else {
6260             die "unknown compression method '$comp'\n";
6261         }
6262         $add_pipe->($cmd);
6263     }
6264
6265     my $tmpdir = "/var/tmp/vzdumptmp$$";
6266     rmtree $tmpdir;
6267
6268     # disable interrupts (always do cleanups)
6269     local $SIG{INT} =
6270         local $SIG{TERM} =
6271         local $SIG{QUIT} =
6272         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6273
6274     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6275     POSIX::mkfifo($mapfifo, 0600);
6276     my $fifofh;
6277
6278     my $openfifo = sub {
6279         open($fifofh, '>', $mapfifo) || die $!;
6280     };
6281
6282     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6283
6284     my $oldtimeout;
6285     my $timeout = 5;
6286
6287     my $devinfo = {};
6288
6289     my $rpcenv = PVE::RPCEnvironment::get();
6290
6291     my $conffile = PVE::QemuConfig->config_file($vmid);
6292     my $tmpfn = "$conffile.$$.tmp";
6293
6294     # Note: $oldconf is undef if VM does not exists
6295     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6296     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6297
6298     my %storage_limits;
6299
6300     my $print_devmap = sub {
6301         my $virtdev_hash = {};
6302
6303         my $cfgfn = "$tmpdir/qemu-server.conf";
6304
6305         # we can read the config - that is already extracted
6306         my $fh = IO::File->new($cfgfn, "r") ||
6307             "unable to read qemu-server.conf - $!\n";
6308
6309         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6310         if (-f $fwcfgfn) {
6311             my $pve_firewall_dir = '/etc/pve/firewall';
6312             mkdir $pve_firewall_dir; # make sure the dir exists
6313             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6314         }
6315
6316         while (defined(my $line = <$fh>)) {
6317             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6318                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6319                 die "archive does not contain data for drive '$virtdev'\n"
6320                     if !$devinfo->{$devname};
6321                 if (defined($opts->{storage})) {
6322                     $storeid = $opts->{storage} || 'local';
6323                 } elsif (!$storeid) {
6324                     $storeid = 'local';
6325                 }
6326                 $format = 'raw' if !$format;
6327                 $devinfo->{$devname}->{devname} = $devname;
6328                 $devinfo->{$devname}->{virtdev} = $virtdev;
6329                 $devinfo->{$devname}->{format} = $format;
6330                 $devinfo->{$devname}->{storeid} = $storeid;
6331
6332                 # check permission on storage
6333                 my $pool = $opts->{pool}; # todo: do we need that?
6334                 if ($user ne 'root@pam') {
6335                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6336                 }
6337
6338                 $storage_limits{$storeid} = $bwlimit;
6339
6340                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6341             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6342                 my $virtdev = $1;
6343                 my $drive = parse_drive($virtdev, $2);
6344                 if (drive_is_cloudinit($drive)) {
6345                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6346                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6347                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6348
6349                     my $d = {
6350                         format => $format,
6351                         storeid => $opts->{storage} // $storeid,
6352                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6353                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6354                         name => "vm-$vmid-cloudinit",
6355                         is_cloudinit => 1,
6356                     };
6357                     $virtdev_hash->{$virtdev} = $d;
6358                 }
6359             }
6360         }
6361
6362         foreach my $key (keys %storage_limits) {
6363             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6364             next if !$limit;
6365             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6366             $storage_limits{$key} = $limit * 1024;
6367         }
6368
6369         foreach my $devname (keys %$devinfo) {
6370             die "found no device mapping information for device '$devname'\n"
6371                 if !$devinfo->{$devname}->{virtdev};
6372         }
6373
6374         # create empty/temp config
6375         if ($oldconf) {
6376             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6377             foreach_drive($oldconf, sub {
6378                 my ($ds, $drive) = @_;
6379
6380                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6381
6382                 my $volid = $drive->{file};
6383                 return if !$volid || $volid =~ m|^/|;
6384
6385                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6386                 return if !$path || !$owner || ($owner != $vmid);
6387
6388                 # Note: only delete disk we want to restore
6389                 # other volumes will become unused
6390                 if ($virtdev_hash->{$ds}) {
6391                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6392                     if (my $err = $@) {
6393                         warn $err;
6394                     }
6395                 }
6396             });
6397
6398             # delete vmstate files, after the restore we have no snapshots anymore
6399             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6400                 my $snap = $oldconf->{snapshots}->{$snapname};
6401                 if ($snap->{vmstate}) {
6402                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6403                     if (my $err = $@) {
6404                         warn $err;
6405                     }
6406                 }
6407             }
6408         }
6409
6410         my $map = {};
6411         foreach my $virtdev (sort keys %$virtdev_hash) {
6412             my $d = $virtdev_hash->{$virtdev};
6413             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6414             my $storeid = $d->{storeid};
6415             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6416
6417             my $map_opts = '';
6418             if (my $limit = $storage_limits{$storeid}) {
6419                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6420             }
6421
6422             # test if requested format is supported
6423             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6424             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6425             $d->{format} = $defFormat if !$supported;
6426
6427             my $name;
6428             if ($d->{is_cloudinit}) {
6429                 $name = $d->{name};
6430                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6431             }
6432
6433             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6434             print STDERR "new volume ID is '$volid'\n";
6435             $d->{volid} = $volid;
6436
6437             PVE::Storage::activate_volumes($cfg, [$volid]);
6438
6439             my $write_zeros = 1;
6440             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6441                 $write_zeros = 0;
6442             }
6443
6444             if (!$d->{is_cloudinit}) {
6445                 my $path = PVE::Storage::path($cfg, $volid);
6446
6447                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6448
6449                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6450             }
6451             $map->{$virtdev} = $volid;
6452         }
6453
6454         $fh->seek(0, 0) || die "seek failed - $!\n";
6455
6456         my $outfd = new IO::File ($tmpfn, "w") ||
6457             die "unable to write config for VM $vmid\n";
6458
6459         my $cookie = { netcount => 0 };
6460         while (defined(my $line = <$fh>)) {
6461             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6462         }
6463
6464         $fh->close();
6465         $outfd->close();
6466     };
6467
6468     eval {
6469         # enable interrupts
6470         local $SIG{INT} =
6471             local $SIG{TERM} =
6472             local $SIG{QUIT} =
6473             local $SIG{HUP} =
6474             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6475         local $SIG{ALRM} = sub { die "got timeout\n"; };
6476
6477         $oldtimeout = alarm($timeout);
6478
6479         my $parser = sub {
6480             my $line = shift;
6481
6482             print "$line\n";
6483
6484             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6485                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6486                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6487             } elsif ($line =~ m/^CTIME: /) {
6488                 # we correctly received the vma config, so we can disable
6489                 # the timeout now for disk allocation (set to 10 minutes, so
6490                 # that we always timeout if something goes wrong)
6491                 alarm(600);
6492                 &$print_devmap();
6493                 print $fifofh "done\n";
6494                 my $tmp = $oldtimeout || 0;
6495                 $oldtimeout = undef;
6496                 alarm($tmp);
6497                 close($fifofh);
6498             }
6499         };
6500
6501         print "restore vma archive: $dbg_cmdstring\n";
6502         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6503     };
6504     my $err = $@;
6505
6506     alarm($oldtimeout) if $oldtimeout;
6507
6508     my $vollist = [];
6509     foreach my $devname (keys %$devinfo) {
6510         my $volid = $devinfo->{$devname}->{volid};
6511         push @$vollist, $volid if $volid;
6512     }
6513
6514     PVE::Storage::deactivate_volumes($cfg, $vollist);
6515
6516     unlink $mapfifo;
6517
6518     if ($err) {
6519         rmtree $tmpdir;
6520         unlink $tmpfn;
6521
6522         foreach my $devname (keys %$devinfo) {
6523             my $volid = $devinfo->{$devname}->{volid};
6524             next if !$volid;
6525             eval {
6526                 if ($volid =~ m|^/|) {
6527                     unlink $volid || die 'unlink failed\n';
6528                 } else {
6529                     PVE::Storage::vdisk_free($cfg, $volid);
6530                 }
6531                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6532             };
6533             print STDERR "unable to cleanup '$volid' - $@" if $@;
6534         }
6535         die $err;
6536     }
6537
6538     rmtree $tmpdir;
6539
6540     rename($tmpfn, $conffile) ||
6541         die "unable to commit configuration file '$conffile'\n";
6542
6543     PVE::Cluster::cfs_update(); # make sure we read new file
6544
6545     eval { rescan($vmid, 1); };
6546     warn $@ if $@;
6547 }
6548
6549 sub restore_tar_archive {
6550     my ($archive, $vmid, $user, $opts) = @_;
6551
6552     if ($archive ne '-') {
6553         my $firstfile = tar_archive_read_firstfile($archive);
6554         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6555             if $firstfile ne 'qemu-server.conf';
6556     }
6557
6558     my $storecfg = PVE::Storage::config();
6559
6560     # destroy existing data - keep empty config
6561     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6562     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6563
6564     my $tocmd = "/usr/lib/qemu-server/qmextract";
6565
6566     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6567     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6568     $tocmd .= ' --prealloc' if $opts->{prealloc};
6569     $tocmd .= ' --info' if $opts->{info};
6570
6571     # tar option "xf" does not autodetect compression when read from STDIN,
6572     # so we pipe to zcat
6573     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6574         PVE::Tools::shellquote("--to-command=$tocmd");
6575
6576     my $tmpdir = "/var/tmp/vzdumptmp$$";
6577     mkpath $tmpdir;
6578
6579     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6580     local $ENV{VZDUMP_VMID} = $vmid;
6581     local $ENV{VZDUMP_USER} = $user;
6582
6583     my $conffile = PVE::QemuConfig->config_file($vmid);
6584     my $tmpfn = "$conffile.$$.tmp";
6585
6586     # disable interrupts (always do cleanups)
6587     local $SIG{INT} =
6588         local $SIG{TERM} =
6589         local $SIG{QUIT} =
6590         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6591
6592     eval {
6593         # enable interrupts
6594         local $SIG{INT} =
6595             local $SIG{TERM} =
6596             local $SIG{QUIT} =
6597             local $SIG{HUP} =
6598             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6599
6600         if ($archive eq '-') {
6601             print "extracting archive from STDIN\n";
6602             run_command($cmd, input => "<&STDIN");
6603         } else {
6604             print "extracting archive '$archive'\n";
6605             run_command($cmd);
6606         }
6607
6608         return if $opts->{info};
6609
6610         # read new mapping
6611         my $map = {};
6612         my $statfile = "$tmpdir/qmrestore.stat";
6613         if (my $fd = IO::File->new($statfile, "r")) {
6614             while (defined (my $line = <$fd>)) {
6615                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6616                     $map->{$1} = $2 if $1;
6617                 } else {
6618                     print STDERR "unable to parse line in statfile - $line\n";
6619                 }
6620             }
6621             $fd->close();
6622         }
6623
6624         my $confsrc = "$tmpdir/qemu-server.conf";
6625
6626         my $srcfd = new IO::File($confsrc, "r") ||
6627             die "unable to open file '$confsrc'\n";
6628
6629         my $outfd = new IO::File ($tmpfn, "w") ||
6630             die "unable to write config for VM $vmid\n";
6631
6632         my $cookie = { netcount => 0 };
6633         while (defined (my $line = <$srcfd>)) {
6634             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6635         }
6636
6637         $srcfd->close();
6638         $outfd->close();
6639     };
6640     my $err = $@;
6641
6642     if ($err) {
6643
6644         unlink $tmpfn;
6645
6646         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6647
6648         die $err;
6649     }
6650
6651     rmtree $tmpdir;
6652
6653     rename $tmpfn, $conffile ||
6654         die "unable to commit configuration file '$conffile'\n";
6655
6656     PVE::Cluster::cfs_update(); # make sure we read new file
6657
6658     eval { rescan($vmid, 1); };
6659     warn $@ if $@;
6660 };
6661
6662 sub foreach_storage_used_by_vm {
6663     my ($conf, $func) = @_;
6664
6665     my $sidhash = {};
6666
6667     foreach_drive($conf, sub {
6668         my ($ds, $drive) = @_;
6669         return if drive_is_cdrom($drive);
6670
6671         my $volid = $drive->{file};
6672
6673         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6674         $sidhash->{$sid} = $sid if $sid;
6675     });
6676
6677     foreach my $sid (sort keys %$sidhash) {
6678         &$func($sid);
6679     }
6680 }
6681
6682 sub do_snapshots_with_qemu {
6683     my ($storecfg, $volid) = @_;
6684
6685     my $storage_name = PVE::Storage::parse_volume_id($volid);
6686     my $scfg = $storecfg->{ids}->{$storage_name};
6687
6688     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6689         return 1;
6690     }
6691
6692     if ($volid =~ m/\.(qcow2|qed)$/){
6693         return 1;
6694     }
6695
6696     return undef;
6697 }
6698
6699 sub qga_check_running {
6700     my ($vmid, $nowarn) = @_;
6701
6702     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6703     if ($@) {
6704         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6705         return 0;
6706     }
6707     return 1;
6708 }
6709
6710 sub template_create {
6711     my ($vmid, $conf, $disk) = @_;
6712
6713     my $storecfg = PVE::Storage::config();
6714
6715     foreach_drive($conf, sub {
6716         my ($ds, $drive) = @_;
6717
6718         return if drive_is_cdrom($drive);
6719         return if $disk && $ds ne $disk;
6720
6721         my $volid = $drive->{file};
6722         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6723
6724         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6725         $drive->{file} = $voliddst;
6726         $conf->{$ds} = print_drive($vmid, $drive);
6727         PVE::QemuConfig->write_config($vmid, $conf);
6728     });
6729 }
6730
6731 sub convert_iscsi_path {
6732     my ($path) = @_;
6733
6734     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6735         my $portal = $1;
6736         my $target = $2;
6737         my $lun = $3;
6738
6739         my $initiator_name = get_initiator_name();
6740
6741         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6742                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6743     }
6744
6745     die "cannot convert iscsi path '$path', unkown format\n";
6746 }
6747
6748 sub qemu_img_convert {
6749     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6750
6751     my $storecfg = PVE::Storage::config();
6752     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6753     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6754
6755     if ($src_storeid && $dst_storeid) {
6756
6757         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6758
6759         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6760         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6761
6762         my $src_format = qemu_img_format($src_scfg, $src_volname);
6763         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6764
6765         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6766         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6767
6768         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6769         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6770
6771         my $cmd = [];
6772         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6773         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6774         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6775         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6776
6777         if ($src_is_iscsi) {
6778             push @$cmd, '--image-opts';
6779             $src_path = convert_iscsi_path($src_path);
6780         } else {
6781             push @$cmd, '-f', $src_format;
6782         }
6783
6784         if ($dst_is_iscsi) {
6785             push @$cmd, '--target-image-opts';
6786             $dst_path = convert_iscsi_path($dst_path);
6787         } else {
6788             push @$cmd, '-O', $dst_format;
6789         }
6790
6791         push @$cmd, $src_path;
6792
6793         if (!$dst_is_iscsi && $is_zero_initialized) {
6794             push @$cmd, "zeroinit:$dst_path";
6795         } else {
6796             push @$cmd, $dst_path;
6797         }
6798
6799         my $parser = sub {
6800             my $line = shift;
6801             if($line =~ m/\((\S+)\/100\%\)/){
6802                 my $percent = $1;
6803                 my $transferred = int($size * $percent / 100);
6804                 my $remaining = $size - $transferred;
6805
6806                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6807             }
6808
6809         };
6810
6811         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6812         my $err = $@;
6813         die "copy failed: $err" if $err;
6814     }
6815 }
6816
6817 sub qemu_img_format {
6818     my ($scfg, $volname) = @_;
6819
6820     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6821         return $1;
6822     } else {
6823         return "raw";
6824     }
6825 }
6826
6827 sub qemu_drive_mirror {
6828     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6829
6830     $jobs = {} if !$jobs;
6831
6832     my $qemu_target;
6833     my $format;
6834     $jobs->{"drive-$drive"} = {};
6835
6836     if ($dst_volid =~ /^nbd:/) {
6837         $qemu_target = $dst_volid;
6838         $format = "nbd";
6839     } else {
6840         my $storecfg = PVE::Storage::config();
6841         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6842
6843         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6844
6845         $format = qemu_img_format($dst_scfg, $dst_volname);
6846
6847         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6848
6849         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6850     }
6851
6852     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6853     $opts->{format} = $format if $format;
6854
6855     if (defined($bwlimit)) {
6856         $opts->{speed} = $bwlimit * 1024;
6857         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6858     } else {
6859         print "drive mirror is starting for drive-$drive\n";
6860     }
6861
6862     # if a job already runs for this device we get an error, catch it for cleanup
6863     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6864     if (my $err = $@) {
6865         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6866         warn "$@\n" if $@;
6867         die "mirroring error: $err\n";
6868     }
6869
6870     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6871 }
6872
6873 sub qemu_drive_mirror_monitor {
6874     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6875
6876     eval {
6877         my $err_complete = 0;
6878
6879         while (1) {
6880             die "storage migration timed out\n" if $err_complete > 300;
6881
6882             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6883
6884             my $running_mirror_jobs = {};
6885             foreach my $stat (@$stats) {
6886                 next if $stat->{type} ne 'mirror';
6887                 $running_mirror_jobs->{$stat->{device}} = $stat;
6888             }
6889
6890             my $readycounter = 0;
6891
6892             foreach my $job (keys %$jobs) {
6893
6894                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6895                     print "$job : finished\n";
6896                     delete $jobs->{$job};
6897                     next;
6898                 }
6899
6900                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6901
6902                 my $busy = $running_mirror_jobs->{$job}->{busy};
6903                 my $ready = $running_mirror_jobs->{$job}->{ready};
6904                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6905                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6906                     my $remaining = $total - $transferred;
6907                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6908
6909                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6910                 }
6911
6912                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6913             }
6914
6915             last if scalar(keys %$jobs) == 0;
6916
6917             if ($readycounter == scalar(keys %$jobs)) {
6918                 print "all mirroring jobs are ready \n";
6919                 last if $skipcomplete; #do the complete later
6920
6921                 if ($vmiddst && $vmiddst != $vmid) {
6922                     my $agent_running = $qga && qga_check_running($vmid);
6923                     if ($agent_running) {
6924                         print "freeze filesystem\n";
6925                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6926                     } else {
6927                         print "suspend vm\n";
6928                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6929                     }
6930
6931                     # if we clone a disk for a new target vm, we don't switch the disk
6932                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6933
6934                     if ($agent_running) {
6935                         print "unfreeze filesystem\n";
6936                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6937                     } else {
6938                         print "resume vm\n";
6939                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6940                     }
6941
6942                     last;
6943                 } else {
6944
6945                     foreach my $job (keys %$jobs) {
6946                         # try to switch the disk if source and destination are on the same guest
6947                         print "$job: Completing block job...\n";
6948
6949                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6950                         if ($@ =~ m/cannot be completed/) {
6951                             print "$job: Block job cannot be completed, try again.\n";
6952                             $err_complete++;
6953                         }else {
6954                             print "$job: Completed successfully.\n";
6955                             $jobs->{$job}->{complete} = 1;
6956                         }
6957                     }
6958                 }
6959             }
6960             sleep 1;
6961         }
6962     };
6963     my $err = $@;
6964
6965     if ($err) {
6966         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6967         die "mirroring error: $err";
6968     }
6969
6970 }
6971
6972 sub qemu_blockjobs_cancel {
6973     my ($vmid, $jobs) = @_;
6974
6975     foreach my $job (keys %$jobs) {
6976         print "$job: Cancelling block job\n";
6977         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6978         $jobs->{$job}->{cancel} = 1;
6979     }
6980
6981     while (1) {
6982         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6983
6984         my $running_jobs = {};
6985         foreach my $stat (@$stats) {
6986             $running_jobs->{$stat->{device}} = $stat;
6987         }
6988
6989         foreach my $job (keys %$jobs) {
6990
6991             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6992                 print "$job: Done.\n";
6993                 delete $jobs->{$job};
6994             }
6995         }
6996
6997         last if scalar(keys %$jobs) == 0;
6998
6999         sleep 1;
7000     }
7001 }
7002
7003 sub clone_disk {
7004     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7005         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7006
7007     my $newvolid;
7008
7009     if (!$full) {
7010         print "create linked clone of drive $drivename ($drive->{file})\n";
7011         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7012         push @$newvollist, $newvolid;
7013     } else {
7014
7015         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7016         $storeid = $storage if $storage;
7017
7018         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7019         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7020
7021         print "create full clone of drive $drivename ($drive->{file})\n";
7022         my $name = undef;
7023         if (drive_is_cloudinit($drive)) {
7024             $name = "vm-$newvmid-cloudinit";
7025             $snapname = undef;
7026             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7027             if ($dst_format ne 'raw') {
7028                 $name .= ".$dst_format";
7029             }
7030         }
7031         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7032         push @$newvollist, $newvolid;
7033
7034         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7035
7036         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7037         if (!$running || $snapname) {
7038             # TODO: handle bwlimits
7039             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7040         } else {
7041
7042             my $kvmver = get_running_qemu_version ($vmid);
7043             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7044                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7045                     if $drive->{iothread};
7046             }
7047
7048             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7049         }
7050     }
7051
7052     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7053
7054     my $disk = $drive;
7055     $disk->{format} = undef;
7056     $disk->{file} = $newvolid;
7057     $disk->{size} = $size;
7058
7059     return $disk;
7060 }
7061
7062 # this only works if VM is running
7063 sub get_current_qemu_machine {
7064     my ($vmid) = @_;
7065
7066     my $cmd = { execute => 'query-machines', arguments => {} };
7067     my $res = vm_qmp_command($vmid, $cmd);
7068
7069     my ($current, $default);
7070     foreach my $e (@$res) {
7071         $default = $e->{name} if $e->{'is-default'};
7072         $current = $e->{name} if $e->{'is-current'};
7073     }
7074
7075     # fallback to the default machine if current is not supported by qemu
7076     return $current || $default || 'pc';
7077 }
7078
7079 sub get_running_qemu_version {
7080     my ($vmid) = @_;
7081     my $cmd = { execute => 'query-version', arguments => {} };
7082     my $res = vm_qmp_command($vmid, $cmd);
7083     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7084 }
7085
7086 sub qemu_machine_feature_enabled {
7087     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7088
7089     my $current_major;
7090     my $current_minor;
7091
7092     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7093
7094         $current_major = $3;
7095         $current_minor = $4;
7096
7097     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7098
7099         $current_major = $1;
7100         $current_minor = $2;
7101     }
7102
7103     return 1 if $current_major > $version_major ||
7104                 ($current_major == $version_major &&
7105                  $current_minor >= $version_minor);
7106 }
7107
7108 sub qemu_machine_pxe {
7109     my ($vmid, $conf, $machine) = @_;
7110
7111     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
7112
7113     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7114         $machine .= '.pxe';
7115     }
7116
7117     return $machine;
7118 }
7119
7120 sub qemu_use_old_bios_files {
7121     my ($machine_type) = @_;
7122
7123     return if !$machine_type;
7124
7125     my $use_old_bios_files = undef;
7126
7127     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7128         $machine_type = $1;
7129         $use_old_bios_files = 1;
7130     } else {
7131         my $kvmver = kvm_user_version();
7132         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7133         # load new efi bios files on migration. So this hack is required to allow
7134         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7135         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7136         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7137     }
7138
7139     return ($use_old_bios_files, $machine_type);
7140 }
7141
7142 sub create_efidisk($$$$$) {
7143     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7144
7145     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7146     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7147
7148     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7149     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7150     PVE::Storage::activate_volumes($storecfg, [$volid]);
7151
7152     my $path = PVE::Storage::path($storecfg, $volid);
7153     eval {
7154         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7155     };
7156     die "Copying EFI vars image failed: $@" if $@;
7157
7158     return ($volid, $vars_size);
7159 }
7160
7161 sub vm_iothreads_list {
7162     my ($vmid) = @_;
7163
7164     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7165
7166     my $iothreads = {};
7167     foreach my $iothread (@$res) {
7168         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7169     }
7170
7171     return $iothreads;
7172 }
7173
7174 sub scsihw_infos {
7175     my ($conf, $drive) = @_;
7176
7177     my $maxdev = 0;
7178
7179     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7180         $maxdev = 7;
7181     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7182         $maxdev = 1;
7183     } else {
7184         $maxdev = 256;
7185     }
7186
7187     my $controller = int($drive->{index} / $maxdev);
7188     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7189
7190     return ($maxdev, $controller, $controller_prefix);
7191 }
7192
7193 sub add_hyperv_enlightenments {
7194     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7195
7196     return if $winversion < 6;
7197     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7198
7199     if ($gpu_passthrough || defined($hv_vendor_id)) {
7200         $hv_vendor_id //= 'proxmox';
7201         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7202     }
7203
7204     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7205         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7206         push @$cpuFlags , 'hv_vapic';
7207         push @$cpuFlags , 'hv_time';
7208     } else {
7209         push @$cpuFlags , 'hv_spinlocks=0xffff';
7210     }
7211
7212     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7213         push @$cpuFlags , 'hv_reset';
7214         push @$cpuFlags , 'hv_vpindex';
7215         push @$cpuFlags , 'hv_runtime';
7216     }
7217
7218     if ($winversion >= 7) {
7219         push @$cpuFlags , 'hv_relaxed';
7220
7221         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7222             push @$cpuFlags , 'hv_synic';
7223             push @$cpuFlags , 'hv_stimer';
7224         }
7225
7226         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7227             push @$cpuFlags , 'hv_tlbflush';
7228             push @$cpuFlags , 'hv_ipi';
7229             # FIXME: AMD does not supports this currently, only add with special flag??
7230             #push @$cpuFlags , 'hv_evmcs';
7231         }
7232     }
7233 }
7234
7235 sub windows_version {
7236     my ($ostype) = @_;
7237
7238     return 0 if !$ostype;
7239
7240     my $winversion = 0;
7241
7242     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7243         $winversion = 5;
7244     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7245         $winversion = 6;
7246     } elsif ($ostype =~ m/^win(\d+)$/) {
7247         $winversion = $1;
7248     }
7249
7250     return $winversion;
7251 }
7252
7253 sub resolve_dst_disk_format {
7254         my ($storecfg, $storeid, $src_volname, $format) = @_;
7255         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7256
7257         if (!$format) {
7258             # if no target format is specified, use the source disk format as hint
7259             if ($src_volname) {
7260                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7261                 $format = qemu_img_format($scfg, $src_volname);
7262             } else {
7263                 return $defFormat;
7264             }
7265         }
7266
7267         # test if requested format is supported - else use default
7268         my $supported = grep { $_ eq $format } @$validFormats;
7269         $format = $defFormat if !$supported;
7270         return $format;
7271 }
7272
7273 sub resolve_first_disk {
7274     my $conf = shift;
7275     my @disks = PVE::QemuServer::valid_drive_names();
7276     my $firstdisk;
7277     foreach my $ds (reverse @disks) {
7278         next if !$conf->{$ds};
7279         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7280         next if PVE::QemuServer::drive_is_cdrom($disk);
7281         $firstdisk = $ds;
7282     }
7283     return $firstdisk;
7284 }
7285
7286 sub generate_uuid {
7287     my ($uuid, $uuid_str);
7288     UUID::generate($uuid);
7289     UUID::unparse($uuid, $uuid_str);
7290     return $uuid_str;
7291 }
7292
7293 sub generate_smbios1_uuid {
7294     return "uuid=".generate_uuid();
7295 }
7296
7297 sub nbd_stop {
7298     my ($vmid) = @_;
7299
7300     vm_mon_cmd($vmid, 'nbd-server-stop');
7301 }
7302
7303 # bash completion helper
7304
7305 sub complete_backup_archives {
7306     my ($cmdname, $pname, $cvalue) = @_;
7307
7308     my $cfg = PVE::Storage::config();
7309
7310     my $storeid;
7311
7312     if ($cvalue =~ m/^([^:]+):/) {
7313         $storeid = $1;
7314     }
7315
7316     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7317
7318     my $res = [];
7319     foreach my $id (keys %$data) {
7320         foreach my $item (@{$data->{$id}}) {
7321             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7322             push @$res, $item->{volid} if defined($item->{volid});
7323         }
7324     }
7325
7326     return $res;
7327 }
7328
7329 my $complete_vmid_full = sub {
7330     my ($running) = @_;
7331
7332     my $idlist = vmstatus();
7333
7334     my $res = [];
7335
7336     foreach my $id (keys %$idlist) {
7337         my $d = $idlist->{$id};
7338         if (defined($running)) {
7339             next if $d->{template};
7340             next if $running && $d->{status} ne 'running';
7341             next if !$running && $d->{status} eq 'running';
7342         }
7343         push @$res, $id;
7344
7345     }
7346     return $res;
7347 };
7348
7349 sub complete_vmid {
7350     return &$complete_vmid_full();
7351 }
7352
7353 sub complete_vmid_stopped {
7354     return &$complete_vmid_full(0);
7355 }
7356
7357 sub complete_vmid_running {
7358     return &$complete_vmid_full(1);
7359 }
7360
7361 sub complete_storage {
7362
7363     my $cfg = PVE::Storage::config();
7364     my $ids = $cfg->{ids};
7365
7366     my $res = [];
7367     foreach my $sid (keys %$ids) {
7368         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7369         next if !$ids->{$sid}->{content}->{images};
7370         push @$res, $sid;
7371     }
7372
7373     return $res;
7374 }
7375
7376 1;