PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. '
 558                      . 'Also note that older cloud-init versions do not support hashed passwords.',
 559     },
 560     searchdomain => {
 561         optional => 1,
 562         type => 'string',
 563         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 564     },
 565     nameserver => {
 566         optional => 1,
 567         type => 'string', format => 'address-list',
 568         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 569     },
 570     sshkeys => {
 571         optional => 1,
 572         type => 'string',
 573         format => 'urlencoded',
 574         description => "cloud-init : Setup public SSH keys (one key per line, " .
 575                         "OpenSSH format).",
 576     },
 577 };
 578
 579 # what about other qemu settings ?
 580 #cpu => 'string',
 581 #machine => 'string',
 582 #fda => 'file',
 583 #fdb => 'file',
 584 #mtdblock => 'file',
 585 #sd => 'file',
 586 #pflash => 'file',
 587 #snapshot => 'bool',
 588 #bootp => 'file',
 589 ##tftp => 'dir',
 590 ##smb => 'dir',
 591 #kernel => 'file',
 592 #append => 'string',
 593 #initrd => 'file',
 594 ##soundhw => 'string',
 595
 596 while (my ($k, $v) = each %$confdesc) {
 597     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 598 }
 599
 600 my $MAX_IDE_DISKS = 4;
 601 my $MAX_SCSI_DISKS = 14;
 602 my $MAX_VIRTIO_DISKS = 16;
 603 my $MAX_SATA_DISKS = 6;
 604 my $MAX_USB_DEVICES = 5;
 605 my $MAX_NETS = 32;
 606 my $MAX_UNUSED_DISKS = 8;
 607 my $MAX_HOSTPCI_DEVICES = 4;
 608 my $MAX_SERIAL_PORTS = 4;
 609 my $MAX_PARALLEL_PORTS = 3;
 610 my $MAX_NUMA = 8;
 611
 612 my $numa_fmt = {
 613     cpus => {
 614         type => "string",
 615         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 616         description => "CPUs accessing this NUMA node.",
 617         format_description => "id[-id];...",
 618     },
 619     memory => {
 620         type => "number",
 621         description => "Amount of memory this NUMA node provides.",
 622         optional => 1,
 623     },
 624     hostnodes => {
 625         type => "string",
 626         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 627         description => "Host NUMA nodes to use.",
 628         format_description => "id[-id];...",
 629         optional => 1,
 630     },
 631     policy => {
 632         type => 'string',
 633         enum => [qw(preferred bind interleave)],
 634         description => "NUMA allocation policy.",
 635         optional => 1,
 636     },
 637 };
 638 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 639 my $numadesc = {
 640     optional => 1,
 641     type => 'string', format => $numa_fmt,
 642     description => "NUMA topology.",
 643 };
 644 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 645
 646 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 647     $confdesc->{"numa$i"} = $numadesc;
 648 }
 649
 650 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 651                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 652                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 653 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 654
 655 my $net_fmt_bridge_descr = <<__EOD__;
 656 Bridge to attach the network device to. The Proxmox VE standard bridge
 657 is called 'vmbr0'.
 658
 659 If you do not specify a bridge, we create a kvm user (NATed) network
 660 device, which provides DHCP and DNS services. The following addresses
 661 are used:
 662
 663  10.0.2.2   Gateway
 664  10.0.2.3   DNS Server
 665  10.0.2.4   SMB Server
 666
 667 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 668 __EOD__
 669
 670 my $net_fmt = {
 671     macaddr => {
 672         type => 'string',
 673         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 674         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 675         format_description => "XX:XX:XX:XX:XX:XX",
 676         optional => 1,
 677     },
 678     model => {
 679         type => 'string',
 680         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 681         enum => $nic_model_list,
 682         default_key => 1,
 683     },
 684     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 685     bridge => {
 686         type => 'string',
 687         description => $net_fmt_bridge_descr,
 688         format_description => 'bridge',
 689         optional => 1,
 690     },
 691     queues => {
 692         type => 'integer',
 693         minimum => 0, maximum => 16,
 694         description => 'Number of packet queues to be used on the device.',
 695         optional => 1,
 696     },
 697     rate => {
 698         type => 'number',
 699         minimum => 0,
 700         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 701         optional => 1,
 702     },
 703     tag => {
 704         type => 'integer',
 705         minimum => 1, maximum => 4094,
 706         description => 'VLAN tag to apply to packets on this interface.',
 707         optional => 1,
 708     },
 709     trunks => {
 710         type => 'string',
 711         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 712         description => 'VLAN trunks to pass through this interface.',
 713         format_description => 'vlanid[;vlanid...]',
 714         optional => 1,
 715     },
 716     firewall => {
 717         type => 'boolean',
 718         description => 'Whether this interface should be protected by the firewall.',
 719         optional => 1,
 720     },
 721     link_down => {
 722         type => 'boolean',
 723         description => 'Whether this interface should be disconnected (like pulling the plug).',
 724         optional => 1,
 725     },
 726 };
 727
 728 my $netdesc = {
 729     optional => 1,
 730     type => 'string', format => $net_fmt,
 731     description => "Specify network devices.",
 732 };
 733
 734 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 735
 736 my $ipconfig_fmt = {
 737     ip => {
 738         type => 'string',
 739         format => 'pve-ipv4-config',
 740         format_description => 'IPv4Format/CIDR',
 741         description => 'IPv4 address in CIDR format.',
 742         optional => 1,
 743         default => 'dhcp',
 744     },
 745     gw => {
 746         type => 'string',
 747         format => 'ipv4',
 748         format_description => 'GatewayIPv4',
 749         description => 'Default gateway for IPv4 traffic.',
 750         optional => 1,
 751         requires => 'ip',
 752     },
 753     ip6 => {
 754         type => 'string',
 755         format => 'pve-ipv6-config',
 756         format_description => 'IPv6Format/CIDR',
 757         description => 'IPv6 address in CIDR format.',
 758         optional => 1,
 759         default => 'dhcp',
 760     },
 761     gw6 => {
 762         type => 'string',
 763         format => 'ipv6',
 764         format_description => 'GatewayIPv6',
 765         description => 'Default gateway for IPv6 traffic.',
 766         optional => 1,
 767         requires => 'ip6',
 768     },
 769 };
 770 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 771 my $ipconfigdesc = {
 772     optional => 1,
 773     type => 'string', format => 'pve-qm-ipconfig',
 774     description => <<'EODESCR',
 775 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 776
 777 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 778
 779 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 780 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 781
 782 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 783 EODESCR
 784 };
 785 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 786
 787 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 788     $confdesc->{"net$i"} = $netdesc;
 789     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 790 }
 791
 792 foreach my $key (keys %$confdesc_cloudinit) {
 793     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 794 }
 795
 796 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 797 sub verify_volume_id_or_qm_path {
 798     my ($volid, $noerr) = @_;
 799
 800     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 801         return $volid;
 802     }
 803
 804     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 805     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 806     if ($@) {
 807         return undef if $noerr;
 808         die $@;
 809     }
 810     return $volid;
 811 }
 812
 813 my $drivename_hash;
 814
 815 my %drivedesc_base = (
 816     volume => { alias => 'file' },
 817     file => {
 818         type => 'string',
 819         format => 'pve-volume-id-or-qm-path',
 820         default_key => 1,
 821         format_description => 'volume',
 822         description => "The drive's backing volume.",
 823     },
 824     media => {
 825         type => 'string',
 826         enum => [qw(cdrom disk)],
 827         description => "The drive's media type.",
 828         default => 'disk',
 829         optional => 1
 830     },
 831     cyls => {
 832         type => 'integer',
 833         description => "Force the drive's physical geometry to have a specific cylinder count.",
 834         optional => 1
 835     },
 836     heads => {
 837         type => 'integer',
 838         description => "Force the drive's physical geometry to have a specific head count.",
 839         optional => 1
 840     },
 841     secs => {
 842         type => 'integer',
 843         description => "Force the drive's physical geometry to have a specific sector count.",
 844         optional => 1
 845     },
 846     trans => {
 847         type => 'string',
 848         enum => [qw(none lba auto)],
 849         description => "Force disk geometry bios translation mode.",
 850         optional => 1,
 851     },
 852     snapshot => {
 853         type => 'boolean',
 854         description => "Controls qemu's snapshot mode feature."
 855             . " If activated, changes made to the disk are temporary and will"
 856             . " be discarded when the VM is shutdown.",
 857         optional => 1,
 858     },
 859     cache => {
 860         type => 'string',
 861         enum => [qw(none writethrough writeback unsafe directsync)],
 862         description => "The drive's cache mode",
 863         optional => 1,
 864     },
 865     format => get_standard_option('pve-qm-image-format'),
 866     size => {
 867         type => 'string',
 868         format => 'disk-size',
 869         format_description => 'DiskSize',
 870         description => "Disk size. This is purely informational and has no effect.",
 871         optional => 1,
 872     },
 873     backup => {
 874         type => 'boolean',
 875         description => "Whether the drive should be included when making backups.",
 876         optional => 1,
 877     },
 878     replicate => {
 879         type => 'boolean',
 880         description => 'Whether the drive should considered for replication jobs.',
 881         optional => 1,
 882         default => 1,
 883     },
 884     rerror => {
 885         type => 'string',
 886         enum => [qw(ignore report stop)],
 887         description => 'Read error action.',
 888         optional => 1,
 889     },
 890     werror => {
 891         type => 'string',
 892         enum => [qw(enospc ignore report stop)],
 893         description => 'Write error action.',
 894         optional => 1,
 895     },
 896     aio => {
 897         type => 'string',
 898         enum => [qw(native threads)],
 899         description => 'AIO type to use.',
 900         optional => 1,
 901     },
 902     discard => {
 903         type => 'string',
 904         enum => [qw(ignore on)],
 905         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 906         optional => 1,
 907     },
 908     detect_zeroes => {
 909         type => 'boolean',
 910         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 911         optional => 1,
 912     },
 913     serial => {
 914         type => 'string',
 915         format => 'urlencoded',
 916         format_description => 'serial',
 917         maxLength => 20*3, # *3 since it's %xx url enoded
 918         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 919         optional => 1,
 920     },
 921     shared => {
 922         type => 'boolean',
 923         description => 'Mark this locally-managed volume as available on all nodes',
 924         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 925         optional => 1,
 926         default => 0,
 927     }
 928 );
 929
 930 my %iothread_fmt = ( iothread => {
 931         type => 'boolean',
 932         description => "Whether to use iothreads for this drive",
 933         optional => 1,
 934 });
 935
 936 my %model_fmt = (
 937     model => {
 938         type => 'string',
 939         format => 'urlencoded',
 940         format_description => 'model',
 941         maxLength => 40*3, # *3 since it's %xx url enoded
 942         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 943         optional => 1,
 944     },
 945 );
 946
 947 my %queues_fmt = (
 948     queues => {
 949         type => 'integer',
 950         description => "Number of queues.",
 951         minimum => 2,
 952         optional => 1
 953     }
 954 );
 955
 956 my %scsiblock_fmt = (
 957     scsiblock => {
 958         type => 'boolean',
 959         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 960         optional => 1,
 961         default => 0,
 962     },
 963 );
 964
 965 my $add_throttle_desc = sub {
 966     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 967     my $d = {
 968         type => $type,
 969         format_description => $unit,
 970         description => "Maximum $what in $longunit.",
 971         optional => 1,
 972     };
 973     $d->{minimum} = $minimum if defined($minimum);
 974     $drivedesc_base{$key} = $d;
 975 };
 976 # throughput: (leaky bucket)
 977 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 979 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 980 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 982 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 983 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 984 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 985 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 986
 987 # pools: (pool of IO before throttling starts taking effect)
 988 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 990 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 991 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 992 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 993 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 994
 995 # burst lengths
 996 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 998 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1001 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1002
1003 # legacy support
1004 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1005 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1006 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1007 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1008
1009 my $ide_fmt = {
1010     %drivedesc_base,
1011     %model_fmt,
1012 };
1013 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1014
1015 my $idedesc = {
1016     optional => 1,
1017     type => 'string', format => $ide_fmt,
1018     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1019 };
1020 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1021
1022 my $scsi_fmt = {
1023     %drivedesc_base,
1024     %iothread_fmt,
1025     %queues_fmt,
1026     %scsiblock_fmt,
1027 };
1028 my $scsidesc = {
1029     optional => 1,
1030     type => 'string', format => $scsi_fmt,
1031     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1032 };
1033 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1034
1035 my $sata_fmt = {
1036     %drivedesc_base,
1037 };
1038 my $satadesc = {
1039     optional => 1,
1040     type => 'string', format => $sata_fmt,
1041     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1044
1045 my $virtio_fmt = {
1046     %drivedesc_base,
1047     %iothread_fmt,
1048 };
1049 my $virtiodesc = {
1050     optional => 1,
1051     type => 'string', format => $virtio_fmt,
1052     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1053 };
1054 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1055
1056 my $alldrive_fmt = {
1057     %drivedesc_base,
1058     %iothread_fmt,
1059     %model_fmt,
1060     %queues_fmt,
1061     %scsiblock_fmt,
1062 };
1063
1064 my $efidisk_fmt = {
1065     volume => { alias => 'file' },
1066     file => {
1067         type => 'string',
1068         format => 'pve-volume-id-or-qm-path',
1069         default_key => 1,
1070         format_description => 'volume',
1071         description => "The drive's backing volume.",
1072     },
1073     format => get_standard_option('pve-qm-image-format'),
1074     size => {
1075         type => 'string',
1076         format => 'disk-size',
1077         format_description => 'DiskSize',
1078         description => "Disk size. This is purely informational and has no effect.",
1079         optional => 1,
1080     },
1081 };
1082
1083 my $efidisk_desc = {
1084     optional => 1,
1085     type => 'string', format => $efidisk_fmt,
1086     description => "Configure a Disk for storing EFI vars",
1087 };
1088
1089 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1090
1091 my $usb_fmt = {
1092     host => {
1093         default_key => 1,
1094         type => 'string', format => 'pve-qm-usb-device',
1095         format_description => 'HOSTUSBDEVICE|spice',
1096         description => <<EODESCR,
1097 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1098
1099  'bus-port(.port)*' (decimal numbers) or
1100  'vendor_id:product_id' (hexadeciaml numbers) or
1101  'spice'
1102
1103 You can use the 'lsusb -t' command to list existing usb devices.
1104
1105 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1106
1107 The value 'spice' can be used to add a usb redirection devices for spice.
1108 EODESCR
1109     },
1110     usb3 => {
1111         optional => 1,
1112         type => 'boolean',
1113         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1114         default => 0,
1115     },
1116 };
1117
1118 my $usbdesc = {
1119     optional => 1,
1120     type => 'string', format => $usb_fmt,
1121     description => "Configure an USB device (n is 0 to 4).",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1124
1125 # NOTE: the match-groups of this regex are used in parse_hostpci
1126 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1127 my $hostpci_fmt = {
1128     host => {
1129         default_key => 1,
1130         type => 'string',
1131         pattern => qr/$PCIRE(;$PCIRE)*/,
1132         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1133         description => <<EODESCR,
1134 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1135 of PCI virtual functions of the host. HOSTPCIID syntax is:
1136
1137 'bus:dev.func' (hexadecimal numbers)
1138
1139 You can us the 'lspci' command to list existing PCI devices.
1140 EODESCR
1141     },
1142     rombar => {
1143         type => 'boolean',
1144         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1145         optional => 1,
1146         default => 1,
1147     },
1148     romfile => {
1149         type => 'string',
1150         pattern => '[^,;]+',
1151         format_description => 'string',
1152         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1153         optional => 1,
1154     },
1155     pcie => {
1156         type => 'boolean',
1157         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1158         optional => 1,
1159         default => 0,
1160     },
1161     'x-vga' => {
1162         type => 'boolean',
1163         description =>  "Enable vfio-vga device support.",
1164         optional => 1,
1165         default => 0,
1166     },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1169
1170 my $hostpcidesc = {
1171         optional => 1,
1172         type => 'string', format => 'pve-qm-hostpci',
1173         description => "Map host PCI devices into guest.",
1174         verbose_description =>  <<EODESCR,
1175 Map host PCI devices into guest.
1176
1177 NOTE: This option allows direct access to host hardware. So it is no longer
1178 possible to migrate such machines - use with special care.
1179
1180 CAUTION: Experimental! User reported problems with this option.
1181 EODESCR
1182 };
1183 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1184
1185 my $serialdesc = {
1186         optional => 1,
1187         type => 'string',
1188         pattern => '(/dev/.+|socket)',
1189         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1190         verbose_description =>  <<EODESCR,
1191 Create a serial device inside the VM (n is 0 to 3), and pass through a
1192 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1193 host side (use 'qm terminal' to open a terminal connection).
1194
1195 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1196
1197 CAUTION: Experimental! User reported problems with this option.
1198 EODESCR
1199 };
1200
1201 my $paralleldesc= {
1202         optional => 1,
1203         type => 'string',
1204         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1205         description =>  "Map host parallel devices (n is 0 to 2).",
1206         verbose_description =>  <<EODESCR,
1207 Map host parallel devices (n is 0 to 2).
1208
1209 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1210
1211 CAUTION: Experimental! User reported problems with this option.
1212 EODESCR
1213 };
1214
1215 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1216     $confdesc->{"parallel$i"} = $paralleldesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1220     $confdesc->{"serial$i"} = $serialdesc;
1221 }
1222
1223 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1224     $confdesc->{"hostpci$i"} = $hostpcidesc;
1225 }
1226
1227 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1228     $drivename_hash->{"ide$i"} = 1;
1229     $confdesc->{"ide$i"} = $idedesc;
1230 }
1231
1232 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1233     $drivename_hash->{"sata$i"} = 1;
1234     $confdesc->{"sata$i"} = $satadesc;
1235 }
1236
1237 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1238     $drivename_hash->{"scsi$i"} = 1;
1239     $confdesc->{"scsi$i"} = $scsidesc ;
1240 }
1241
1242 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1243     $drivename_hash->{"virtio$i"} = 1;
1244     $confdesc->{"virtio$i"} = $virtiodesc;
1245 }
1246
1247 $drivename_hash->{efidisk0} = 1;
1248 $confdesc->{efidisk0} = $efidisk_desc;
1249
1250 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1251     $confdesc->{"usb$i"} = $usbdesc;
1252 }
1253
1254 my $unuseddesc = {
1255     optional => 1,
1256     type => 'string', format => 'pve-volume-id',
1257     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1258 };
1259
1260 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1261     $confdesc->{"unused$i"} = $unuseddesc;
1262 }
1263
1264 my $kvm_api_version = 0;
1265
1266 sub kvm_version {
1267
1268     return $kvm_api_version if $kvm_api_version;
1269
1270     my $fh = IO::File->new("</dev/kvm") ||
1271         return 0;
1272
1273     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1274         $kvm_api_version = $v;
1275     }
1276
1277     $fh->close();
1278
1279     return  $kvm_api_version;
1280 }
1281
1282 my $kvm_user_version;
1283
1284 sub kvm_user_version {
1285
1286     return $kvm_user_version if $kvm_user_version;
1287
1288     $kvm_user_version = 'unknown';
1289
1290     my $code = sub {
1291         my $line = shift;
1292         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1293             $kvm_user_version = $2;
1294         }
1295     };
1296
1297     eval { run_command("kvm -version", outfunc => $code); };
1298     warn $@ if $@;
1299
1300     return $kvm_user_version;
1301
1302 }
1303
1304 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1305
1306 sub valid_drive_names {
1307     # order is important - used to autoselect boot disk
1308     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1309             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1310             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1311             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1312             'efidisk0');
1313 }
1314
1315 sub is_valid_drivename {
1316     my $dev = shift;
1317
1318     return defined($drivename_hash->{$dev});
1319 }
1320
1321 sub option_exists {
1322     my $key = shift;
1323     return defined($confdesc->{$key});
1324 }
1325
1326 sub nic_models {
1327     return $nic_model_list;
1328 }
1329
1330 sub os_list_description {
1331
1332     return {
1333         other => 'Other',
1334         wxp => 'Windows XP',
1335         w2k => 'Windows 2000',
1336         w2k3 =>, 'Windows 2003',
1337         w2k8 => 'Windows 2008',
1338         wvista => 'Windows Vista',
1339         win7 => 'Windows 7',
1340         win8 => 'Windows 8/2012',
1341         win10 => 'Windows 10/2016',
1342         l24 => 'Linux 2.4',
1343         l26 => 'Linux 2.6',
1344     };
1345 }
1346
1347 my $cdrom_path;
1348
1349 sub get_cdrom_path {
1350
1351     return  $cdrom_path if $cdrom_path;
1352
1353     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1354     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1355     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1356 }
1357
1358 sub get_iso_path {
1359     my ($storecfg, $vmid, $cdrom) = @_;
1360
1361     if ($cdrom eq 'cdrom') {
1362         return get_cdrom_path();
1363     } elsif ($cdrom eq 'none') {
1364         return '';
1365     } elsif ($cdrom =~ m|^/|) {
1366         return $cdrom;
1367     } else {
1368         return PVE::Storage::path($storecfg, $cdrom);
1369     }
1370 }
1371
1372 # try to convert old style file names to volume IDs
1373 sub filename_to_volume_id {
1374     my ($vmid, $file, $media) = @_;
1375
1376      if (!($file eq 'none' || $file eq 'cdrom' ||
1377           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1378
1379         return undef if $file =~ m|/|;
1380
1381         if ($media && $media eq 'cdrom') {
1382             $file = "local:iso/$file";
1383         } else {
1384             $file = "local:$vmid/$file";
1385         }
1386     }
1387
1388     return $file;
1389 }
1390
1391 sub verify_media_type {
1392     my ($opt, $vtype, $media) = @_;
1393
1394     return if !$media;
1395
1396     my $etype;
1397     if ($media eq 'disk') {
1398         $etype = 'images';
1399     } elsif ($media eq 'cdrom') {
1400         $etype = 'iso';
1401     } else {
1402         die "internal error";
1403     }
1404
1405     return if ($vtype eq $etype);
1406
1407     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1408 }
1409
1410 sub cleanup_drive_path {
1411     my ($opt, $storecfg, $drive) = @_;
1412
1413     # try to convert filesystem paths to volume IDs
1414
1415     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1416         ($drive->{file} !~ m|^/dev/.+|) &&
1417         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1418         ($drive->{file} !~ m/^\d+$/)) {
1419         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1420         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1421         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1422         verify_media_type($opt, $vtype, $drive->{media});
1423         $drive->{file} = $volid;
1424     }
1425
1426     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1427 }
1428
1429 sub parse_hotplug_features {
1430     my ($data) = @_;
1431
1432     my $res = {};
1433
1434     return $res if $data eq '0';
1435
1436     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1437
1438     foreach my $feature (PVE::Tools::split_list($data)) {
1439         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1440             $res->{$1} = 1;
1441         } else {
1442             die "invalid hotplug feature '$feature'\n";
1443         }
1444     }
1445     return $res;
1446 }
1447
1448 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1449 sub pve_verify_hotplug_features {
1450     my ($value, $noerr) = @_;
1451
1452     return $value if parse_hotplug_features($value);
1453
1454     return undef if $noerr;
1455
1456     die "unable to parse hotplug option\n";
1457 }
1458
1459 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1460 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1461 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1462 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1463 #        [,iothread=on][,serial=serial][,model=model]
1464
1465 sub parse_drive {
1466     my ($key, $data) = @_;
1467
1468     my ($interface, $index);
1469
1470     if ($key =~ m/^([^\d]+)(\d+)$/) {
1471         $interface = $1;
1472         $index = $2;
1473     } else {
1474         return undef;
1475     }
1476
1477     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1478                                      : $confdesc->{$key}->{format};
1479     if (!$desc) {
1480         warn "invalid drive key: $key\n";
1481         return undef;
1482     }
1483     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1484     return undef if !$res;
1485     $res->{interface} = $interface;
1486     $res->{index} = $index;
1487
1488     my $error = 0;
1489     foreach my $opt (qw(bps bps_rd bps_wr)) {
1490         if (my $bps = defined(delete $res->{$opt})) {
1491             if (defined($res->{"m$opt"})) {
1492                 warn "both $opt and m$opt specified\n";
1493                 ++$error;
1494                 next;
1495             }
1496             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1497         }
1498     }
1499
1500     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1501     for my $requirement (
1502         [mbps_max => 'mbps'],
1503         [mbps_rd_max => 'mbps_rd'],
1504         [mbps_wr_max => 'mbps_wr'],
1505         [miops_max => 'miops'],
1506         [miops_rd_max => 'miops_rd'],
1507         [miops_wr_max => 'miops_wr'],
1508         [bps_max_length => 'mbps_max'],
1509         [bps_rd_max_length => 'mbps_rd_max'],
1510         [bps_wr_max_length => 'mbps_wr_max'],
1511         [iops_max_length => 'iops_max'],
1512         [iops_rd_max_length => 'iops_rd_max'],
1513         [iops_wr_max_length => 'iops_wr_max']) {
1514         my ($option, $requires) = @$requirement;
1515         if ($res->{$option} && !$res->{$requires}) {
1516             warn "$option requires $requires\n";
1517             ++$error;
1518         }
1519     }
1520
1521     return undef if $error;
1522
1523     return undef if $res->{mbps_rd} && $res->{mbps};
1524     return undef if $res->{mbps_wr} && $res->{mbps};
1525     return undef if $res->{iops_rd} && $res->{iops};
1526     return undef if $res->{iops_wr} && $res->{iops};
1527
1528     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1529         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1530         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1531         return undef if $res->{interface} eq 'virtio';
1532     }
1533
1534     if (my $size = $res->{size}) {
1535         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1536     }
1537
1538     return $res;
1539 }
1540
1541 sub print_drive {
1542     my ($vmid, $drive) = @_;
1543     my $data = { %$drive };
1544     delete $data->{$_} for qw(index interface);
1545     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1546 }
1547
1548 sub scsi_inquiry {
1549     my($fh, $noerr) = @_;
1550
1551     my $SG_IO = 0x2285;
1552     my $SG_GET_VERSION_NUM = 0x2282;
1553
1554     my $versionbuf = "\x00" x 8;
1555     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1556     if (!$ret) {
1557         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1558         return undef;
1559     }
1560     my $version = unpack("I", $versionbuf);
1561     if ($version < 30000) {
1562         die "scsi generic interface too old\n"  if !$noerr;
1563         return undef;
1564     }
1565
1566     my $buf = "\x00" x 36;
1567     my $sensebuf = "\x00" x 8;
1568     my $cmd = pack("C x3 C x1", 0x12, 36);
1569
1570     # see /usr/include/scsi/sg.h
1571     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1572
1573     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1574                       length($sensebuf), 0, length($buf), $buf,
1575                       $cmd, $sensebuf, 6000);
1576
1577     $ret = ioctl($fh, $SG_IO, $packet);
1578     if (!$ret) {
1579         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1580         return undef;
1581     }
1582
1583     my @res = unpack($sg_io_hdr_t, $packet);
1584     if ($res[17] || $res[18]) {
1585         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1586         return undef;
1587     }
1588
1589     my $res = {};
1590     (my $byte0, my $byte1, $res->{vendor},
1591      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1592
1593     $res->{removable} = $byte1 & 128 ? 1 : 0;
1594     $res->{type} = $byte0 & 31;
1595
1596     return $res;
1597 }
1598
1599 sub path_is_scsi {
1600     my ($path) = @_;
1601
1602     my $fh = IO::File->new("+<$path") || return undef;
1603     my $res = scsi_inquiry($fh, 1);
1604     close($fh);
1605
1606     return $res;
1607 }
1608
1609 sub machine_type_is_q35 {
1610     my ($conf) = @_;
1611
1612     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1613 }
1614
1615 sub print_tabletdevice_full {
1616     my ($conf) = @_;
1617
1618     my $q35 = machine_type_is_q35($conf);
1619
1620     # we use uhci for old VMs because tablet driver was buggy in older qemu
1621     my $usbbus = $q35 ? "ehci" : "uhci";
1622
1623     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1624 }
1625
1626 sub print_drivedevice_full {
1627     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1628
1629     my $device = '';
1630     my $maxdev = 0;
1631
1632     if ($drive->{interface} eq 'virtio') {
1633         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1634         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1635         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1636     } elsif ($drive->{interface} eq 'scsi') {
1637
1638         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1639         my $unit = $drive->{index} % $maxdev;
1640         my $devicetype = 'hd';
1641         my $path = '';
1642         if (drive_is_cdrom($drive)) {
1643             $devicetype = 'cd';
1644         } else {
1645             if ($drive->{file} =~ m|^/|) {
1646                 $path = $drive->{file};
1647                 if (my $info = path_is_scsi($path)) {
1648                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1649                         $devicetype = 'block';
1650                     } elsif ($info->{type} == 1) { # tape
1651                         $devicetype = 'generic';
1652                     }
1653                 }
1654             } else {
1655                  $path = PVE::Storage::path($storecfg, $drive->{file});
1656             }
1657
1658             if($path =~ m/^iscsi\:\/\//){
1659                 $devicetype = 'generic';
1660             }
1661         }
1662
1663         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         } else {
1666             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1667         }
1668
1669     } elsif ($drive->{interface} eq 'ide'){
1670         $maxdev = 2;
1671         my $controller = int($drive->{index} / $maxdev);
1672         my $unit = $drive->{index} % $maxdev;
1673         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1674
1675         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1676         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1677             $model = URI::Escape::uri_unescape($model);
1678             $device .= ",model=$model";
1679         }
1680     } elsif ($drive->{interface} eq 'sata'){
1681         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1682         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1683         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1684     } elsif ($drive->{interface} eq 'usb') {
1685         die "implement me";
1686         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1687     } else {
1688         die "unsupported interface type";
1689     }
1690
1691     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1692
1693     return $device;
1694 }
1695
1696 sub get_initiator_name {
1697     my $initiator;
1698
1699     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1700     while (defined(my $line = <$fh>)) {
1701         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1702         $initiator = $1;
1703         last;
1704     }
1705     $fh->close();
1706
1707     return $initiator;
1708 }
1709
1710 sub print_drive_full {
1711     my ($storecfg, $vmid, $drive) = @_;
1712
1713     my $path;
1714     my $volid = $drive->{file};
1715     my $format;
1716
1717     if (drive_is_cdrom($drive)) {
1718         $path = get_iso_path($storecfg, $vmid, $volid);
1719     } else {
1720         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1721         if ($storeid) {
1722             $path = PVE::Storage::path($storecfg, $volid);
1723             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1724             $format = qemu_img_format($scfg, $volname);
1725         } else {
1726             $path = $volid;
1727             $format = "raw";
1728         }
1729    }
1730
1731     my $opts = '';
1732     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1733     foreach my $o (@qemu_drive_options) {
1734         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1735     }
1736
1737     # snapshot only accepts on|off
1738     if (defined($drive->{snapshot})) {
1739         my $v = $drive->{snapshot} ? 'on' : 'off';
1740         $opts .= ",snapshot=$v";
1741     }
1742
1743     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1744         my ($dir, $qmpname) = @$type;
1745         if (my $v = $drive->{"mbps$dir"}) {
1746             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1747         }
1748         if (my $v = $drive->{"mbps${dir}_max"}) {
1749             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1750         }
1751         if (my $v = $drive->{"bps${dir}_max_length"}) {
1752             $opts .= ",throttling.bps$qmpname-max-length=$v";
1753         }
1754         if (my $v = $drive->{"iops${dir}"}) {
1755             $opts .= ",throttling.iops$qmpname=$v";
1756         }
1757         if (my $v = $drive->{"iops${dir}_max"}) {
1758             $opts .= ",throttling.iops$qmpname-max=$v";
1759         }
1760         if (my $v = $drive->{"iops${dir}_max_length"}) {
1761             $opts .= ",throttling.iops$qmpname-max-length=$v";
1762         }
1763     }
1764
1765     if (my $serial = $drive->{serial}) {
1766         $serial = URI::Escape::uri_unescape($serial);
1767         $opts .= ",serial=$serial";
1768     }
1769
1770     $opts .= ",format=$format" if $format && !$drive->{format};
1771
1772     my $cache_direct = 0;
1773
1774     if (my $cache = $drive->{cache}) {
1775         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         $opts .= ",cache=none";
1778         $cache_direct = 1;
1779     }
1780
1781     # aio native works only with O_DIRECT
1782     if (!$drive->{aio}) {
1783         if($cache_direct) {
1784             $opts .= ",aio=native";
1785         } else {
1786             $opts .= ",aio=threads";
1787         }
1788     }
1789
1790     if (!drive_is_cdrom($drive)) {
1791         my $detectzeroes;
1792         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1793             $detectzeroes = 'off';
1794         } elsif ($drive->{discard}) {
1795             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1796         } else {
1797             # This used to be our default with discard not being specified:
1798             $detectzeroes = 'on';
1799         }
1800         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1801     }
1802
1803     my $pathinfo = $path ? "file=$path," : '';
1804
1805     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1806 }
1807
1808 sub print_netdevice_full {
1809     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1810
1811     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1812
1813     my $device = $net->{model};
1814     if ($net->{model} eq 'virtio') {
1815          $device = 'virtio-net-pci';
1816      };
1817
1818     my $pciaddr = print_pci_addr("$netid", $bridges);
1819     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1820     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1821         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1822         my $vectors = $net->{queues} * 2 + 2;
1823         $tmpstr .= ",vectors=$vectors,mq=on";
1824     }
1825     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1826
1827     if ($use_old_bios_files) {
1828         my $romfile;
1829         if ($device eq 'virtio-net-pci') {
1830             $romfile = 'pxe-virtio.rom';
1831         } elsif ($device eq 'e1000') {
1832             $romfile = 'pxe-e1000.rom';
1833         } elsif ($device eq 'ne2k') {
1834             $romfile = 'pxe-ne2k_pci.rom';
1835         } elsif ($device eq 'pcnet') {
1836             $romfile = 'pxe-pcnet.rom';
1837         } elsif ($device eq 'rtl8139') {
1838             $romfile = 'pxe-rtl8139.rom';
1839         }
1840         $tmpstr .= ",romfile=$romfile" if $romfile;
1841     }
1842
1843     return $tmpstr;
1844 }
1845
1846 sub print_netdev_full {
1847     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1848
1849     my $i = '';
1850     if ($netid =~ m/^net(\d+)$/) {
1851         $i = int($1);
1852     }
1853
1854     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1855
1856     my $ifname = "tap${vmid}i$i";
1857
1858     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1859     die "interface name '$ifname' is too long (max 15 character)\n"
1860         if length($ifname) >= 16;
1861
1862     my $vhostparam = '';
1863     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1864
1865     my $vmname = $conf->{name} || "vm$vmid";
1866
1867     my $netdev = "";
1868     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1869
1870     if ($net->{bridge}) {
1871         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881
1882 sub print_cpu_device {
1883     my ($conf, $id) = @_;
1884
1885     my $kvm = $conf->{kvm} // 1;
1886     my $cpu = $kvm ? "kvm64" : "qemu64";
1887     if (my $cputype = $conf->{cpu}) {
1888         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1889             or die "Cannot parse cpu description: $cputype\n";
1890         $cpu = $cpuconf->{cputype};
1891     }
1892
1893     my $cores = $conf->{cores} || 1;
1894
1895     my $current_core = ($id - 1) % $cores;
1896     my $current_socket = int(($id - 1 - $current_core)/$cores);
1897
1898     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1899 }
1900
1901 sub drive_is_cloudinit {
1902     my ($drive) = @_;
1903     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1904 }
1905
1906 sub drive_is_cdrom {
1907     my ($drive, $exclude_cloudinit) = @_;
1908
1909     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1910
1911     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1912
1913 }
1914
1915 sub parse_number_sets {
1916     my ($set) = @_;
1917     my $res = [];
1918     foreach my $part (split(/;/, $set)) {
1919         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1920             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1921             push @$res, [ $1, $2 ];
1922         } else {
1923             die "invalid range: $part\n";
1924         }
1925     }
1926     return $res;
1927 }
1928
1929 sub parse_numa {
1930     my ($data) = @_;
1931
1932     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1933     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1934     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1935     return $res;
1936 }
1937
1938 sub parse_hostpci {
1939     my ($value) = @_;
1940
1941     return undef if !$value;
1942
1943     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1944
1945     my @idlist = split(/;/, $res->{host});
1946     delete $res->{host};
1947     foreach my $id (@idlist) {
1948         if ($id =~ /^$PCIRE$/) {
1949             if (defined($2)) {
1950                 push @{$res->{pciid}}, { id => $1, function => $2 };
1951             } else {
1952                 my $pcidevices = lspci($1);
1953                 $res->{pciid} = $pcidevices->{$1};
1954             }
1955         } else {
1956             # should have been caught by parse_property_string already
1957             die "failed to parse PCI id: $id\n";
1958         }
1959     }
1960     return $res;
1961 }
1962
1963 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1964 sub parse_net {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1968     if ($@) {
1969         warn $@;
1970         return undef;
1971     }
1972     if (!defined($res->{macaddr})) {
1973         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1974         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1975     }
1976     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1977     return $res;
1978 }
1979
1980 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1981 sub parse_ipconfig {
1982     my ($data) = @_;
1983
1984     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return undef;
1988     }
1989
1990     if ($res->{gw} && !$res->{ip}) {
1991         warn 'gateway specified without specifying an IP address';
1992         return undef;
1993     }
1994     if ($res->{gw6} && !$res->{ip6}) {
1995         warn 'IPv6 gateway specified without specifying an IPv6 address';
1996         return undef;
1997     }
1998     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1999         warn 'gateway specified together with DHCP';
2000         return undef;
2001     }
2002     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2003         # gw6 + auto/dhcp
2004         warn "IPv6 gateway specified together with $res->{ip6} address";
2005         return undef;
2006     }
2007
2008     if (!$res->{ip} && !$res->{ip6}) {
2009         return { ip => 'dhcp', ip6 => 'dhcp' };
2010     }
2011
2012     return $res;
2013 }
2014
2015 sub print_net {
2016     my $net = shift;
2017
2018     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2019 }
2020
2021 sub add_random_macs {
2022     my ($settings) = @_;
2023
2024     foreach my $opt (keys %$settings) {
2025         next if $opt !~ m/^net(\d+)$/;
2026         my $net = parse_net($settings->{$opt});
2027         next if !$net;
2028         $settings->{$opt} = print_net($net);
2029     }
2030 }
2031
2032 sub vm_is_volid_owner {
2033     my ($storecfg, $vmid, $volid) = @_;
2034
2035     if ($volid !~  m|^/|) {
2036         my ($path, $owner);
2037         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2038         if ($owner && ($owner == $vmid)) {
2039             return 1;
2040         }
2041     }
2042
2043     return undef;
2044 }
2045
2046 sub split_flagged_list {
2047     my $text = shift || '';
2048     $text =~ s/[,;]/ /g;
2049     $text =~ s/^\s+//;
2050     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2051 }
2052
2053 sub join_flagged_list {
2054     my ($how, $lst) = @_;
2055     join $how, map { $lst->{$_} . $_ } keys %$lst;
2056 }
2057
2058 sub vmconfig_delete_pending_option {
2059     my ($conf, $key, $force) = @_;
2060
2061     delete $conf->{pending}->{$key};
2062     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2063     $pending_delete_hash->{$key} = $force ? '!' : '';
2064     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2065 }
2066
2067 sub vmconfig_undelete_pending_option {
2068     my ($conf, $key) = @_;
2069
2070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2071     delete $pending_delete_hash->{$key};
2072
2073     if (%$pending_delete_hash) {
2074         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2075     } else {
2076         delete $conf->{pending}->{delete};
2077     }
2078 }
2079
2080 sub vmconfig_register_unused_drive {
2081     my ($storecfg, $vmid, $conf, $drive) = @_;
2082
2083     if (drive_is_cloudinit($drive)) {
2084         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2085         warn $@ if $@;
2086     } elsif (!drive_is_cdrom($drive)) {
2087         my $volid = $drive->{file};
2088         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2089             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2090         }
2091     }
2092 }
2093
2094 sub vmconfig_cleanup_pending {
2095     my ($conf) = @_;
2096
2097     # remove pending changes when nothing changed
2098     my $changes;
2099     foreach my $opt (keys %{$conf->{pending}}) {
2100         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2101             $changes = 1;
2102             delete $conf->{pending}->{$opt};
2103         }
2104     }
2105
2106     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2107     my $pending_delete_hash = {};
2108     while (my ($opt, $force) = each %$current_delete_hash) {
2109         if (defined($conf->{$opt})) {
2110             $pending_delete_hash->{$opt} = $force;
2111         } else {
2112             $changes = 1;
2113         }
2114     }
2115
2116     if (%$pending_delete_hash) {
2117         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2118     } else {
2119         delete $conf->{pending}->{delete};
2120     }
2121
2122     return $changes;
2123 }
2124
2125 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2126 my $smbios1_fmt = {
2127     uuid => {
2128         type => 'string',
2129         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2130         format_description => 'UUID',
2131         description => "Set SMBIOS1 UUID.",
2132         optional => 1,
2133     },
2134     version => {
2135         type => 'string',
2136         pattern => '\S+',
2137         format_description => 'string',
2138         description => "Set SMBIOS1 version.",
2139         optional => 1,
2140     },
2141     serial => {
2142         type => 'string',
2143         pattern => '\S+',
2144         format_description => 'string',
2145         description => "Set SMBIOS1 serial number.",
2146         optional => 1,
2147     },
2148     manufacturer => {
2149         type => 'string',
2150         pattern => '\S+',
2151         format_description => 'string',
2152         description => "Set SMBIOS1 manufacturer.",
2153         optional => 1,
2154     },
2155     product => {
2156         type => 'string',
2157         pattern => '\S+',
2158         format_description => 'string',
2159         description => "Set SMBIOS1 product ID.",
2160         optional => 1,
2161     },
2162     sku => {
2163         type => 'string',
2164         pattern => '\S+',
2165         format_description => 'string',
2166         description => "Set SMBIOS1 SKU string.",
2167         optional => 1,
2168     },
2169     family => {
2170         type => 'string',
2171         pattern => '\S+',
2172         format_description => 'string',
2173         description => "Set SMBIOS1 family string.",
2174         optional => 1,
2175     },
2176 };
2177
2178 sub parse_smbios1 {
2179     my ($data) = @_;
2180
2181     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub print_smbios1 {
2187     my ($smbios1) = @_;
2188     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2189 }
2190
2191 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2192
2193 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2194 sub verify_bootdisk {
2195     my ($value, $noerr) = @_;
2196
2197     return $value if is_valid_drivename($value);
2198
2199     return undef if $noerr;
2200
2201     die "invalid boot disk '$value'\n";
2202 }
2203
2204 sub parse_watchdog {
2205     my ($value) = @_;
2206
2207     return undef if !$value;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2210     warn $@ if $@;
2211     return $res;
2212 }
2213
2214 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2215 sub verify_usb_device {
2216     my ($value, $noerr) = @_;
2217
2218     return $value if parse_usb_device($value);
2219
2220     return undef if $noerr;
2221
2222     die "unable to parse usb device\n";
2223 }
2224
2225 # add JSON properties for create and set function
2226 sub json_config_properties {
2227     my $prop = shift;
2228
2229     foreach my $opt (keys %$confdesc) {
2230         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2231         $prop->{$opt} = $confdesc->{$opt};
2232     }
2233
2234     return $prop;
2235 }
2236
2237 # return copy of $confdesc_cloudinit to generate documentation
2238 sub cloudinit_config_properties {
2239
2240     return dclone($confdesc_cloudinit);
2241 }
2242
2243 sub check_type {
2244     my ($key, $value) = @_;
2245
2246     die "unknown setting '$key'\n" if !$confdesc->{$key};
2247
2248     my $type = $confdesc->{$key}->{type};
2249
2250     if (!defined($value)) {
2251         die "got undefined value\n";
2252     }
2253
2254     if ($value =~ m/[\n\r]/) {
2255         die "property contains a line feed\n";
2256     }
2257
2258     if ($type eq 'boolean') {
2259         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2260         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2261         die "type check ('boolean') failed - got '$value'\n";
2262     } elsif ($type eq 'integer') {
2263         return int($1) if $value =~ m/^(\d+)$/;
2264         die "type check ('integer') failed - got '$value'\n";
2265     } elsif ($type eq 'number') {
2266         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2267         die "type check ('number') failed - got '$value'\n";
2268     } elsif ($type eq 'string') {
2269         if (my $fmt = $confdesc->{$key}->{format}) {
2270             PVE::JSONSchema::check_format($fmt, $value);
2271             return $value;
2272         }
2273         $value =~ s/^\"(.*)\"$/$1/;
2274         return $value;
2275     } else {
2276         die "internal error"
2277     }
2278 }
2279
2280 sub check_iommu_support{
2281     #fixme : need to check IOMMU support
2282     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2283
2284     my $iommu=1;
2285     return $iommu;
2286
2287 }
2288
2289 sub touch_config {
2290     my ($vmid) = @_;
2291
2292     my $conf = PVE::QemuConfig->config_file($vmid);
2293     utime undef, undef, $conf;
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2298
2299     my $conffile = PVE::QemuConfig->config_file($vmid);
2300
2301     my $conf = PVE::QemuConfig->load_config($vmid);
2302
2303     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2304
2305     if ($conf->{template}) {
2306         # check if any base image is still used by a linked clone
2307         foreach_drive($conf, sub {
2308                 my ($ds, $drive) = @_;
2309
2310                 return if drive_is_cdrom($drive);
2311
2312                 my $volid = $drive->{file};
2313
2314                 return if !$volid || $volid =~ m|^/|;
2315
2316                 die "base volume '$volid' is still in use by linked cloned\n"
2317                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2318
2319         });
2320     }
2321
2322     # only remove disks owned by this VM
2323     foreach_drive($conf, sub {
2324         my ($ds, $drive) = @_;
2325
2326         return if drive_is_cdrom($drive, 1);
2327
2328         my $volid = $drive->{file};
2329
2330         return if !$volid || $volid =~ m|^/|;
2331
2332         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2333         return if !$path || !$owner || ($owner != $vmid);
2334
2335         eval {
2336             PVE::Storage::vdisk_free($storecfg, $volid);
2337         };
2338         warn "Could not remove disk '$volid', check manually: $@" if $@;
2339
2340     });
2341
2342     if ($keep_empty_config) {
2343         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2344     } else {
2345         unlink $conffile;
2346     }
2347
2348     # also remove unused disk
2349     eval {
2350         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2351
2352         eval {
2353             PVE::Storage::foreach_volid($dl, sub {
2354                 my ($volid, $sid, $volname, $d) = @_;
2355                 PVE::Storage::vdisk_free($storecfg, $volid);
2356             });
2357         };
2358         warn $@ if $@;
2359
2360     };
2361     warn $@ if $@;
2362 }
2363
2364 sub parse_vm_config {
2365     my ($filename, $raw) = @_;
2366
2367     return undef if !defined($raw);
2368
2369     my $res = {
2370         digest => Digest::SHA::sha1_hex($raw),
2371         snapshots => {},
2372         pending => {},
2373     };
2374
2375     $filename =~ m|/qemu-server/(\d+)\.conf$|
2376         || die "got strange filename '$filename'";
2377
2378     my $vmid = $1;
2379
2380     my $conf = $res;
2381     my $descr;
2382     my $section = '';
2383
2384     my @lines = split(/\n/, $raw);
2385     foreach my $line (@lines) {
2386         next if $line =~ m/^\s*$/;
2387
2388         if ($line =~ m/^\[PENDING\]\s*$/i) {
2389             $section = 'pending';
2390             if (defined($descr)) {
2391                 $descr =~ s/\s+$//;
2392                 $conf->{description} = $descr;
2393             }
2394             $descr = undef;
2395             $conf = $res->{$section} = {};
2396             next;
2397
2398         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2399             $section = $1;
2400             if (defined($descr)) {
2401                 $descr =~ s/\s+$//;
2402                 $conf->{description} = $descr;
2403             }
2404             $descr = undef;
2405             $conf = $res->{snapshots}->{$section} = {};
2406             next;
2407         }
2408
2409         if ($line =~ m/^\#(.*)\s*$/) {
2410             $descr = '' if !defined($descr);
2411             $descr .= PVE::Tools::decode_text($1) . "\n";
2412             next;
2413         }
2414
2415         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2416             $descr = '' if !defined($descr);
2417             $descr .= PVE::Tools::decode_text($2);
2418         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2419             $conf->{snapstate} = $1;
2420         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2421             my $key = $1;
2422             my $value = $2;
2423             $conf->{$key} = $value;
2424         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2425             my $value = $1;
2426             if ($section eq 'pending') {
2427                 $conf->{delete} = $value; # we parse this later
2428             } else {
2429                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2430             }
2431         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2432             my $key = $1;
2433             my $value = $2;
2434             eval { $value = check_type($key, $value); };
2435             if ($@) {
2436                 warn "vm $vmid - unable to parse value of '$key' - $@";
2437             } else {
2438                 $key = 'ide2' if $key eq 'cdrom';
2439                 my $fmt = $confdesc->{$key}->{format};
2440                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2441                     my $v = parse_drive($key, $value);
2442                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2443                         $v->{file} = $volid;
2444                         $value = print_drive($vmid, $v);
2445                     } else {
2446                         warn "vm $vmid - unable to parse value of '$key'\n";
2447                         next;
2448                     }
2449                 }
2450
2451                 $conf->{$key} = $value;
2452             }
2453         }
2454     }
2455
2456     if (defined($descr)) {
2457         $descr =~ s/\s+$//;
2458         $conf->{description} = $descr;
2459     }
2460     delete $res->{snapstate}; # just to be sure
2461
2462     return $res;
2463 }
2464
2465 sub write_vm_config {
2466     my ($filename, $conf) = @_;
2467
2468     delete $conf->{snapstate}; # just to be sure
2469
2470     if ($conf->{cdrom}) {
2471         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2472         $conf->{ide2} = $conf->{cdrom};
2473         delete $conf->{cdrom};
2474     }
2475
2476     # we do not use 'smp' any longer
2477     if ($conf->{sockets}) {
2478         delete $conf->{smp};
2479     } elsif ($conf->{smp}) {
2480         $conf->{sockets} = $conf->{smp};
2481         delete $conf->{cores};
2482         delete $conf->{smp};
2483     }
2484
2485     my $used_volids = {};
2486
2487     my $cleanup_config = sub {
2488         my ($cref, $pending, $snapname) = @_;
2489
2490         foreach my $key (keys %$cref) {
2491             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2492                 $key eq 'snapstate' || $key eq 'pending';
2493             my $value = $cref->{$key};
2494             if ($key eq 'delete') {
2495                 die "propertry 'delete' is only allowed in [PENDING]\n"
2496                     if !$pending;
2497                 # fixme: check syntax?
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             die "unable to parse value of '$key' - $@" if $@;
2502
2503             $cref->{$key} = $value;
2504
2505             if (!$snapname && is_valid_drivename($key)) {
2506                 my $drive = parse_drive($key, $value);
2507                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2508             }
2509         }
2510     };
2511
2512     &$cleanup_config($conf);
2513
2514     &$cleanup_config($conf->{pending}, 1);
2515
2516     foreach my $snapname (keys %{$conf->{snapshots}}) {
2517         die "internal error" if $snapname eq 'pending';
2518         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2519     }
2520
2521     # remove 'unusedX' settings if we re-add a volume
2522     foreach my $key (keys %$conf) {
2523         my $value = $conf->{$key};
2524         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2525             delete $conf->{$key};
2526         }
2527     }
2528
2529     my $generate_raw_config = sub {
2530         my ($conf, $pending) = @_;
2531
2532         my $raw = '';
2533
2534         # add description as comment to top of file
2535         if (defined(my $descr = $conf->{description})) {
2536             if ($descr) {
2537                 foreach my $cl (split(/\n/, $descr)) {
2538                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2539                 }
2540             } else {
2541                 $raw .= "#\n" if $pending;
2542             }
2543         }
2544
2545         foreach my $key (sort keys %$conf) {
2546             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2547             $raw .= "$key: $conf->{$key}\n";
2548         }
2549         return $raw;
2550     };
2551
2552     my $raw = &$generate_raw_config($conf);
2553
2554     if (scalar(keys %{$conf->{pending}})){
2555         $raw .= "\n[PENDING]\n";
2556         $raw .= &$generate_raw_config($conf->{pending}, 1);
2557     }
2558
2559     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2560         $raw .= "\n[$snapname]\n";
2561         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2562     }
2563
2564     return $raw;
2565 }
2566
2567 sub load_defaults {
2568
2569     my $res = {};
2570
2571     # we use static defaults from our JSON schema configuration
2572     foreach my $key (keys %$confdesc) {
2573         if (defined(my $default = $confdesc->{$key}->{default})) {
2574             $res->{$key} = $default;
2575         }
2576     }
2577
2578     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2579     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2580
2581     return $res;
2582 }
2583
2584 sub config_list {
2585     my $vmlist = PVE::Cluster::get_vmlist();
2586     my $res = {};
2587     return $res if !$vmlist || !$vmlist->{ids};
2588     my $ids = $vmlist->{ids};
2589
2590     foreach my $vmid (keys %$ids) {
2591         my $d = $ids->{$vmid};
2592         next if !$d->{node} || $d->{node} ne $nodename;
2593         next if !$d->{type} || $d->{type} ne 'qemu';
2594         $res->{$vmid}->{exists} = 1;
2595     }
2596     return $res;
2597 }
2598
2599 # test if VM uses local resources (to prevent migration)
2600 sub check_local_resources {
2601     my ($conf, $noerr) = @_;
2602
2603     my $loc_res = 0;
2604
2605     $loc_res = 1 if $conf->{hostusb}; # old syntax
2606     $loc_res = 1 if $conf->{hostpci}; # old syntax
2607
2608     foreach my $k (keys %$conf) {
2609         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2610         # sockets are safe: they will recreated be on the target side post-migrate
2611         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2612         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2613     }
2614
2615     die "VM uses local resources\n" if $loc_res && !$noerr;
2616
2617     return $loc_res;
2618 }
2619
2620 # check if used storages are available on all nodes (use by migrate)
2621 sub check_storage_availability {
2622     my ($storecfg, $conf, $node) = @_;
2623
2624     foreach_drive($conf, sub {
2625         my ($ds, $drive) = @_;
2626
2627         my $volid = $drive->{file};
2628         return if !$volid;
2629
2630         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2631         return if !$sid;
2632
2633         # check if storage is available on both nodes
2634         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2635         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2636    });
2637 }
2638
2639 # list nodes where all VM images are available (used by has_feature API)
2640 sub shared_nodes {
2641     my ($conf, $storecfg) = @_;
2642
2643     my $nodelist = PVE::Cluster::get_nodelist();
2644     my $nodehash = { map { $_ => 1 } @$nodelist };
2645     my $nodename = PVE::INotify::nodename();
2646
2647     foreach_drive($conf, sub {
2648         my ($ds, $drive) = @_;
2649
2650         my $volid = $drive->{file};
2651         return if !$volid;
2652
2653         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2654         if ($storeid) {
2655             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2656             if ($scfg->{disable}) {
2657                 $nodehash = {};
2658             } elsif (my $avail = $scfg->{nodes}) {
2659                 foreach my $node (keys %$nodehash) {
2660                     delete $nodehash->{$node} if !$avail->{$node};
2661                 }
2662             } elsif (!$scfg->{shared}) {
2663                 foreach my $node (keys %$nodehash) {
2664                     delete $nodehash->{$node} if $node ne $nodename
2665                 }
2666             }
2667         }
2668     });
2669
2670     return $nodehash
2671 }
2672
2673 sub check_cmdline {
2674     my ($pidfile, $pid) = @_;
2675
2676     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2677     if (defined($fh)) {
2678         my $line = <$fh>;
2679         $fh->close;
2680         return undef if !$line;
2681         my @param = split(/\0/, $line);
2682
2683         my $cmd = $param[0];
2684         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2685
2686         for (my $i = 0; $i < scalar (@param); $i++) {
2687             my $p = $param[$i];
2688             next if !$p;
2689             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2690                 my $p = $param[$i+1];
2691                 return 1 if $p && ($p eq $pidfile);
2692                 return undef;
2693             }
2694         }
2695     }
2696     return undef;
2697 }
2698
2699 sub check_running {
2700     my ($vmid, $nocheck, $node) = @_;
2701
2702     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2703
2704     die "unable to find configuration file for VM $vmid - no such machine\n"
2705         if !$nocheck && ! -f $filename;
2706
2707     my $pidfile = pidfile_name($vmid);
2708
2709     if (my $fd = IO::File->new("<$pidfile")) {
2710         my $st = stat($fd);
2711         my $line = <$fd>;
2712         close($fd);
2713
2714         my $mtime = $st->mtime;
2715         if ($mtime > time()) {
2716             warn "file '$filename' modified in future\n";
2717         }
2718
2719         if ($line =~ m/^(\d+)$/) {
2720             my $pid = $1;
2721             if (check_cmdline($pidfile, $pid)) {
2722                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2723                     return $pid;
2724                 }
2725             }
2726         }
2727     }
2728
2729     return undef;
2730 }
2731
2732 sub vzlist {
2733
2734     my $vzlist = config_list();
2735
2736     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2737
2738     while (defined(my $de = $fd->read)) {
2739         next if $de !~ m/^(\d+)\.pid$/;
2740         my $vmid = $1;
2741         next if !defined($vzlist->{$vmid});
2742         if (my $pid = check_running($vmid)) {
2743             $vzlist->{$vmid}->{pid} = $pid;
2744         }
2745     }
2746
2747     return $vzlist;
2748 }
2749
2750 sub disksize {
2751     my ($storecfg, $conf) = @_;
2752
2753     my $bootdisk = $conf->{bootdisk};
2754     return undef if !$bootdisk;
2755     return undef if !is_valid_drivename($bootdisk);
2756
2757     return undef if !$conf->{$bootdisk};
2758
2759     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2760     return undef if !defined($drive);
2761
2762     return undef if drive_is_cdrom($drive);
2763
2764     my $volid = $drive->{file};
2765     return undef if !$volid;
2766
2767     return $drive->{size};
2768 }
2769
2770 my $last_proc_pid_stat;
2771
2772 # get VM status information
2773 # This must be fast and should not block ($full == false)
2774 # We only query KVM using QMP if $full == true (this can be slow)
2775 sub vmstatus {
2776     my ($opt_vmid, $full) = @_;
2777
2778     my $res = {};
2779
2780     my $storecfg = PVE::Storage::config();
2781
2782     my $list = vzlist();
2783     my $defaults = load_defaults();
2784
2785     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2786
2787     my $cpucount = $cpuinfo->{cpus} || 1;
2788
2789     foreach my $vmid (keys %$list) {
2790         next if $opt_vmid && ($vmid ne $opt_vmid);
2791
2792         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2793         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2794
2795         my $d = {};
2796         $d->{pid} = $list->{$vmid}->{pid};
2797
2798         # fixme: better status?
2799         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2800
2801         my $size = disksize($storecfg, $conf);
2802         if (defined($size)) {
2803             $d->{disk} = 0; # no info available
2804             $d->{maxdisk} = $size;
2805         } else {
2806             $d->{disk} = 0;
2807             $d->{maxdisk} = 0;
2808         }
2809
2810         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2811             * ($conf->{cores} || $defaults->{cores});
2812         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2813         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2814
2815         $d->{name} = $conf->{name} || "VM $vmid";
2816         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2817             : $defaults->{memory}*(1024*1024);
2818
2819         if ($conf->{balloon}) {
2820             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2821             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2822                 : $defaults->{shares};
2823         }
2824
2825         $d->{uptime} = 0;
2826         $d->{cpu} = 0;
2827         $d->{mem} = 0;
2828
2829         $d->{netout} = 0;
2830         $d->{netin} = 0;
2831
2832         $d->{diskread} = 0;
2833         $d->{diskwrite} = 0;
2834
2835         $d->{template} = PVE::QemuConfig->is_template($conf);
2836
2837         $d->{serial} = 1 if conf_has_serial($conf);
2838
2839         $res->{$vmid} = $d;
2840     }
2841
2842     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2843     foreach my $dev (keys %$netdev) {
2844         next if $dev !~ m/^tap([1-9]\d*)i/;
2845         my $vmid = $1;
2846         my $d = $res->{$vmid};
2847         next if !$d;
2848
2849         $d->{netout} += $netdev->{$dev}->{receive};
2850         $d->{netin} += $netdev->{$dev}->{transmit};
2851
2852         if ($full) {
2853             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2854             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2855         }
2856
2857     }
2858
2859     my $ctime = gettimeofday;
2860
2861     foreach my $vmid (keys %$list) {
2862
2863         my $d = $res->{$vmid};
2864         my $pid = $d->{pid};
2865         next if !$pid;
2866
2867         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2868         next if !$pstat; # not running
2869
2870         my $used = $pstat->{utime} + $pstat->{stime};
2871
2872         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2873
2874         if ($pstat->{vsize}) {
2875             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2876         }
2877
2878         my $old = $last_proc_pid_stat->{$pid};
2879         if (!$old) {
2880             $last_proc_pid_stat->{$pid} = {
2881                 time => $ctime,
2882                 used => $used,
2883                 cpu => 0,
2884             };
2885             next;
2886         }
2887
2888         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2889
2890         if ($dtime > 1000) {
2891             my $dutime = $used -  $old->{used};
2892
2893             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2894             $last_proc_pid_stat->{$pid} = {
2895                 time => $ctime,
2896                 used => $used,
2897                 cpu => $d->{cpu},
2898             };
2899         } else {
2900             $d->{cpu} = $old->{cpu};
2901         }
2902     }
2903
2904     return $res if !$full;
2905
2906     my $qmpclient = PVE::QMPClient->new();
2907
2908     my $ballooncb = sub {
2909         my ($vmid, $resp) = @_;
2910
2911         my $info = $resp->{'return'};
2912         return if !$info->{max_mem};
2913
2914         my $d = $res->{$vmid};
2915
2916         # use memory assigned to VM
2917         $d->{maxmem} = $info->{max_mem};
2918         $d->{balloon} = $info->{actual};
2919
2920         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2921             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2922             $d->{freemem} = $info->{free_mem};
2923         }
2924
2925         $d->{ballooninfo} = $info;
2926     };
2927
2928     my $blockstatscb = sub {
2929         my ($vmid, $resp) = @_;
2930         my $data = $resp->{'return'} || [];
2931         my $totalrdbytes = 0;
2932         my $totalwrbytes = 0;
2933
2934         for my $blockstat (@$data) {
2935             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2936             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2937
2938             $blockstat->{device} =~ s/drive-//;
2939             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2940         }
2941         $res->{$vmid}->{diskread} = $totalrdbytes;
2942         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2943     };
2944
2945     my $statuscb = sub {
2946         my ($vmid, $resp) = @_;
2947
2948         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2949         # this fails if ballon driver is not loaded, so this must be
2950         # the last commnand (following command are aborted if this fails).
2951         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2952
2953         my $status = 'unknown';
2954         if (!defined($status = $resp->{'return'}->{status})) {
2955             warn "unable to get VM status\n";
2956             return;
2957         }
2958
2959         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2960     };
2961
2962     foreach my $vmid (keys %$list) {
2963         next if $opt_vmid && ($vmid ne $opt_vmid);
2964         next if !$res->{$vmid}->{pid}; # not running
2965         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2966     }
2967
2968     $qmpclient->queue_execute(undef, 2);
2969
2970     foreach my $vmid (keys %$list) {
2971         next if $opt_vmid && ($vmid ne $opt_vmid);
2972         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2973     }
2974
2975     return $res;
2976 }
2977
2978 sub foreach_drive {
2979     my ($conf, $func, @param) = @_;
2980
2981     foreach my $ds (valid_drive_names()) {
2982         next if !defined($conf->{$ds});
2983
2984         my $drive = parse_drive($ds, $conf->{$ds});
2985         next if !$drive;
2986
2987         &$func($ds, $drive, @param);
2988     }
2989 }
2990
2991 sub foreach_volid {
2992     my ($conf, $func, @param) = @_;
2993
2994     my $volhash = {};
2995
2996     my $test_volid = sub {
2997         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2998
2999         return if !$volid;
3000
3001         $volhash->{$volid}->{cdrom} //= 1;
3002         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3003
3004         $volhash->{$volid}->{replicate} //= 0;
3005         $volhash->{$volid}->{replicate} = 1 if $replicate;
3006
3007         $volhash->{$volid}->{shared} //= 0;
3008         $volhash->{$volid}->{shared} = 1 if $shared;
3009
3010         $volhash->{$volid}->{referenced_in_config} //= 0;
3011         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3012
3013         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3014             if defined($snapname);
3015     };
3016
3017     foreach_drive($conf, sub {
3018         my ($ds, $drive) = @_;
3019         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3020     });
3021
3022     foreach my $snapname (keys %{$conf->{snapshots}}) {
3023         my $snap = $conf->{snapshots}->{$snapname};
3024         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3025         foreach_drive($snap, sub {
3026             my ($ds, $drive) = @_;
3027             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3028         });
3029     }
3030
3031     foreach my $volid (keys %$volhash) {
3032         &$func($volid, $volhash->{$volid}, @param);
3033     }
3034 }
3035
3036 sub conf_has_serial {
3037     my ($conf) = @_;
3038
3039     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3040         if ($conf->{"serial$i"}) {
3041             return 1;
3042         }
3043     }
3044
3045     return 0;
3046 }
3047
3048 sub vga_conf_has_spice {
3049     my ($vga) = @_;
3050
3051     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3052
3053     return $1 || 1;
3054 }
3055
3056 sub config_to_command {
3057     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3058
3059     my $cmd = [];
3060     my $globalFlags = [];
3061     my $machineFlags = [];
3062     my $rtcFlags = [];
3063     my $cpuFlags = [];
3064     my $devices = [];
3065     my $pciaddr = '';
3066     my $bridges = {};
3067     my $kvmver = kvm_user_version();
3068     my $vernum = 0; # unknown
3069     my $ostype = $conf->{ostype};
3070     my $winversion = windows_version($ostype);
3071     my $kvm = $conf->{kvm} // 1;
3072
3073     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3074
3075     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3076         $vernum = $1*1000000+$2*1000;
3077     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3078         $vernum = $1*1000000+$2*1000+$3;
3079     }
3080
3081     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3082
3083     my $have_ovz = -f '/proc/vz/vestat';
3084
3085     my $q35 = machine_type_is_q35($conf);
3086     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3087     my $machine_type = $forcemachine || $conf->{machine};
3088     my $use_old_bios_files = undef;
3089     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3090
3091     my $cpuunits = defined($conf->{cpuunits}) ?
3092             $conf->{cpuunits} : $defaults->{cpuunits};
3093
3094     push @$cmd, '/usr/bin/kvm';
3095
3096     push @$cmd, '-id', $vmid;
3097
3098     my $use_virtio = 0;
3099
3100     my $qmpsocket = qmp_socket($vmid);
3101     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3102     push @$cmd, '-mon', "chardev=qmp,mode=control";
3103
3104
3105     push @$cmd, '-pidfile' , pidfile_name($vmid);
3106
3107     push @$cmd, '-daemonize';
3108
3109     if ($conf->{smbios1}) {
3110         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3111     }
3112
3113     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3114         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3115
3116         my $path;
3117         my $format;
3118         if (my $efidisk = $conf->{efidisk0}) {
3119             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3120             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3121             $format = $d->{format};
3122             if ($storeid) {
3123                 $path = PVE::Storage::path($storecfg, $d->{file});
3124                 if (!defined($format)) {
3125                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3126                     $format = qemu_img_format($scfg, $volname);
3127                 }
3128             } else {
3129                 $path = $d->{file};
3130                 die "efidisk format must be specified\n"
3131                     if !defined($format);
3132             }
3133         } else {
3134             warn "no efidisk configured! Using temporary efivars disk.\n";
3135             $path = "/tmp/$vmid-ovmf.fd";
3136             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3137             $format = 'raw';
3138         }
3139
3140         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3141         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3142     }
3143
3144
3145     # add usb controllers
3146     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3147     push @$devices, @usbcontrollers if @usbcontrollers;
3148     my $vga = $conf->{vga};
3149
3150     my $qxlnum = vga_conf_has_spice($vga);
3151     $vga = 'qxl' if $qxlnum;
3152
3153     if (!$vga) {
3154         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3155             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3156         } else {
3157             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3158         }
3159     }
3160
3161     # enable absolute mouse coordinates (needed by vnc)
3162     my $tablet;
3163     if (defined($conf->{tablet})) {
3164         $tablet = $conf->{tablet};
3165     } else {
3166         $tablet = $defaults->{tablet};
3167         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3168         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3169     }
3170
3171     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3172
3173     my $kvm_off = 0;
3174     my $gpu_passthrough;
3175
3176     # host pci devices
3177     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3178         my $d = parse_hostpci($conf->{"hostpci$i"});
3179         next if !$d;
3180
3181         my $pcie = $d->{pcie};
3182         if($pcie){
3183             die "q35 machine model is not enabled" if !$q35;
3184             $pciaddr = print_pcie_addr("hostpci$i");
3185         }else{
3186             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3187         }
3188
3189         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3190         my $romfile = $d->{romfile};
3191
3192         my $xvga = '';
3193         if ($d->{'x-vga'}) {
3194             $xvga = ',x-vga=on';
3195             $kvm_off = 1;
3196             $vga = 'none';
3197             $gpu_passthrough = 1;
3198
3199             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3200                 $xvga = "";
3201             }
3202         }
3203         my $pcidevices = $d->{pciid};
3204         my $multifunction = 1 if @$pcidevices > 1;
3205
3206         my $j=0;
3207         foreach my $pcidevice (@$pcidevices) {
3208
3209             my $id = "hostpci$i";
3210             $id .= ".$j" if $multifunction;
3211             my $addr = $pciaddr;
3212             $addr .= ".$j" if $multifunction;
3213             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3214
3215             if($j == 0){
3216                 $devicestr .= "$rombar$xvga";
3217                 $devicestr .= ",multifunction=on" if $multifunction;
3218                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3219             }
3220
3221             push @$devices, '-device', $devicestr;
3222             $j++;
3223         }
3224     }
3225
3226     # usb devices
3227     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3228     push @$devices, @usbdevices if @usbdevices;
3229     # serial devices
3230     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3231         if (my $path = $conf->{"serial$i"}) {
3232             if ($path eq 'socket') {
3233                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3234                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3235                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3236             } else {
3237                 die "no such serial device\n" if ! -c $path;
3238                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3239                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3240             }
3241         }
3242     }
3243
3244     # parallel devices
3245     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3246         if (my $path = $conf->{"parallel$i"}) {
3247             die "no such parallel device\n" if ! -c $path;
3248             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3249             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3250             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3251         }
3252     }
3253
3254     my $vmname = $conf->{name} || "vm$vmid";
3255
3256     push @$cmd, '-name', $vmname;
3257
3258     my $sockets = 1;
3259     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3260     $sockets = $conf->{sockets} if  $conf->{sockets};
3261
3262     my $cores = $conf->{cores} || 1;
3263
3264     my $maxcpus = $sockets * $cores;
3265
3266     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3267
3268     my $allowed_vcpus = $cpuinfo->{cpus};
3269
3270     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3271         if ($allowed_vcpus < $maxcpus);
3272
3273     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3274
3275         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3276         for (my $i = 2; $i <= $vcpus; $i++)  {
3277             my $cpustr = print_cpu_device($conf,$i);
3278             push @$cmd, '-device', $cpustr;
3279         }
3280
3281     } else {
3282
3283         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3284     }
3285     push @$cmd, '-nodefaults';
3286
3287     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3288
3289     my $bootindex_hash = {};
3290     my $i = 1;
3291     foreach my $o (split(//, $bootorder)) {
3292         $bootindex_hash->{$o} = $i*100;
3293         $i++;
3294     }
3295
3296     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3297
3298     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3299
3300     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3301
3302     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3303
3304     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3305         my $socket = vnc_socket($vmid);
3306         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3307     } else {
3308         push @$cmd, '-nographic';
3309     }
3310
3311     # time drift fix
3312     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3313
3314     my $useLocaltime = $conf->{localtime};
3315
3316     if ($winversion >= 5) { # windows
3317         $useLocaltime = 1 if !defined($conf->{localtime});
3318
3319         # use time drift fix when acpi is enabled
3320         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3321             $tdf = 1 if !defined($conf->{tdf});
3322         }
3323     }
3324
3325     if ($winversion >= 6) {
3326         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3327         push @$cmd, '-no-hpet';
3328     }
3329
3330     push @$rtcFlags, 'driftfix=slew' if $tdf;
3331
3332     if (!$kvm) {
3333         push @$machineFlags, 'accel=tcg';
3334     }
3335
3336     if ($machine_type) {
3337         push @$machineFlags, "type=${machine_type}";
3338     }
3339
3340     if ($conf->{startdate}) {
3341         push @$rtcFlags, "base=$conf->{startdate}";
3342     } elsif ($useLocaltime) {
3343         push @$rtcFlags, 'base=localtime';
3344     }
3345
3346     my $cpu = $kvm ? "kvm64" : "qemu64";
3347     if (my $cputype = $conf->{cpu}) {
3348         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3349             or die "Cannot parse cpu description: $cputype\n";
3350         $cpu = $cpuconf->{cputype};
3351         $kvm_off = 1 if $cpuconf->{hidden};
3352
3353         if (defined(my $flags = $cpuconf->{flags})) {
3354             push @$cpuFlags, split(";", $flags);
3355         }
3356     }
3357
3358     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3359
3360     push @$cpuFlags , '-x2apic'
3361         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3362
3363     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3364
3365     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3366
3367     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3368
3369         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3370         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3371     }
3372
3373     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3374
3375     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3376
3377     push @$cpuFlags, 'kvm=off' if $kvm_off;
3378
3379     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3380         die "internal error"; # should not happen
3381
3382     push @$cpuFlags, "vendor=${cpu_vendor}"
3383         if $cpu_vendor ne 'default';
3384
3385     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3386
3387     push @$cmd, '-cpu', $cpu;
3388
3389     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3390
3391     push @$cmd, '-S' if $conf->{freeze};
3392
3393     # set keyboard layout
3394     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3395     push @$cmd, '-k', $kb if $kb;
3396
3397     # enable sound
3398     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3399     #push @$cmd, '-soundhw', 'es1370';
3400     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3401
3402     if($conf->{agent}) {
3403         my $qgasocket = qmp_socket($vmid, 1);
3404         my $pciaddr = print_pci_addr("qga0", $bridges);
3405         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3406         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3407         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3408     }
3409
3410     my $spice_port;
3411
3412     if ($qxlnum) {
3413         if ($qxlnum > 1) {
3414             if ($winversion){
3415                 for(my $i = 1; $i < $qxlnum; $i++){
3416                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3417                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3418                 }
3419             } else {
3420                 # assume other OS works like Linux
3421                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3422                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3423             }
3424         }
3425
3426         my $pciaddr = print_pci_addr("spice", $bridges);
3427
3428         my $nodename = PVE::INotify::nodename();
3429         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3430         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3431         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3432         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3433         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3434
3435         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3436
3437         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3438         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3439         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3440     }
3441
3442     # enable balloon by default, unless explicitly disabled
3443     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3444         $pciaddr = print_pci_addr("balloon0", $bridges);
3445         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3446     }
3447
3448     if ($conf->{watchdog}) {
3449         my $wdopts = parse_watchdog($conf->{watchdog});
3450         $pciaddr = print_pci_addr("watchdog", $bridges);
3451         my $watchdog = $wdopts->{model} || 'i6300esb';
3452         push @$devices, '-device', "$watchdog$pciaddr";
3453         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3454     }
3455
3456     my $vollist = [];
3457     my $scsicontroller = {};
3458     my $ahcicontroller = {};
3459     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3460
3461     # Add iscsi initiator name if available
3462     if (my $initiator = get_initiator_name()) {
3463         push @$devices, '-iscsi', "initiator-name=$initiator";
3464     }
3465
3466     foreach_drive($conf, sub {
3467         my ($ds, $drive) = @_;
3468
3469         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3470             push @$vollist, $drive->{file};
3471         }
3472
3473         # ignore efidisk here, already added in bios/fw handling code above
3474         return if $drive->{interface} eq 'efidisk';
3475
3476         $use_virtio = 1 if $ds =~ m/^virtio/;
3477
3478         if (drive_is_cdrom ($drive)) {
3479             if ($bootindex_hash->{d}) {
3480                 $drive->{bootindex} = $bootindex_hash->{d};
3481                 $bootindex_hash->{d} += 1;
3482             }
3483         } else {
3484             if ($bootindex_hash->{c}) {
3485                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3486                 $bootindex_hash->{c} += 1;
3487             }
3488         }
3489
3490         if($drive->{interface} eq 'virtio'){
3491            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3492         }
3493
3494         if ($drive->{interface} eq 'scsi') {
3495
3496             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3497
3498             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3499             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3500
3501             my $iothread = '';
3502             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3503                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3504                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3505             } elsif ($drive->{iothread}) {
3506                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3507             }
3508
3509             my $queues = '';
3510             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3511                 $queues = ",num_queues=$drive->{queues}";
3512             }
3513
3514             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3515             $scsicontroller->{$controller}=1;
3516         }
3517
3518         if ($drive->{interface} eq 'sata') {
3519            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3520            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3521            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3522            $ahcicontroller->{$controller}=1;
3523         }
3524
3525         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3526         push @$devices, '-drive',$drive_cmd;
3527         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3528     });
3529
3530     for (my $i = 0; $i < $MAX_NETS; $i++) {
3531          next if !$conf->{"net$i"};
3532          my $d = parse_net($conf->{"net$i"});
3533          next if !$d;
3534
3535          $use_virtio = 1 if $d->{model} eq 'virtio';
3536
3537          if ($bootindex_hash->{n}) {
3538             $d->{bootindex} = $bootindex_hash->{n};
3539             $bootindex_hash->{n} += 1;
3540          }
3541
3542          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3543          push @$devices, '-netdev', $netdevfull;
3544
3545          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3546          push @$devices, '-device', $netdevicefull;
3547     }
3548
3549     if (!$q35) {
3550         # add pci bridges
3551         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3552            $bridges->{1} = 1;
3553            $bridges->{2} = 1;
3554         }
3555
3556         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3557
3558         while (my ($k, $v) = each %$bridges) {
3559             $pciaddr = print_pci_addr("pci.$k");
3560             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3561         }
3562     }
3563
3564     # add custom args
3565     if ($conf->{args}) {
3566         my $aa = PVE::Tools::split_args($conf->{args});
3567         push @$cmd, @$aa;
3568     }
3569
3570     push @$cmd, @$devices;
3571     push @$cmd, '-rtc', join(',', @$rtcFlags)
3572         if scalar(@$rtcFlags);
3573     push @$cmd, '-machine', join(',', @$machineFlags)
3574         if scalar(@$machineFlags);
3575     push @$cmd, '-global', join(',', @$globalFlags)
3576         if scalar(@$globalFlags);
3577
3578     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3579 }
3580
3581 sub vnc_socket {
3582     my ($vmid) = @_;
3583     return "${var_run_tmpdir}/$vmid.vnc";
3584 }
3585
3586 sub spice_port {
3587     my ($vmid) = @_;
3588
3589     my $res = vm_mon_cmd($vmid, 'query-spice');
3590
3591     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3592 }
3593
3594 sub qmp_socket {
3595     my ($vmid, $qga) = @_;
3596     my $sockettype = $qga ? 'qga' : 'qmp';
3597     return "${var_run_tmpdir}/$vmid.$sockettype";
3598 }
3599
3600 sub pidfile_name {
3601     my ($vmid) = @_;
3602     return "${var_run_tmpdir}/$vmid.pid";
3603 }
3604
3605 sub vm_devices_list {
3606     my ($vmid) = @_;
3607
3608     my $res = vm_mon_cmd($vmid, 'query-pci');
3609     my $devices = {};
3610     foreach my $pcibus (@$res) {
3611         foreach my $device (@{$pcibus->{devices}}) {
3612             next if !$device->{'qdev_id'};
3613             if ($device->{'pci_bridge'}) {
3614                 $devices->{$device->{'qdev_id'}} = 1;
3615                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3616                     next if !$bridge_device->{'qdev_id'};
3617                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3618                     $devices->{$device->{'qdev_id'}}++;
3619                 }
3620             } else {
3621                 $devices->{$device->{'qdev_id'}} = 1;
3622             }
3623         }
3624     }
3625
3626     my $resblock = vm_mon_cmd($vmid, 'query-block');
3627     foreach my $block (@$resblock) {
3628         if($block->{device} =~ m/^drive-(\S+)/){
3629                 $devices->{$1} = 1;
3630         }
3631     }
3632
3633     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3634     foreach my $mice (@$resmice) {
3635         if ($mice->{name} eq 'QEMU HID Tablet') {
3636             $devices->{tablet} = 1;
3637             last;
3638         }
3639     }
3640
3641     # for usb devices there is no query-usb
3642     # but we can iterate over the entries in
3643     # qom-list path=/machine/peripheral
3644     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3645     foreach my $per (@$resperipheral) {
3646         if ($per->{name} =~ m/^usb\d+$/) {
3647             $devices->{$per->{name}} = 1;
3648         }
3649     }
3650
3651     return $devices;
3652 }
3653
3654 sub vm_deviceplug {
3655     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3656
3657     my $q35 = machine_type_is_q35($conf);
3658
3659     my $devices_list = vm_devices_list($vmid);
3660     return 1 if defined($devices_list->{$deviceid});
3661
3662     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3663
3664     if ($deviceid eq 'tablet') {
3665
3666         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3667
3668     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3669
3670         die "usb hotplug currently not reliable\n";
3671         # since we can't reliably hot unplug all added usb devices
3672         # and usb passthrough disables live migration
3673         # we disable usb hotplugging for now
3674         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3675
3676     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3677
3678         qemu_iothread_add($vmid, $deviceid, $device);
3679
3680         qemu_driveadd($storecfg, $vmid, $device);
3681         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3682
3683         qemu_deviceadd($vmid, $devicefull);
3684         eval { qemu_deviceaddverify($vmid, $deviceid); };
3685         if (my $err = $@) {
3686             eval { qemu_drivedel($vmid, $deviceid); };
3687             warn $@ if $@;
3688             die $err;
3689         }
3690
3691     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3692
3693
3694         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3695         my $pciaddr = print_pci_addr($deviceid);
3696         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3697
3698         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3699
3700         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3701             qemu_iothread_add($vmid, $deviceid, $device);
3702             $devicefull .= ",iothread=iothread-$deviceid";
3703         }
3704
3705         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3706             $devicefull .= ",num_queues=$device->{queues}";
3707         }
3708
3709         qemu_deviceadd($vmid, $devicefull);
3710         qemu_deviceaddverify($vmid, $deviceid);
3711
3712     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3713
3714         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3715         qemu_driveadd($storecfg, $vmid, $device);
3716
3717         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3718         eval { qemu_deviceadd($vmid, $devicefull); };
3719         if (my $err = $@) {
3720             eval { qemu_drivedel($vmid, $deviceid); };
3721             warn $@ if $@;
3722             die $err;
3723         }
3724
3725     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3726
3727         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3728
3729         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3730         my $use_old_bios_files = undef;
3731         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3732
3733         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3734         qemu_deviceadd($vmid, $netdevicefull);
3735         eval { qemu_deviceaddverify($vmid, $deviceid); };
3736         if (my $err = $@) {
3737             eval { qemu_netdevdel($vmid, $deviceid); };
3738             warn $@ if $@;
3739             die $err;
3740         }
3741
3742     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3743
3744         my $bridgeid = $2;
3745         my $pciaddr = print_pci_addr($deviceid);
3746         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3747
3748         qemu_deviceadd($vmid, $devicefull);
3749         qemu_deviceaddverify($vmid, $deviceid);
3750
3751     } else {
3752         die "can't hotplug device '$deviceid'\n";
3753     }
3754
3755     return 1;
3756 }
3757
3758 # fixme: this should raise exceptions on error!
3759 sub vm_deviceunplug {
3760     my ($vmid, $conf, $deviceid) = @_;
3761
3762     my $devices_list = vm_devices_list($vmid);
3763     return 1 if !defined($devices_list->{$deviceid});
3764
3765     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3766
3767     if ($deviceid eq 'tablet') {
3768
3769         qemu_devicedel($vmid, $deviceid);
3770
3771     } elsif ($deviceid =~ m/^usb\d+$/) {
3772
3773         die "usb hotplug currently not reliable\n";
3774         # when unplugging usb devices this way,
3775         # there may be remaining usb controllers/hubs
3776         # so we disable it for now
3777         qemu_devicedel($vmid, $deviceid);
3778         qemu_devicedelverify($vmid, $deviceid);
3779
3780     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3781
3782         qemu_devicedel($vmid, $deviceid);
3783         qemu_devicedelverify($vmid, $deviceid);
3784         qemu_drivedel($vmid, $deviceid);
3785         qemu_iothread_del($conf, $vmid, $deviceid);
3786
3787     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3788
3789         qemu_devicedel($vmid, $deviceid);
3790         qemu_devicedelverify($vmid, $deviceid);
3791         qemu_iothread_del($conf, $vmid, $deviceid);
3792
3793     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3794
3795         qemu_devicedel($vmid, $deviceid);
3796         qemu_drivedel($vmid, $deviceid);
3797         qemu_deletescsihw($conf, $vmid, $deviceid);
3798
3799     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3800
3801         qemu_devicedel($vmid, $deviceid);
3802         qemu_devicedelverify($vmid, $deviceid);
3803         qemu_netdevdel($vmid, $deviceid);
3804
3805     } else {
3806         die "can't unplug device '$deviceid'\n";
3807     }
3808
3809     return 1;
3810 }
3811
3812 sub qemu_deviceadd {
3813     my ($vmid, $devicefull) = @_;
3814
3815     $devicefull = "driver=".$devicefull;
3816     my %options =  split(/[=,]/, $devicefull);
3817
3818     vm_mon_cmd($vmid, "device_add" , %options);
3819 }
3820
3821 sub qemu_devicedel {
3822     my ($vmid, $deviceid) = @_;
3823
3824     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3825 }
3826
3827 sub qemu_iothread_add {
3828     my($vmid, $deviceid, $device) = @_;
3829
3830     if ($device->{iothread}) {
3831         my $iothreads = vm_iothreads_list($vmid);
3832         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3833     }
3834 }
3835
3836 sub qemu_iothread_del {
3837     my($conf, $vmid, $deviceid) = @_;
3838
3839     my $device = parse_drive($deviceid, $conf->{$deviceid});
3840     if ($device->{iothread}) {
3841         my $iothreads = vm_iothreads_list($vmid);
3842         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3843     }
3844 }
3845
3846 sub qemu_objectadd {
3847     my($vmid, $objectid, $qomtype) = @_;
3848
3849     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3850
3851     return 1;
3852 }
3853
3854 sub qemu_objectdel {
3855     my($vmid, $objectid) = @_;
3856
3857     vm_mon_cmd($vmid, "object-del", id => $objectid);
3858
3859     return 1;
3860 }
3861
3862 sub qemu_driveadd {
3863     my ($storecfg, $vmid, $device) = @_;
3864
3865     my $drive = print_drive_full($storecfg, $vmid, $device);
3866     $drive =~ s/\\/\\\\/g;
3867     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3868
3869     # If the command succeeds qemu prints: "OK"
3870     return 1 if $ret =~ m/OK/s;
3871
3872     die "adding drive failed: $ret\n";
3873 }
3874
3875 sub qemu_drivedel {
3876     my($vmid, $deviceid) = @_;
3877
3878     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3879     $ret =~ s/^\s+//;
3880
3881     return 1 if $ret eq "";
3882
3883     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3884     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3885
3886     die "deleting drive $deviceid failed : $ret\n";
3887 }
3888
3889 sub qemu_deviceaddverify {
3890     my ($vmid, $deviceid) = @_;
3891
3892     for (my $i = 0; $i <= 5; $i++) {
3893          my $devices_list = vm_devices_list($vmid);
3894          return 1 if defined($devices_list->{$deviceid});
3895          sleep 1;
3896     }
3897
3898     die "error on hotplug device '$deviceid'\n";
3899 }
3900
3901
3902 sub qemu_devicedelverify {
3903     my ($vmid, $deviceid) = @_;
3904
3905     # need to verify that the device is correctly removed as device_del
3906     # is async and empty return is not reliable
3907
3908     for (my $i = 0; $i <= 5; $i++) {
3909          my $devices_list = vm_devices_list($vmid);
3910          return 1 if !defined($devices_list->{$deviceid});
3911          sleep 1;
3912     }
3913
3914     die "error on hot-unplugging device '$deviceid'\n";
3915 }
3916
3917 sub qemu_findorcreatescsihw {
3918     my ($storecfg, $conf, $vmid, $device) = @_;
3919
3920     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3921
3922     my $scsihwid="$controller_prefix$controller";
3923     my $devices_list = vm_devices_list($vmid);
3924
3925     if(!defined($devices_list->{$scsihwid})) {
3926         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3927     }
3928
3929     return 1;
3930 }
3931
3932 sub qemu_deletescsihw {
3933     my ($conf, $vmid, $opt) = @_;
3934
3935     my $device = parse_drive($opt, $conf->{$opt});
3936
3937     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3938         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3939         return 1;
3940     }
3941
3942     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3943
3944     my $devices_list = vm_devices_list($vmid);
3945     foreach my $opt (keys %{$devices_list}) {
3946         if (PVE::QemuServer::is_valid_drivename($opt)) {
3947             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3948             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3949                 return 1;
3950             }
3951         }
3952     }
3953
3954     my $scsihwid="scsihw$controller";
3955
3956     vm_deviceunplug($vmid, $conf, $scsihwid);
3957
3958     return 1;
3959 }
3960
3961 sub qemu_add_pci_bridge {
3962     my ($storecfg, $conf, $vmid, $device) = @_;
3963
3964     my $bridges = {};
3965
3966     my $bridgeid;
3967
3968     print_pci_addr($device, $bridges);
3969
3970     while (my ($k, $v) = each %$bridges) {
3971         $bridgeid = $k;
3972     }
3973     return 1 if !defined($bridgeid) || $bridgeid < 1;
3974
3975     my $bridge = "pci.$bridgeid";
3976     my $devices_list = vm_devices_list($vmid);
3977
3978     if (!defined($devices_list->{$bridge})) {
3979         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3980     }
3981
3982     return 1;
3983 }
3984
3985 sub qemu_set_link_status {
3986     my ($vmid, $device, $up) = @_;
3987
3988     vm_mon_cmd($vmid, "set_link", name => $device,
3989                up => $up ? JSON::true : JSON::false);
3990 }
3991
3992 sub qemu_netdevadd {
3993     my ($vmid, $conf, $device, $deviceid) = @_;
3994
3995     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3996     my %options =  split(/[=,]/, $netdev);
3997
3998     vm_mon_cmd($vmid, "netdev_add",  %options);
3999     return 1;
4000 }
4001
4002 sub qemu_netdevdel {
4003     my ($vmid, $deviceid) = @_;
4004
4005     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4006 }
4007
4008 sub qemu_usb_hotplug {
4009     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4010
4011     return if !$device;
4012
4013     # remove the old one first
4014     vm_deviceunplug($vmid, $conf, $deviceid);
4015
4016     # check if xhci controller is necessary and available
4017     if ($device->{usb3}) {
4018
4019         my $devicelist = vm_devices_list($vmid);
4020
4021         if (!$devicelist->{xhci}) {
4022             my $pciaddr = print_pci_addr("xhci");
4023             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4024         }
4025     }
4026     my $d = parse_usb_device($device->{host});
4027     $d->{usb3} = $device->{usb3};
4028
4029     # add the new one
4030     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4031 }
4032
4033 sub qemu_cpu_hotplug {
4034     my ($vmid, $conf, $vcpus) = @_;
4035
4036     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4037
4038     my $sockets = 1;
4039     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4040     $sockets = $conf->{sockets} if  $conf->{sockets};
4041     my $cores = $conf->{cores} || 1;
4042     my $maxcpus = $sockets * $cores;
4043
4044     $vcpus = $maxcpus if !$vcpus;
4045
4046     die "you can't add more vcpus than maxcpus\n"
4047         if $vcpus > $maxcpus;
4048
4049     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4050
4051     if ($vcpus < $currentvcpus) {
4052
4053         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4054
4055             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4056                 qemu_devicedel($vmid, "cpu$i");
4057                 my $retry = 0;
4058                 my $currentrunningvcpus = undef;
4059                 while (1) {
4060                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4061                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4062                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4063                     $retry++;
4064                     sleep 1;
4065                 }
4066                 #update conf after each succesfull cpu unplug
4067                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4068                 PVE::QemuConfig->write_config($vmid, $conf);
4069             }
4070         } else {
4071             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4072         }
4073
4074         return;
4075     }
4076
4077     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4078     die "vcpus in running vm does not match its configuration\n"
4079         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4080
4081     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4082
4083         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4084             my $cpustr = print_cpu_device($conf, $i);
4085             qemu_deviceadd($vmid, $cpustr);
4086
4087             my $retry = 0;
4088             my $currentrunningvcpus = undef;
4089             while (1) {
4090                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4091                 last if scalar(@{$currentrunningvcpus}) == $i;
4092                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4093                 sleep 1;
4094                 $retry++;
4095             }
4096             #update conf after each succesfull cpu hotplug
4097             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4098             PVE::QemuConfig->write_config($vmid, $conf);
4099         }
4100     } else {
4101
4102         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4103             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4104         }
4105     }
4106 }
4107
4108 sub qemu_block_set_io_throttle {
4109     my ($vmid, $deviceid,
4110         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4111         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4112         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4113         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4114
4115     return if !check_running($vmid) ;
4116
4117     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4118         bps => int($bps),
4119         bps_rd => int($bps_rd),
4120         bps_wr => int($bps_wr),
4121         iops => int($iops),
4122         iops_rd => int($iops_rd),
4123         iops_wr => int($iops_wr),
4124         bps_max => int($bps_max),
4125         bps_rd_max => int($bps_rd_max),
4126         bps_wr_max => int($bps_wr_max),
4127         iops_max => int($iops_max),
4128         iops_rd_max => int($iops_rd_max),
4129         iops_wr_max => int($iops_wr_max),
4130         bps_max_length => int($bps_max_length),
4131         bps_rd_max_length => int($bps_rd_max_length),
4132         bps_wr_max_length => int($bps_wr_max_length),
4133         iops_max_length => int($iops_max_length),
4134         iops_rd_max_length => int($iops_rd_max_length),
4135         iops_wr_max_length => int($iops_wr_max_length),
4136     );
4137
4138 }
4139
4140 # old code, only used to shutdown old VM after update
4141 sub __read_avail {
4142     my ($fh, $timeout) = @_;
4143
4144     my $sel = new IO::Select;
4145     $sel->add($fh);
4146
4147     my $res = '';
4148     my $buf;
4149
4150     my @ready;
4151     while (scalar (@ready = $sel->can_read($timeout))) {
4152         my $count;
4153         if ($count = $fh->sysread($buf, 8192)) {
4154             if ($buf =~ /^(.*)\(qemu\) $/s) {
4155                 $res .= $1;
4156                 last;
4157             } else {
4158                 $res .= $buf;
4159             }
4160         } else {
4161             if (!defined($count)) {
4162                 die "$!\n";
4163             }
4164             last;
4165         }
4166     }
4167
4168     die "monitor read timeout\n" if !scalar(@ready);
4169
4170     return $res;
4171 }
4172
4173 # old code, only used to shutdown old VM after update
4174 sub vm_monitor_command {
4175     my ($vmid, $cmdstr, $nocheck) = @_;
4176
4177     my $res;
4178
4179     eval {
4180         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4181
4182         my $sname = "${var_run_tmpdir}/$vmid.mon";
4183
4184         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4185             die "unable to connect to VM $vmid socket - $!\n";
4186
4187         my $timeout = 3;
4188
4189         # hack: migrate sometime blocks the monitor (when migrate_downtime
4190         # is set)
4191         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4192             $timeout = 60*60; # 1 hour
4193         }
4194
4195         # read banner;
4196         my $data = __read_avail($sock, $timeout);
4197
4198         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4199             die "got unexpected qemu monitor banner\n";
4200         }
4201
4202         my $sel = new IO::Select;
4203         $sel->add($sock);
4204
4205         if (!scalar(my @ready = $sel->can_write($timeout))) {
4206             die "monitor write error - timeout";
4207         }
4208
4209         my $fullcmd = "$cmdstr\r";
4210
4211         # syslog('info', "VM $vmid monitor command: $cmdstr");
4212
4213         my $b;
4214         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4215             die "monitor write error - $!";
4216         }
4217
4218         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4219
4220         $timeout = 20;
4221
4222         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4223             $timeout = 60*60; # 1 hour
4224         } elsif ($cmdstr =~ m/^(eject|change)/) {
4225             $timeout = 60; # note: cdrom mount command is slow
4226         }
4227         if ($res = __read_avail($sock, $timeout)) {
4228
4229             my @lines = split("\r?\n", $res);
4230
4231             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4232
4233             $res = join("\n", @lines);
4234             $res .= "\n";
4235         }
4236     };
4237
4238     my $err = $@;
4239
4240     if ($err) {
4241         syslog("err", "VM $vmid monitor command failed - $err");
4242         die $err;
4243     }
4244
4245     return $res;
4246 }
4247
4248 sub qemu_block_resize {
4249     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4250
4251     my $running = check_running($vmid);
4252
4253     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4254
4255     return if !$running;
4256
4257     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4258
4259 }
4260
4261 sub qemu_volume_snapshot {
4262     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4263
4264     my $running = check_running($vmid);
4265
4266     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4267         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4268     } else {
4269         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4270     }
4271 }
4272
4273 sub qemu_volume_snapshot_delete {
4274     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4275
4276     my $running = check_running($vmid);
4277
4278     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4279         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4280     } else {
4281         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4282     }
4283 }
4284
4285 sub set_migration_caps {
4286     my ($vmid) = @_;
4287
4288     my $cap_ref = [];
4289
4290     my $enabled_cap = {
4291         "auto-converge" => 1,
4292         "xbzrle" => 1,
4293         "x-rdma-pin-all" => 0,
4294         "zero-blocks" => 0,
4295         "compress" => 0
4296     };
4297
4298     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4299
4300     for my $supported_capability (@$supported_capabilities) {
4301         push @$cap_ref, {
4302             capability => $supported_capability->{capability},
4303             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4304         };
4305     }
4306
4307     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4308 }
4309
4310 my $fast_plug_option = {
4311     'lock' => 1,
4312     'name' => 1,
4313     'onboot' => 1,
4314     'shares' => 1,
4315     'startup' => 1,
4316     'description' => 1,
4317     'protection' => 1,
4318     'vmstatestorage' => 1,
4319 };
4320
4321 # hotplug changes in [PENDING]
4322 # $selection hash can be used to only apply specified options, for
4323 # example: { cores => 1 } (only apply changed 'cores')
4324 # $errors ref is used to return error messages
4325 sub vmconfig_hotplug_pending {
4326     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4327
4328     my $defaults = load_defaults();
4329
4330     # commit values which do not have any impact on running VM first
4331     # Note: those option cannot raise errors, we we do not care about
4332     # $selection and always apply them.
4333
4334     my $add_error = sub {
4335         my ($opt, $msg) = @_;
4336         $errors->{$opt} = "hotplug problem - $msg";
4337     };
4338
4339     my $changes = 0;
4340     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4341         if ($fast_plug_option->{$opt}) {
4342             $conf->{$opt} = $conf->{pending}->{$opt};
4343             delete $conf->{pending}->{$opt};
4344             $changes = 1;
4345         }
4346     }
4347
4348     if ($changes) {
4349         PVE::QemuConfig->write_config($vmid, $conf);
4350         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4351     }
4352
4353     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4354
4355     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4356     while (my ($opt, $force) = each %$pending_delete_hash) {
4357         next if $selection && !$selection->{$opt};
4358         eval {
4359             if ($opt eq 'hotplug') {
4360                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4361             } elsif ($opt eq 'tablet') {
4362                 die "skip\n" if !$hotplug_features->{usb};
4363                 if ($defaults->{tablet}) {
4364                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4365                 } else {
4366                     vm_deviceunplug($vmid, $conf, $opt);
4367                 }
4368             } elsif ($opt =~ m/^usb\d+/) {
4369                 die "skip\n";
4370                 # since we cannot reliably hot unplug usb devices
4371                 # we are disabling it
4372                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4373                 vm_deviceunplug($vmid, $conf, $opt);
4374             } elsif ($opt eq 'vcpus') {
4375                 die "skip\n" if !$hotplug_features->{cpu};
4376                 qemu_cpu_hotplug($vmid, $conf, undef);
4377             } elsif ($opt eq 'balloon') {
4378                 # enable balloon device is not hotpluggable
4379                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4380             } elsif ($fast_plug_option->{$opt}) {
4381                 # do nothing
4382             } elsif ($opt =~ m/^net(\d+)$/) {
4383                 die "skip\n" if !$hotplug_features->{network};
4384                 vm_deviceunplug($vmid, $conf, $opt);
4385             } elsif (is_valid_drivename($opt)) {
4386                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4387                 vm_deviceunplug($vmid, $conf, $opt);
4388                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4389             } elsif ($opt =~ m/^memory$/) {
4390                 die "skip\n" if !$hotplug_features->{memory};
4391                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4392             } elsif ($opt eq 'cpuunits') {
4393                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4394             } elsif ($opt eq 'cpulimit') {
4395                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4396             } else {
4397                 die "skip\n";
4398             }
4399         };
4400         if (my $err = $@) {
4401             &$add_error($opt, $err) if $err ne "skip\n";
4402         } else {
4403             # save new config if hotplug was successful
4404             delete $conf->{$opt};
4405             vmconfig_undelete_pending_option($conf, $opt);
4406             PVE::QemuConfig->write_config($vmid, $conf);
4407             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4408         }
4409     }
4410
4411     my $apply_pending_cloudinit;
4412     $apply_pending_cloudinit = sub {
4413         my ($key, $value) = @_;
4414         $apply_pending_cloudinit = sub {}; # once is enough
4415
4416         my @cloudinit_opts = keys %$confdesc_cloudinit;
4417         foreach my $opt (keys %{$conf->{pending}}) {
4418             next if !grep { $_ eq $opt } @cloudinit_opts;
4419             $conf->{$opt} = delete $conf->{pending}->{$opt};
4420         }
4421
4422         my $new_conf = { %$conf };
4423         $new_conf->{$key} = $value;
4424         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4425     };
4426
4427     foreach my $opt (keys %{$conf->{pending}}) {
4428         next if $selection && !$selection->{$opt};
4429         my $value = $conf->{pending}->{$opt};
4430         eval {
4431             if ($opt eq 'hotplug') {
4432                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4433             } elsif ($opt eq 'tablet') {
4434                 die "skip\n" if !$hotplug_features->{usb};
4435                 if ($value == 1) {
4436                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4437                 } elsif ($value == 0) {
4438                     vm_deviceunplug($vmid, $conf, $opt);
4439                 }
4440             } elsif ($opt =~ m/^usb\d+$/) {
4441                 die "skip\n";
4442                 # since we cannot reliably hot unplug usb devices
4443                 # we are disabling it
4444                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4445                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4446                 die "skip\n" if !$d;
4447                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4448             } elsif ($opt eq 'vcpus') {
4449                 die "skip\n" if !$hotplug_features->{cpu};
4450                 qemu_cpu_hotplug($vmid, $conf, $value);
4451             } elsif ($opt eq 'balloon') {
4452                 # enable/disable balloning device is not hotpluggable
4453                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4454                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4455                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4456
4457                 # allow manual ballooning if shares is set to zero
4458                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4459                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4460                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4461                 }
4462             } elsif ($opt =~ m/^net(\d+)$/) {
4463                 # some changes can be done without hotplug
4464                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4465                                     $vmid, $opt, $value);
4466             } elsif (is_valid_drivename($opt)) {
4467                 # some changes can be done without hotplug
4468                 my $drive = parse_drive($opt, $value);
4469                 if (drive_is_cloudinit($drive)) {
4470                     &$apply_pending_cloudinit($opt, $value);
4471                 }
4472                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4473                                      $vmid, $opt, $value, 1);
4474             } elsif ($opt =~ m/^memory$/) { #dimms
4475                 die "skip\n" if !$hotplug_features->{memory};
4476                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4477             } elsif ($opt eq 'cpuunits') {
4478                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4479             } elsif ($opt eq 'cpulimit') {
4480                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4481                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4482             } else {
4483                 die "skip\n";  # skip non-hot-pluggable options
4484             }
4485         };
4486         if (my $err = $@) {
4487             &$add_error($opt, $err) if $err ne "skip\n";
4488         } else {
4489             # save new config if hotplug was successful
4490             $conf->{$opt} = $value;
4491             delete $conf->{pending}->{$opt};
4492             PVE::QemuConfig->write_config($vmid, $conf);
4493             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4494         }
4495     }
4496 }
4497
4498 sub try_deallocate_drive {
4499     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4500
4501     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4502         my $volid = $drive->{file};
4503         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4504             my $sid = PVE::Storage::parse_volume_id($volid);
4505             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4506
4507             # check if the disk is really unused
4508             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4509                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4510             PVE::Storage::vdisk_free($storecfg, $volid);
4511             return 1;
4512         } else {
4513             # If vm is not owner of this disk remove from config
4514             return 1;
4515         }
4516     }
4517
4518     return undef;
4519 }
4520
4521 sub vmconfig_delete_or_detach_drive {
4522     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4523
4524     my $drive = parse_drive($opt, $conf->{$opt});
4525
4526     my $rpcenv = PVE::RPCEnvironment::get();
4527     my $authuser = $rpcenv->get_user();
4528
4529     if ($force) {
4530         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4531         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4532     } else {
4533         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4534     }
4535 }
4536
4537 sub vmconfig_apply_pending {
4538     my ($vmid, $conf, $storecfg) = @_;
4539
4540     # cold plug
4541
4542     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4543     while (my ($opt, $force) = each %$pending_delete_hash) {
4544         die "internal error" if $opt =~ m/^unused/;
4545         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4546         if (!defined($conf->{$opt})) {
4547             vmconfig_undelete_pending_option($conf, $opt);
4548             PVE::QemuConfig->write_config($vmid, $conf);
4549         } elsif (is_valid_drivename($opt)) {
4550             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4551             vmconfig_undelete_pending_option($conf, $opt);
4552             delete $conf->{$opt};
4553             PVE::QemuConfig->write_config($vmid, $conf);
4554         } else {
4555             vmconfig_undelete_pending_option($conf, $opt);
4556             delete $conf->{$opt};
4557             PVE::QemuConfig->write_config($vmid, $conf);
4558         }
4559     }
4560
4561     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4562
4563     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4564         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4565
4566         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4567             # skip if nothing changed
4568         } elsif (is_valid_drivename($opt)) {
4569             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4570                 if defined($conf->{$opt});
4571             $conf->{$opt} = $conf->{pending}->{$opt};
4572         } else {
4573             $conf->{$opt} = $conf->{pending}->{$opt};
4574         }
4575
4576         delete $conf->{pending}->{$opt};
4577         PVE::QemuConfig->write_config($vmid, $conf);
4578     }
4579 }
4580
4581 my $safe_num_ne = sub {
4582     my ($a, $b) = @_;
4583
4584     return 0 if !defined($a) && !defined($b);
4585     return 1 if !defined($a);
4586     return 1 if !defined($b);
4587
4588     return $a != $b;
4589 };
4590
4591 my $safe_string_ne = sub {
4592     my ($a, $b) = @_;
4593
4594     return 0 if !defined($a) && !defined($b);
4595     return 1 if !defined($a);
4596     return 1 if !defined($b);
4597
4598     return $a ne $b;
4599 };
4600
4601 sub vmconfig_update_net {
4602     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4603
4604     my $newnet = parse_net($value);
4605
4606     if ($conf->{$opt}) {
4607         my $oldnet = parse_net($conf->{$opt});
4608
4609         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4610             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4611             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4612             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4613
4614             # for non online change, we try to hot-unplug
4615             die "skip\n" if !$hotplug;
4616             vm_deviceunplug($vmid, $conf, $opt);
4617         } else {
4618
4619             die "internal error" if $opt !~ m/net(\d+)/;
4620             my $iface = "tap${vmid}i$1";
4621
4622             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4623                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4624                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4625                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4626                 PVE::Network::tap_unplug($iface);
4627                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4628             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4629                 # Rate can be applied on its own but any change above needs to
4630                 # include the rate in tap_plug since OVS resets everything.
4631                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4632             }
4633
4634             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4635                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4636             }
4637
4638             return 1;
4639         }
4640     }
4641
4642     if ($hotplug) {
4643         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4644     } else {
4645         die "skip\n";
4646     }
4647 }
4648
4649 sub vmconfig_update_disk {
4650     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4651
4652     # fixme: do we need force?
4653
4654     my $drive = parse_drive($opt, $value);
4655
4656     if ($conf->{$opt}) {
4657
4658         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4659
4660             my $media = $drive->{media} || 'disk';
4661             my $oldmedia = $old_drive->{media} || 'disk';
4662             die "unable to change media type\n" if $media ne $oldmedia;
4663
4664             if (!drive_is_cdrom($old_drive)) {
4665
4666                 if ($drive->{file} ne $old_drive->{file}) {
4667
4668                     die "skip\n" if !$hotplug;
4669
4670                     # unplug and register as unused
4671                     vm_deviceunplug($vmid, $conf, $opt);
4672                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4673
4674                 } else {
4675                     # update existing disk
4676
4677                     # skip non hotpluggable value
4678                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4679                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4680                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4681                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4682                         die "skip\n";
4683                     }
4684
4685                     # apply throttle
4686                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4687                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4688                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4689                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4690                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4691                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4692                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4693                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4694                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4695                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4696                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4697                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4698                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4699                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4700                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4701                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4702                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4703                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4704
4705                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4706                                                    ($drive->{mbps} || 0)*1024*1024,
4707                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4708                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4709                                                    $drive->{iops} || 0,
4710                                                    $drive->{iops_rd} || 0,
4711                                                    $drive->{iops_wr} || 0,
4712                                                    ($drive->{mbps_max} || 0)*1024*1024,
4713                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4714                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4715                                                    $drive->{iops_max} || 0,
4716                                                    $drive->{iops_rd_max} || 0,
4717                                                    $drive->{iops_wr_max} || 0,
4718                                                    $drive->{bps_max_length} || 1,
4719                                                    $drive->{bps_rd_max_length} || 1,
4720                                                    $drive->{bps_wr_max_length} || 1,
4721                                                    $drive->{iops_max_length} || 1,
4722                                                    $drive->{iops_rd_max_length} || 1,
4723                                                    $drive->{iops_wr_max_length} || 1);
4724
4725                     }
4726
4727                     return 1;
4728                 }
4729
4730             } else { # cdrom
4731
4732                 if ($drive->{file} eq 'none') {
4733                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4734                     if (drive_is_cloudinit($old_drive)) {
4735                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4736                     }
4737                 } else {
4738                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4739                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4740                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4741                 }
4742
4743                 return 1;
4744             }
4745         }
4746     }
4747
4748     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4749     # hotplug new disks
4750     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4751     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4752 }
4753
4754 sub vm_start {
4755     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4756         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4757
4758     PVE::QemuConfig->lock_config($vmid, sub {
4759         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4760
4761         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4762
4763         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4764
4765         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4766
4767         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4768             vmconfig_apply_pending($vmid, $conf, $storecfg);
4769             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4770         }
4771
4772         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4773
4774         my $defaults = load_defaults();
4775
4776         # set environment variable useful inside network script
4777         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4778
4779         my $local_volumes = {};
4780
4781         if ($targetstorage) {
4782             foreach_drive($conf, sub {
4783                 my ($ds, $drive) = @_;
4784
4785                 return if drive_is_cdrom($drive);
4786
4787                 my $volid = $drive->{file};
4788
4789                 return if !$volid;
4790
4791                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4792
4793                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4794                 return if $scfg->{shared};
4795                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4796             });
4797
4798             my $format = undef;
4799
4800             foreach my $opt (sort keys %$local_volumes) {
4801
4802                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4803                 my $drive = parse_drive($opt, $conf->{$opt});
4804
4805                 #if remote storage is specified, use default format
4806                 if ($targetstorage && $targetstorage ne "1") {
4807                     $storeid = $targetstorage;
4808                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4809                     $format = $defFormat;
4810                 } else {
4811                     #else we use same format than original
4812                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4813                     $format = qemu_img_format($scfg, $volid);
4814                 }
4815
4816                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4817                 my $newdrive = $drive;
4818                 $newdrive->{format} = $format;
4819                 $newdrive->{file} = $newvolid;
4820                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4821                 $local_volumes->{$opt} = $drivestr;
4822                 #pass drive to conf for command line
4823                 $conf->{$opt} = $drivestr;
4824             }
4825         }
4826
4827         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4828
4829         my $migrate_port = 0;
4830         my $migrate_uri;
4831         if ($statefile) {
4832             if ($statefile eq 'tcp') {
4833                 my $localip = "localhost";
4834                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4835                 my $nodename = PVE::INotify::nodename();
4836
4837                 if (!defined($migration_type)) {
4838                     if (defined($datacenterconf->{migration}->{type})) {
4839                         $migration_type = $datacenterconf->{migration}->{type};
4840                     } else {
4841                         $migration_type = 'secure';
4842                     }
4843                 }
4844
4845                 if ($migration_type eq 'insecure') {
4846                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4847                     if ($migrate_network_addr) {
4848                         $localip = $migrate_network_addr;
4849                     } else {
4850                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4851                     }
4852
4853                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4854                 }
4855
4856                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4857                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4858                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4859                 push @$cmd, '-incoming', $migrate_uri;
4860                 push @$cmd, '-S';
4861
4862             } elsif ($statefile eq 'unix') {
4863                 # should be default for secure migrations as a ssh TCP forward
4864                 # tunnel is not deterministic reliable ready and fails regurarly
4865                 # to set up in time, so use UNIX socket forwards
4866                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4867                 unlink $socket_addr;
4868
4869                 $migrate_uri = "unix:$socket_addr";
4870
4871                 push @$cmd, '-incoming', $migrate_uri;
4872                 push @$cmd, '-S';
4873
4874             } else {
4875                 push @$cmd, '-loadstate', $statefile;
4876             }
4877         } elsif ($paused) {
4878             push @$cmd, '-S';
4879         }
4880
4881         # host pci devices
4882         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4883           my $d = parse_hostpci($conf->{"hostpci$i"});
4884           next if !$d;
4885           my $pcidevices = $d->{pciid};
4886           foreach my $pcidevice (@$pcidevices) {
4887                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4888
4889                 my $info = pci_device_info("0000:$pciid");
4890                 die "IOMMU not present\n" if !check_iommu_support();
4891                 die "no pci device info for device '$pciid'\n" if !$info;
4892                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4893                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4894           }
4895         }
4896
4897         PVE::Storage::activate_volumes($storecfg, $vollist);
4898
4899         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4900             my $cmd = [];
4901             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4902             eval  { run_command($cmd); };
4903         }
4904
4905         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4906                                                   : $defaults->{cpuunits};
4907
4908         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4909         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4910
4911         my %properties = (
4912             Slice => 'qemu.slice',
4913             KillMode => 'none',
4914             CPUShares => $cpuunits
4915         );
4916
4917         if (my $cpulimit = $conf->{cpulimit}) {
4918             $properties{CPUQuota} = int($cpulimit * 100);
4919         }
4920         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4921
4922         if ($conf->{hugepages}) {
4923
4924             my $code = sub {
4925                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4926                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4927
4928                 PVE::QemuServer::Memory::hugepages_mount();
4929                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4930
4931                 eval  {
4932                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4933                     run_command($cmd, %run_params);
4934                 };
4935
4936                 if (my $err = $@) {
4937                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4938                     die $err;
4939                 }
4940
4941                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4942             };
4943             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4944
4945         } else {
4946             eval  {
4947                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4948                 run_command($cmd, %run_params);
4949             };
4950         }
4951
4952         if (my $err = $@) {
4953             # deactivate volumes if start fails
4954             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4955             die "start failed: $err";
4956         }
4957
4958         print "migration listens on $migrate_uri\n" if $migrate_uri;
4959
4960         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4961             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4962             warn $@ if $@;
4963         }
4964
4965         #start nbd server for storage migration
4966         if ($targetstorage) {
4967             my $nodename = PVE::INotify::nodename();
4968             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4969             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4970             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4971             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4972
4973             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4974
4975             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4976
4977             foreach my $opt (sort keys %$local_volumes) {
4978                 my $volid = $local_volumes->{$opt};
4979                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4980                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4981                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4982             }
4983         }
4984
4985         if ($migratedfrom) {
4986             eval {
4987                 set_migration_caps($vmid);
4988             };
4989             warn $@ if $@;
4990
4991             if ($spice_port) {
4992                 print "spice listens on port $spice_port\n";
4993                 if ($spice_ticket) {
4994                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4995                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4996                 }
4997             }
4998
4999         } else {
5000             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
5001                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5002                     if $conf->{balloon};
5003             }
5004
5005             foreach my $opt (keys %$conf) {
5006                 next if $opt !~  m/^net\d+$/;
5007                 my $nicconf = parse_net($conf->{$opt});
5008                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5009             }
5010         }
5011
5012         vm_mon_cmd_nocheck($vmid, 'qom-set',
5013                     path => "machine/peripheral/balloon0",
5014                     property => "guest-stats-polling-interval",
5015                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5016
5017     });
5018 }
5019
5020 sub vm_mon_cmd {
5021     my ($vmid, $execute, %params) = @_;
5022
5023     my $cmd = { execute => $execute, arguments => \%params };
5024     vm_qmp_command($vmid, $cmd);
5025 }
5026
5027 sub vm_mon_cmd_nocheck {
5028     my ($vmid, $execute, %params) = @_;
5029
5030     my $cmd = { execute => $execute, arguments => \%params };
5031     vm_qmp_command($vmid, $cmd, 1);
5032 }
5033
5034 sub vm_qmp_command {
5035     my ($vmid, $cmd, $nocheck) = @_;
5036
5037     my $res;
5038
5039     my $timeout;
5040     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5041         $timeout = $cmd->{arguments}->{timeout};
5042         delete $cmd->{arguments}->{timeout};
5043     }
5044
5045     eval {
5046         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5047         my $sname = qmp_socket($vmid);
5048         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5049             my $qmpclient = PVE::QMPClient->new();
5050
5051             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5052         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5053             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5054                 if scalar(%{$cmd->{arguments}});
5055             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5056         } else {
5057             die "unable to open monitor socket\n";
5058         }
5059     };
5060     if (my $err = $@) {
5061         syslog("err", "VM $vmid qmp command failed - $err");
5062         die $err;
5063     }
5064
5065     return $res;
5066 }
5067
5068 sub vm_human_monitor_command {
5069     my ($vmid, $cmdline) = @_;
5070
5071     my $res;
5072
5073     my $cmd = {
5074         execute => 'human-monitor-command',
5075         arguments => { 'command-line' => $cmdline},
5076     };
5077
5078     return vm_qmp_command($vmid, $cmd);
5079 }
5080
5081 sub vm_commandline {
5082     my ($storecfg, $vmid) = @_;
5083
5084     my $conf = PVE::QemuConfig->load_config($vmid);
5085
5086     my $defaults = load_defaults();
5087
5088     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5089
5090     return PVE::Tools::cmd2string($cmd);
5091 }
5092
5093 sub vm_reset {
5094     my ($vmid, $skiplock) = @_;
5095
5096     PVE::QemuConfig->lock_config($vmid, sub {
5097
5098         my $conf = PVE::QemuConfig->load_config($vmid);
5099
5100         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5101
5102         vm_mon_cmd($vmid, "system_reset");
5103     });
5104 }
5105
5106 sub get_vm_volumes {
5107     my ($conf) = @_;
5108
5109     my $vollist = [];
5110     foreach_volid($conf, sub {
5111         my ($volid, $attr) = @_;
5112
5113         return if $volid =~ m|^/|;
5114
5115         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5116         return if !$sid;
5117
5118         push @$vollist, $volid;
5119     });
5120
5121     return $vollist;
5122 }
5123
5124 sub vm_stop_cleanup {
5125     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5126
5127     eval {
5128
5129         if (!$keepActive) {
5130             my $vollist = get_vm_volumes($conf);
5131             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5132         }
5133
5134         foreach my $ext (qw(mon qmp pid vnc qga)) {
5135             unlink "/var/run/qemu-server/${vmid}.$ext";
5136         }
5137
5138         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5139     };
5140     warn $@ if $@; # avoid errors - just warn
5141 }
5142
5143 # Note: use $nockeck to skip tests if VM configuration file exists.
5144 # We need that when migration VMs to other nodes (files already moved)
5145 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5146 sub vm_stop {
5147     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5148
5149     $force = 1 if !defined($force) && !$shutdown;
5150
5151     if ($migratedfrom){
5152         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5153         kill 15, $pid if $pid;
5154         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5155         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5156         return;
5157     }
5158
5159     PVE::QemuConfig->lock_config($vmid, sub {
5160
5161         my $pid = check_running($vmid, $nocheck);
5162         return if !$pid;
5163
5164         my $conf;
5165         if (!$nocheck) {
5166             $conf = PVE::QemuConfig->load_config($vmid);
5167             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5168             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5169                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5170                 $timeout = $opts->{down} if $opts->{down};
5171             }
5172         }
5173
5174         $timeout = 60 if !defined($timeout);
5175
5176         eval {
5177             if ($shutdown) {
5178                 if (defined($conf) && $conf->{agent}) {
5179                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5180                 } else {
5181                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5182                 }
5183             } else {
5184                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5185             }
5186         };
5187         my $err = $@;
5188
5189         if (!$err) {
5190             my $count = 0;
5191             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5192                 $count++;
5193                 sleep 1;
5194             }
5195
5196             if ($count >= $timeout) {
5197                 if ($force) {
5198                     warn "VM still running - terminating now with SIGTERM\n";
5199                     kill 15, $pid;
5200                 } else {
5201                     die "VM quit/powerdown failed - got timeout\n";
5202                 }
5203             } else {
5204                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5205                 return;
5206             }
5207         } else {
5208             if ($force) {
5209                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5210                 kill 15, $pid;
5211             } else {
5212                 die "VM quit/powerdown failed\n";
5213             }
5214         }
5215
5216         # wait again
5217         $timeout = 10;
5218
5219         my $count = 0;
5220         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5221             $count++;
5222             sleep 1;
5223         }
5224
5225         if ($count >= $timeout) {
5226             warn "VM still running - terminating now with SIGKILL\n";
5227             kill 9, $pid;
5228             sleep 1;
5229         }
5230
5231         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5232    });
5233 }
5234
5235 sub vm_suspend {
5236     my ($vmid, $skiplock) = @_;
5237
5238     PVE::QemuConfig->lock_config($vmid, sub {
5239
5240         my $conf = PVE::QemuConfig->load_config($vmid);
5241
5242         PVE::QemuConfig->check_lock($conf)
5243             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5244
5245         vm_mon_cmd($vmid, "stop");
5246     });
5247 }
5248
5249 sub vm_resume {
5250     my ($vmid, $skiplock, $nocheck) = @_;
5251
5252     PVE::QemuConfig->lock_config($vmid, sub {
5253
5254         if (!$nocheck) {
5255
5256             my $conf = PVE::QemuConfig->load_config($vmid);
5257
5258             PVE::QemuConfig->check_lock($conf)
5259                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5260
5261             vm_mon_cmd($vmid, "cont");
5262
5263         } else {
5264             vm_mon_cmd_nocheck($vmid, "cont");
5265         }
5266     });
5267 }
5268
5269 sub vm_sendkey {
5270     my ($vmid, $skiplock, $key) = @_;
5271
5272     PVE::QemuConfig->lock_config($vmid, sub {
5273
5274         my $conf = PVE::QemuConfig->load_config($vmid);
5275
5276         # there is no qmp command, so we use the human monitor command
5277         vm_human_monitor_command($vmid, "sendkey $key");
5278     });
5279 }
5280
5281 sub vm_destroy {
5282     my ($storecfg, $vmid, $skiplock) = @_;
5283
5284     PVE::QemuConfig->lock_config($vmid, sub {
5285
5286         my $conf = PVE::QemuConfig->load_config($vmid);
5287
5288         if (!check_running($vmid)) {
5289             destroy_vm($storecfg, $vmid, undef, $skiplock);
5290         } else {
5291             die "VM $vmid is running - destroy failed\n";
5292         }
5293     });
5294 }
5295
5296 # pci helpers
5297
5298 sub file_write {
5299     my ($filename, $buf) = @_;
5300
5301     my $fh = IO::File->new($filename, "w");
5302     return undef if !$fh;
5303
5304     my $res = print $fh $buf;
5305
5306     $fh->close();
5307
5308     return $res;
5309 }
5310
5311 sub pci_device_info {
5312     my ($name) = @_;
5313
5314     my $res;
5315
5316     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5317     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5318
5319     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5320     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5321
5322     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5323     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5324
5325     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5326     return undef if !defined($product) || $product !~ s/^0x//;
5327
5328     $res = {
5329         name => $name,
5330         vendor => $vendor,
5331         product => $product,
5332         domain => $domain,
5333         bus => $bus,
5334         slot => $slot,
5335         func => $func,
5336         irq => $irq,
5337         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5338     };
5339
5340     return $res;
5341 }
5342
5343 sub pci_dev_reset {
5344     my ($dev) = @_;
5345
5346     my $name = $dev->{name};
5347
5348     my $fn = "$pcisysfs/devices/$name/reset";
5349
5350     return file_write($fn, "1");
5351 }
5352
5353 sub pci_dev_bind_to_vfio {
5354     my ($dev) = @_;
5355
5356     my $name = $dev->{name};
5357
5358     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5359
5360     if (!-d $vfio_basedir) {
5361         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5362     }
5363     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5364
5365     my $testdir = "$vfio_basedir/$name";
5366     return 1 if -d $testdir;
5367
5368     my $data = "$dev->{vendor} $dev->{product}";
5369     return undef if !file_write("$vfio_basedir/new_id", $data);
5370
5371     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5372     if (!file_write($fn, $name)) {
5373         return undef if -f $fn;
5374     }
5375
5376     $fn = "$vfio_basedir/bind";
5377     if (! -d $testdir) {
5378         return undef if !file_write($fn, $name);
5379     }
5380
5381     return -d $testdir;
5382 }
5383
5384 sub pci_dev_group_bind_to_vfio {
5385     my ($pciid) = @_;
5386
5387     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5388
5389     if (!-d $vfio_basedir) {
5390         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5391     }
5392     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5393
5394     # get IOMMU group devices
5395     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5396       my @devs = grep /^0000:/, readdir($D);
5397     closedir($D);
5398
5399     foreach my $pciid (@devs) {
5400         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5401
5402         # pci bridges, switches or root ports are not supported
5403         # they have a pci_bus subdirectory so skip them
5404         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5405
5406         my $info = pci_device_info($1);
5407         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5408     }
5409
5410     return 1;
5411 }
5412
5413 # vzdump restore implementaion
5414
5415 sub tar_archive_read_firstfile {
5416     my $archive = shift;
5417
5418     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5419
5420     # try to detect archive type first
5421     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5422         die "unable to open file '$archive'\n";
5423     my $firstfile = <$fh>;
5424     kill 15, $pid;
5425     close $fh;
5426
5427     die "ERROR: archive contaions no data\n" if !$firstfile;
5428     chomp $firstfile;
5429
5430     return $firstfile;
5431 }
5432
5433 sub tar_restore_cleanup {
5434     my ($storecfg, $statfile) = @_;
5435
5436     print STDERR "starting cleanup\n";
5437
5438     if (my $fd = IO::File->new($statfile, "r")) {
5439         while (defined(my $line = <$fd>)) {
5440             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5441                 my $volid = $2;
5442                 eval {
5443                     if ($volid =~ m|^/|) {
5444                         unlink $volid || die 'unlink failed\n';
5445                     } else {
5446                         PVE::Storage::vdisk_free($storecfg, $volid);
5447                     }
5448                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5449                 };
5450                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5451             } else {
5452                 print STDERR "unable to parse line in statfile - $line";
5453             }
5454         }
5455         $fd->close();
5456     }
5457 }
5458
5459 sub restore_archive {
5460     my ($archive, $vmid, $user, $opts) = @_;
5461
5462     my $format = $opts->{format};
5463     my $comp;
5464
5465     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5466         $format = 'tar' if !$format;
5467         $comp = 'gzip';
5468     } elsif ($archive =~ m/\.tar$/) {
5469         $format = 'tar' if !$format;
5470     } elsif ($archive =~ m/.tar.lzo$/) {
5471         $format = 'tar' if !$format;
5472         $comp = 'lzop';
5473     } elsif ($archive =~ m/\.vma$/) {
5474         $format = 'vma' if !$format;
5475     } elsif ($archive =~ m/\.vma\.gz$/) {
5476         $format = 'vma' if !$format;
5477         $comp = 'gzip';
5478     } elsif ($archive =~ m/\.vma\.lzo$/) {
5479         $format = 'vma' if !$format;
5480         $comp = 'lzop';
5481     } else {
5482         $format = 'vma' if !$format; # default
5483     }
5484
5485     # try to detect archive format
5486     if ($format eq 'tar') {
5487         return restore_tar_archive($archive, $vmid, $user, $opts);
5488     } else {
5489         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5490     }
5491 }
5492
5493 sub restore_update_config_line {
5494     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5495
5496     return if $line =~ m/^\#qmdump\#/;
5497     return if $line =~ m/^\#vzdump\#/;
5498     return if $line =~ m/^lock:/;
5499     return if $line =~ m/^unused\d+:/;
5500     return if $line =~ m/^parent:/;
5501     return if $line =~ m/^template:/; # restored VM is never a template
5502
5503     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5504     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5505         # try to convert old 1.X settings
5506         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5507         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5508             my ($model, $macaddr) = split(/\=/, $devconfig);
5509             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5510             my $net = {
5511                 model => $model,
5512                 bridge => "vmbr$ind",
5513                 macaddr => $macaddr,
5514             };
5515             my $netstr = print_net($net);
5516
5517             print $outfd "net$cookie->{netcount}: $netstr\n";
5518             $cookie->{netcount}++;
5519         }
5520     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5521         my ($id, $netstr) = ($1, $2);
5522         my $net = parse_net($netstr);
5523         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5524         $netstr = print_net($net);
5525         print $outfd "$id: $netstr\n";
5526     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5527         my $virtdev = $1;
5528         my $value = $3;
5529         my $di = parse_drive($virtdev, $value);
5530         if (defined($di->{backup}) && !$di->{backup}) {
5531             print $outfd "#$line";
5532         } elsif ($map->{$virtdev}) {
5533             delete $di->{format}; # format can change on restore
5534             $di->{file} = $map->{$virtdev};
5535             $value = print_drive($vmid, $di);
5536             print $outfd "$virtdev: $value\n";
5537         } else {
5538             print $outfd $line;
5539         }
5540     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5541         my ($uuid, $uuid_str);
5542         UUID::generate($uuid);
5543         UUID::unparse($uuid, $uuid_str);
5544         my $smbios1 = parse_smbios1($2);
5545         $smbios1->{uuid} = $uuid_str;
5546         print $outfd $1.print_smbios1($smbios1)."\n";
5547     } else {
5548         print $outfd $line;
5549     }
5550 }
5551
5552 sub scan_volids {
5553     my ($cfg, $vmid) = @_;
5554
5555     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5556
5557     my $volid_hash = {};
5558     foreach my $storeid (keys %$info) {
5559         foreach my $item (@{$info->{$storeid}}) {
5560             next if !($item->{volid} && $item->{size});
5561             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5562             $volid_hash->{$item->{volid}} = $item;
5563         }
5564     }
5565
5566     return $volid_hash;
5567 }
5568
5569 sub is_volume_in_use {
5570     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5571
5572     my $path = PVE::Storage::path($storecfg, $volid);
5573
5574     my $scan_config = sub {
5575         my ($cref, $snapname) = @_;
5576
5577         foreach my $key (keys %$cref) {
5578             my $value = $cref->{$key};
5579             if (is_valid_drivename($key)) {
5580                 next if $skip_drive && $key eq $skip_drive;
5581                 my $drive = parse_drive($key, $value);
5582                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5583                 return 1 if $volid eq $drive->{file};
5584                 if ($drive->{file} =~ m!^/!) {
5585                     return 1 if $drive->{file} eq $path;
5586                 } else {
5587                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5588                     next if !$storeid;
5589                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5590                     next if !$scfg;
5591                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5592                 }
5593             }
5594         }
5595
5596         return 0;
5597     };
5598
5599     return 1 if &$scan_config($conf);
5600
5601     undef $skip_drive;
5602
5603     foreach my $snapname (keys %{$conf->{snapshots}}) {
5604         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5605     }
5606
5607     return 0;
5608 }
5609
5610 sub update_disksize {
5611     my ($vmid, $conf, $volid_hash) = @_;
5612
5613     my $changes;
5614
5615     # used and unused disks
5616     my $referenced = {};
5617
5618     # Note: it is allowed to define multiple storages with same path (alias), so
5619     # we need to check both 'volid' and real 'path' (two different volid can point
5620     # to the same path).
5621
5622     my $referencedpath = {};
5623
5624     # update size info
5625     foreach my $opt (keys %$conf) {
5626         if (is_valid_drivename($opt)) {
5627             my $drive = parse_drive($opt, $conf->{$opt});
5628             my $volid = $drive->{file};
5629             next if !$volid;
5630
5631             $referenced->{$volid} = 1;
5632             if ($volid_hash->{$volid} &&
5633                 (my $path = $volid_hash->{$volid}->{path})) {
5634                 $referencedpath->{$path} = 1;
5635             }
5636
5637             next if drive_is_cdrom($drive);
5638             next if !$volid_hash->{$volid};
5639
5640             $drive->{size} = $volid_hash->{$volid}->{size};
5641             my $new = print_drive($vmid, $drive);
5642             if ($new ne $conf->{$opt}) {
5643                 $changes = 1;
5644                 $conf->{$opt} = $new;
5645             }
5646         }
5647     }
5648
5649     # remove 'unusedX' entry if volume is used
5650     foreach my $opt (keys %$conf) {
5651         next if $opt !~ m/^unused\d+$/;
5652         my $volid = $conf->{$opt};
5653         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5654         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5655             $changes = 1;
5656             delete $conf->{$opt};
5657         }
5658
5659         $referenced->{$volid} = 1;
5660         $referencedpath->{$path} = 1 if $path;
5661     }
5662
5663     foreach my $volid (sort keys %$volid_hash) {
5664         next if $volid =~ m/vm-$vmid-state-/;
5665         next if $referenced->{$volid};
5666         my $path = $volid_hash->{$volid}->{path};
5667         next if !$path; # just to be sure
5668         next if $referencedpath->{$path};
5669         $changes = 1;
5670         PVE::QemuConfig->add_unused_volume($conf, $volid);
5671         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5672     }
5673
5674     return $changes;
5675 }
5676
5677 sub rescan {
5678     my ($vmid, $nolock) = @_;
5679
5680     my $cfg = PVE::Storage::config();
5681
5682     my $volid_hash = scan_volids($cfg, $vmid);
5683
5684     my $updatefn =  sub {
5685         my ($vmid) = @_;
5686
5687         my $conf = PVE::QemuConfig->load_config($vmid);
5688
5689         PVE::QemuConfig->check_lock($conf);
5690
5691         my $vm_volids = {};
5692         foreach my $volid (keys %$volid_hash) {
5693             my $info = $volid_hash->{$volid};
5694             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5695         }
5696
5697         my $changes = update_disksize($vmid, $conf, $vm_volids);
5698
5699         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5700     };
5701
5702     if (defined($vmid)) {
5703         if ($nolock) {
5704             &$updatefn($vmid);
5705         } else {
5706             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5707         }
5708     } else {
5709         my $vmlist = config_list();
5710         foreach my $vmid (keys %$vmlist) {
5711             if ($nolock) {
5712                 &$updatefn($vmid);
5713             } else {
5714                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5715             }
5716         }
5717     }
5718 }
5719
5720 sub restore_vma_archive {
5721     my ($archive, $vmid, $user, $opts, $comp) = @_;
5722
5723     my $input = $archive eq '-' ? "<&STDIN" : undef;
5724     my $readfrom = $archive;
5725
5726     my $uncomp = '';
5727     if ($comp) {
5728         $readfrom = '-';
5729         my $qarchive = PVE::Tools::shellquote($archive);
5730         if ($comp eq 'gzip') {
5731             $uncomp = "zcat $qarchive|";
5732         } elsif ($comp eq 'lzop') {
5733             $uncomp = "lzop -d -c $qarchive|";
5734         } else {
5735             die "unknown compression method '$comp'\n";
5736         }
5737
5738     }
5739
5740     my $tmpdir = "/var/tmp/vzdumptmp$$";
5741     rmtree $tmpdir;
5742
5743     # disable interrupts (always do cleanups)
5744     local $SIG{INT} =
5745         local $SIG{TERM} =
5746         local $SIG{QUIT} =
5747         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5748
5749     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5750     POSIX::mkfifo($mapfifo, 0600);
5751     my $fifofh;
5752
5753     my $openfifo = sub {
5754         open($fifofh, '>', $mapfifo) || die $!;
5755     };
5756
5757     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5758
5759     my $oldtimeout;
5760     my $timeout = 5;
5761
5762     my $devinfo = {};
5763
5764     my $rpcenv = PVE::RPCEnvironment::get();
5765
5766     my $conffile = PVE::QemuConfig->config_file($vmid);
5767     my $tmpfn = "$conffile.$$.tmp";
5768
5769     # Note: $oldconf is undef if VM does not exists
5770     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5771     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5772
5773     my $print_devmap = sub {
5774         my $virtdev_hash = {};
5775
5776         my $cfgfn = "$tmpdir/qemu-server.conf";
5777
5778         # we can read the config - that is already extracted
5779         my $fh = IO::File->new($cfgfn, "r") ||
5780             "unable to read qemu-server.conf - $!\n";
5781
5782         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5783         if (-f $fwcfgfn) {
5784             my $pve_firewall_dir = '/etc/pve/firewall';
5785             mkdir $pve_firewall_dir; # make sure the dir exists
5786             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5787         }
5788
5789         while (defined(my $line = <$fh>)) {
5790             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5791                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5792                 die "archive does not contain data for drive '$virtdev'\n"
5793                     if !$devinfo->{$devname};
5794                 if (defined($opts->{storage})) {
5795                     $storeid = $opts->{storage} || 'local';
5796                 } elsif (!$storeid) {
5797                     $storeid = 'local';
5798                 }
5799                 $format = 'raw' if !$format;
5800                 $devinfo->{$devname}->{devname} = $devname;
5801                 $devinfo->{$devname}->{virtdev} = $virtdev;
5802                 $devinfo->{$devname}->{format} = $format;
5803                 $devinfo->{$devname}->{storeid} = $storeid;
5804
5805                 # check permission on storage
5806                 my $pool = $opts->{pool}; # todo: do we need that?
5807                 if ($user ne 'root@pam') {
5808                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5809                 }
5810
5811                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5812             }
5813         }
5814
5815         foreach my $devname (keys %$devinfo) {
5816             die "found no device mapping information for device '$devname'\n"
5817                 if !$devinfo->{$devname}->{virtdev};
5818         }
5819
5820         my $cfg = PVE::Storage::config();
5821
5822         # create empty/temp config
5823         if ($oldconf) {
5824             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5825             foreach_drive($oldconf, sub {
5826                 my ($ds, $drive) = @_;
5827
5828                 return if drive_is_cdrom($drive);
5829
5830                 my $volid = $drive->{file};
5831
5832                 return if !$volid || $volid =~ m|^/|;
5833
5834                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5835                 return if !$path || !$owner || ($owner != $vmid);
5836
5837                 # Note: only delete disk we want to restore
5838                 # other volumes will become unused
5839                 if ($virtdev_hash->{$ds}) {
5840                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5841                     if (my $err = $@) {
5842                         warn $err;
5843                     }
5844                 }
5845             });
5846
5847             # delete vmstate files
5848             # since after the restore we have no snapshots anymore
5849             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5850                 my $snap = $oldconf->{snapshots}->{$snapname};
5851                 if ($snap->{vmstate}) {
5852                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5853                     if (my $err = $@) {
5854                         warn $err;
5855                     }
5856                 }
5857             }
5858         }
5859
5860         my $map = {};
5861         foreach my $virtdev (sort keys %$virtdev_hash) {
5862             my $d = $virtdev_hash->{$virtdev};
5863             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5864             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5865
5866             # test if requested format is supported
5867             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5868             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5869             $d->{format} = $defFormat if !$supported;
5870
5871             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5872                                                   $d->{format}, undef, $alloc_size);
5873             print STDERR "new volume ID is '$volid'\n";
5874             $d->{volid} = $volid;
5875             my $path = PVE::Storage::path($cfg, $volid);
5876
5877             PVE::Storage::activate_volumes($cfg,[$volid]);
5878
5879             my $write_zeros = 1;
5880             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5881                 $write_zeros = 0;
5882             }
5883
5884             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5885
5886             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5887             $map->{$virtdev} = $volid;
5888         }
5889
5890         $fh->seek(0, 0) || die "seek failed - $!\n";
5891
5892         my $outfd = new IO::File ($tmpfn, "w") ||
5893             die "unable to write config for VM $vmid\n";
5894
5895         my $cookie = { netcount => 0 };
5896         while (defined(my $line = <$fh>)) {
5897             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5898         }
5899
5900         $fh->close();
5901         $outfd->close();
5902     };
5903
5904     eval {
5905         # enable interrupts
5906         local $SIG{INT} =
5907             local $SIG{TERM} =
5908             local $SIG{QUIT} =
5909             local $SIG{HUP} =
5910             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5911         local $SIG{ALRM} = sub { die "got timeout\n"; };
5912
5913         $oldtimeout = alarm($timeout);
5914
5915         my $parser = sub {
5916             my $line = shift;
5917
5918             print "$line\n";
5919
5920             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5921                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5922                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5923             } elsif ($line =~ m/^CTIME: /) {
5924                 # we correctly received the vma config, so we can disable
5925                 # the timeout now for disk allocation (set to 10 minutes, so
5926                 # that we always timeout if something goes wrong)
5927                 alarm(600);
5928                 &$print_devmap();
5929                 print $fifofh "done\n";
5930                 my $tmp = $oldtimeout || 0;
5931                 $oldtimeout = undef;
5932                 alarm($tmp);
5933                 close($fifofh);
5934             }
5935         };
5936
5937         print "restore vma archive: $cmd\n";
5938         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5939     };
5940     my $err = $@;
5941
5942     alarm($oldtimeout) if $oldtimeout;
5943
5944     my $vollist = [];
5945     foreach my $devname (keys %$devinfo) {
5946         my $volid = $devinfo->{$devname}->{volid};
5947         push @$vollist, $volid if $volid;
5948     }
5949
5950     my $cfg = PVE::Storage::config();
5951     PVE::Storage::deactivate_volumes($cfg, $vollist);
5952
5953     unlink $mapfifo;
5954
5955     if ($err) {
5956         rmtree $tmpdir;
5957         unlink $tmpfn;
5958
5959         foreach my $devname (keys %$devinfo) {
5960             my $volid = $devinfo->{$devname}->{volid};
5961             next if !$volid;
5962             eval {
5963                 if ($volid =~ m|^/|) {
5964                     unlink $volid || die 'unlink failed\n';
5965                 } else {
5966                     PVE::Storage::vdisk_free($cfg, $volid);
5967                 }
5968                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5969             };
5970             print STDERR "unable to cleanup '$volid' - $@" if $@;
5971         }
5972         die $err;
5973     }
5974
5975     rmtree $tmpdir;
5976
5977     rename($tmpfn, $conffile) ||
5978         die "unable to commit configuration file '$conffile'\n";
5979
5980     PVE::Cluster::cfs_update(); # make sure we read new file
5981
5982     eval { rescan($vmid, 1); };
5983     warn $@ if $@;
5984 }
5985
5986 sub restore_tar_archive {
5987     my ($archive, $vmid, $user, $opts) = @_;
5988
5989     if ($archive ne '-') {
5990         my $firstfile = tar_archive_read_firstfile($archive);
5991         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5992             if $firstfile ne 'qemu-server.conf';
5993     }
5994
5995     my $storecfg = PVE::Storage::config();
5996
5997     # destroy existing data - keep empty config
5998     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5999     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6000
6001     my $tocmd = "/usr/lib/qemu-server/qmextract";
6002
6003     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6004     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6005     $tocmd .= ' --prealloc' if $opts->{prealloc};
6006     $tocmd .= ' --info' if $opts->{info};
6007
6008     # tar option "xf" does not autodetect compression when read from STDIN,
6009     # so we pipe to zcat
6010     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6011         PVE::Tools::shellquote("--to-command=$tocmd");
6012
6013     my $tmpdir = "/var/tmp/vzdumptmp$$";
6014     mkpath $tmpdir;
6015
6016     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6017     local $ENV{VZDUMP_VMID} = $vmid;
6018     local $ENV{VZDUMP_USER} = $user;
6019
6020     my $conffile = PVE::QemuConfig->config_file($vmid);
6021     my $tmpfn = "$conffile.$$.tmp";
6022
6023     # disable interrupts (always do cleanups)
6024     local $SIG{INT} =
6025         local $SIG{TERM} =
6026         local $SIG{QUIT} =
6027         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6028
6029     eval {
6030         # enable interrupts
6031         local $SIG{INT} =
6032             local $SIG{TERM} =
6033             local $SIG{QUIT} =
6034             local $SIG{HUP} =
6035             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6036
6037         if ($archive eq '-') {
6038             print "extracting archive from STDIN\n";
6039             run_command($cmd, input => "<&STDIN");
6040         } else {
6041             print "extracting archive '$archive'\n";
6042             run_command($cmd);
6043         }
6044
6045         return if $opts->{info};
6046
6047         # read new mapping
6048         my $map = {};
6049         my $statfile = "$tmpdir/qmrestore.stat";
6050         if (my $fd = IO::File->new($statfile, "r")) {
6051             while (defined (my $line = <$fd>)) {
6052                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6053                     $map->{$1} = $2 if $1;
6054                 } else {
6055                     print STDERR "unable to parse line in statfile - $line\n";
6056                 }
6057             }
6058             $fd->close();
6059         }
6060
6061         my $confsrc = "$tmpdir/qemu-server.conf";
6062
6063         my $srcfd = new IO::File($confsrc, "r") ||
6064             die "unable to open file '$confsrc'\n";
6065
6066         my $outfd = new IO::File ($tmpfn, "w") ||
6067             die "unable to write config for VM $vmid\n";
6068
6069         my $cookie = { netcount => 0 };
6070         while (defined (my $line = <$srcfd>)) {
6071             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6072         }
6073
6074         $srcfd->close();
6075         $outfd->close();
6076     };
6077     my $err = $@;
6078
6079     if ($err) {
6080
6081         unlink $tmpfn;
6082
6083         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6084
6085         die $err;
6086     }
6087
6088     rmtree $tmpdir;
6089
6090     rename $tmpfn, $conffile ||
6091         die "unable to commit configuration file '$conffile'\n";
6092
6093     PVE::Cluster::cfs_update(); # make sure we read new file
6094
6095     eval { rescan($vmid, 1); };
6096     warn $@ if $@;
6097 };
6098
6099 sub foreach_storage_used_by_vm {
6100     my ($conf, $func) = @_;
6101
6102     my $sidhash = {};
6103
6104     foreach_drive($conf, sub {
6105         my ($ds, $drive) = @_;
6106         return if drive_is_cdrom($drive);
6107
6108         my $volid = $drive->{file};
6109
6110         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6111         $sidhash->{$sid} = $sid if $sid;
6112     });
6113
6114     foreach my $sid (sort keys %$sidhash) {
6115         &$func($sid);
6116     }
6117 }
6118
6119 sub do_snapshots_with_qemu {
6120     my ($storecfg, $volid) = @_;
6121
6122     my $storage_name = PVE::Storage::parse_volume_id($volid);
6123
6124     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6125         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6126         return 1;
6127     }
6128
6129     if ($volid =~ m/\.(qcow2|qed)$/){
6130         return 1;
6131     }
6132
6133     return undef;
6134 }
6135
6136 sub qga_check_running {
6137     my ($vmid) = @_;
6138
6139     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6140     if ($@) {
6141         warn "Qemu Guest Agent is not running - $@";
6142         return 0;
6143     }
6144     return 1;
6145 }
6146
6147 sub template_create {
6148     my ($vmid, $conf, $disk) = @_;
6149
6150     my $storecfg = PVE::Storage::config();
6151
6152     foreach_drive($conf, sub {
6153         my ($ds, $drive) = @_;
6154
6155         return if drive_is_cdrom($drive);
6156         return if $disk && $ds ne $disk;
6157
6158         my $volid = $drive->{file};
6159         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6160
6161         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6162         $drive->{file} = $voliddst;
6163         $conf->{$ds} = print_drive($vmid, $drive);
6164         PVE::QemuConfig->write_config($vmid, $conf);
6165     });
6166 }
6167
6168 sub qemu_img_convert {
6169     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6170
6171     my $storecfg = PVE::Storage::config();
6172     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6173     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6174
6175     if ($src_storeid && $dst_storeid) {
6176
6177         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6178
6179         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6180         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6181
6182         my $src_format = qemu_img_format($src_scfg, $src_volname);
6183         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6184
6185         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6186         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6187
6188         my $cmd = [];
6189         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6190         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6191         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6192         if ($is_zero_initialized) {
6193             push @$cmd, "zeroinit:$dst_path";
6194         } else {
6195             push @$cmd, $dst_path;
6196         }
6197
6198         my $parser = sub {
6199             my $line = shift;
6200             if($line =~ m/\((\S+)\/100\%\)/){
6201                 my $percent = $1;
6202                 my $transferred = int($size * $percent / 100);
6203                 my $remaining = $size - $transferred;
6204
6205                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6206             }
6207
6208         };
6209
6210         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6211         my $err = $@;
6212         die "copy failed: $err" if $err;
6213     }
6214 }
6215
6216 sub qemu_img_format {
6217     my ($scfg, $volname) = @_;
6218
6219     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6220         return $1;
6221     } else {
6222         return "raw";
6223     }
6224 }
6225
6226 sub qemu_drive_mirror {
6227     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6228
6229     $jobs = {} if !$jobs;
6230
6231     my $qemu_target;
6232     my $format;
6233     $jobs->{"drive-$drive"} = {};
6234
6235     if ($dst_volid =~ /^nbd:/) {
6236         $qemu_target = $dst_volid;
6237         $format = "nbd";
6238     } else {
6239         my $storecfg = PVE::Storage::config();
6240         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6241
6242         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6243
6244         $format = qemu_img_format($dst_scfg, $dst_volname);
6245
6246         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6247
6248         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6249     }
6250
6251     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6252     $opts->{format} = $format if $format;
6253
6254     print "drive mirror is starting for drive-$drive\n";
6255
6256     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6257
6258     if (my $err = $@) {
6259         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6260         die "mirroring error: $err";
6261     }
6262
6263     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6264 }
6265
6266 sub qemu_drive_mirror_monitor {
6267     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6268
6269     eval {
6270         my $err_complete = 0;
6271
6272         while (1) {
6273             die "storage migration timed out\n" if $err_complete > 300;
6274
6275             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6276
6277             my $running_mirror_jobs = {};
6278             foreach my $stat (@$stats) {
6279                 next if $stat->{type} ne 'mirror';
6280                 $running_mirror_jobs->{$stat->{device}} = $stat;
6281             }
6282
6283             my $readycounter = 0;
6284
6285             foreach my $job (keys %$jobs) {
6286
6287                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6288                     print "$job : finished\n";
6289                     delete $jobs->{$job};
6290                     next;
6291                 }
6292
6293                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6294
6295                 my $busy = $running_mirror_jobs->{$job}->{busy};
6296                 my $ready = $running_mirror_jobs->{$job}->{ready};
6297                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6298                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6299                     my $remaining = $total - $transferred;
6300                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6301
6302                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6303                 }
6304
6305                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6306             }
6307
6308             last if scalar(keys %$jobs) == 0;
6309
6310             if ($readycounter == scalar(keys %$jobs)) {
6311                 print "all mirroring jobs are ready \n";
6312                 last if $skipcomplete; #do the complete later
6313
6314                 if ($vmiddst && $vmiddst != $vmid) {
6315                     my $agent_running = $qga && qga_check_running($vmid);
6316                     if ($agent_running) {
6317                         print "freeze filesystem\n";
6318                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6319                     } else {
6320                         print "suspend vm\n";
6321                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6322                     }
6323
6324                     # if we clone a disk for a new target vm, we don't switch the disk
6325                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6326
6327                     if ($agent_running) {
6328                         print "unfreeze filesystem\n";
6329                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6330                     } else {
6331                         print "resume vm\n";
6332                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6333                     }
6334
6335                     last;
6336                 } else {
6337
6338                     foreach my $job (keys %$jobs) {
6339                         # try to switch the disk if source and destination are on the same guest
6340                         print "$job: Completing block job...\n";
6341
6342                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6343                         if ($@ =~ m/cannot be completed/) {
6344                             print "$job: Block job cannot be completed, try again.\n";
6345                             $err_complete++;
6346                         }else {
6347                             print "$job: Completed successfully.\n";
6348                             $jobs->{$job}->{complete} = 1;
6349                         }
6350                     }
6351                 }
6352             }
6353             sleep 1;
6354         }
6355     };
6356     my $err = $@;
6357
6358     if ($err) {
6359         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6360         die "mirroring error: $err";
6361     }
6362
6363 }
6364
6365 sub qemu_blockjobs_cancel {
6366     my ($vmid, $jobs) = @_;
6367
6368     foreach my $job (keys %$jobs) {
6369         print "$job: Cancelling block job\n";
6370         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6371         $jobs->{$job}->{cancel} = 1;
6372     }
6373
6374     while (1) {
6375         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6376
6377         my $running_jobs = {};
6378         foreach my $stat (@$stats) {
6379             $running_jobs->{$stat->{device}} = $stat;
6380         }
6381
6382         foreach my $job (keys %$jobs) {
6383
6384             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6385                 print "$job: Done.\n";
6386                 delete $jobs->{$job};
6387             }
6388         }
6389
6390         last if scalar(keys %$jobs) == 0;
6391
6392         sleep 1;
6393     }
6394 }
6395
6396 sub clone_disk {
6397     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6398         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6399
6400     my $newvolid;
6401
6402     if (!$full) {
6403         print "create linked clone of drive $drivename ($drive->{file})\n";
6404         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6405         push @$newvollist, $newvolid;
6406     } else {
6407
6408         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6409         $storeid = $storage if $storage;
6410
6411         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6412         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6413
6414         print "create full clone of drive $drivename ($drive->{file})\n";
6415         my $name = undef;
6416         if (drive_is_cloudinit($drive)) {
6417             $name = "vm-$newvmid-cloudinit";
6418             # cloudinit only supports raw and qcow2 atm:
6419             if ($dst_format eq 'qcow2') {
6420                 $name .= '.qcow2';
6421             } elsif ($dst_format ne 'raw') {
6422                 die "clone: unhandled format for cloudinit image\n";
6423             }
6424         }
6425         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6426         push @$newvollist, $newvolid;
6427
6428         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6429
6430         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6431         if (!$running || $snapname) {
6432             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6433         } else {
6434
6435             my $kvmver = get_running_qemu_version ($vmid);
6436             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6437                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6438                     if $drive->{iothread};
6439             }
6440
6441             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6442         }
6443     }
6444
6445     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6446
6447     my $disk = $drive;
6448     $disk->{format} = undef;
6449     $disk->{file} = $newvolid;
6450     $disk->{size} = $size;
6451
6452     return $disk;
6453 }
6454
6455 # this only works if VM is running
6456 sub get_current_qemu_machine {
6457     my ($vmid) = @_;
6458
6459     my $cmd = { execute => 'query-machines', arguments => {} };
6460     my $res = vm_qmp_command($vmid, $cmd);
6461
6462     my ($current, $default);
6463     foreach my $e (@$res) {
6464         $default = $e->{name} if $e->{'is-default'};
6465         $current = $e->{name} if $e->{'is-current'};
6466     }
6467
6468     # fallback to the default machine if current is not supported by qemu
6469     return $current || $default || 'pc';
6470 }
6471
6472 sub get_running_qemu_version {
6473     my ($vmid) = @_;
6474     my $cmd = { execute => 'query-version', arguments => {} };
6475     my $res = vm_qmp_command($vmid, $cmd);
6476     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6477 }
6478
6479 sub qemu_machine_feature_enabled {
6480     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6481
6482     my $current_major;
6483     my $current_minor;
6484
6485     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6486
6487         $current_major = $3;
6488         $current_minor = $4;
6489
6490     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6491
6492         $current_major = $1;
6493         $current_minor = $2;
6494     }
6495
6496     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6497
6498
6499 }
6500
6501 sub qemu_machine_pxe {
6502     my ($vmid, $conf, $machine) = @_;
6503
6504     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6505
6506     foreach my $opt (keys %$conf) {
6507         next if $opt !~ m/^net(\d+)$/;
6508         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6509         next if !$net;
6510         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6511         return $machine.".pxe" if $romfile =~ m/pxe/;
6512         last;
6513     }
6514
6515     return $machine;
6516 }
6517
6518 sub qemu_use_old_bios_files {
6519     my ($machine_type) = @_;
6520
6521     return if !$machine_type;
6522
6523     my $use_old_bios_files = undef;
6524
6525     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6526         $machine_type = $1;
6527         $use_old_bios_files = 1;
6528     } else {
6529         my $kvmver = kvm_user_version();
6530         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6531         # load new efi bios files on migration. So this hack is required to allow
6532         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6533         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6534         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6535     }
6536
6537     return ($use_old_bios_files, $machine_type);
6538 }
6539
6540 sub create_efidisk {
6541     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6542
6543     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6544
6545     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6546     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6547     PVE::Storage::activate_volumes($storecfg, [$volid]);
6548
6549     my $path = PVE::Storage::path($storecfg, $volid);
6550     eval {
6551         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6552     };
6553     die "Copying EFI vars image failed: $@" if $@;
6554
6555     return ($volid, $vars_size);
6556 }
6557
6558 sub lspci {
6559
6560     my $devices = {};
6561
6562     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6563             my (undef, $id, $function) = @_;
6564             my $res = { id => $id, function => $function};
6565             push @{$devices->{$id}}, $res;
6566     });
6567
6568     # Entries should be sorted by functions.
6569     foreach my $id (keys %$devices) {
6570         my $dev = $devices->{$id};
6571         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6572     }
6573
6574     return $devices;
6575 }
6576
6577 sub vm_iothreads_list {
6578     my ($vmid) = @_;
6579
6580     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6581
6582     my $iothreads = {};
6583     foreach my $iothread (@$res) {
6584         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6585     }
6586
6587     return $iothreads;
6588 }
6589
6590 sub scsihw_infos {
6591     my ($conf, $drive) = @_;
6592
6593     my $maxdev = 0;
6594
6595     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6596         $maxdev = 7;
6597     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6598         $maxdev = 1;
6599     } else {
6600         $maxdev = 256;
6601     }
6602
6603     my $controller = int($drive->{index} / $maxdev);
6604     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6605
6606     return ($maxdev, $controller, $controller_prefix);
6607 }
6608
6609 sub add_hyperv_enlightenments {
6610     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6611
6612     return if $winversion < 6;
6613     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6614
6615     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6616
6617     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6618         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6619         push @$cpuFlags , 'hv_vapic';
6620         push @$cpuFlags , 'hv_time';
6621     } else {
6622         push @$cpuFlags , 'hv_spinlocks=0xffff';
6623     }
6624
6625     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6626         push @$cpuFlags , 'hv_reset';
6627         push @$cpuFlags , 'hv_vpindex';
6628         push @$cpuFlags , 'hv_runtime';
6629     }
6630
6631     if ($winversion >= 7) {
6632         push @$cpuFlags , 'hv_relaxed';
6633     }
6634 }
6635
6636 sub windows_version {
6637     my ($ostype) = @_;
6638
6639     return 0 if !$ostype;
6640
6641     my $winversion = 0;
6642
6643     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6644         $winversion = 5;
6645     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6646         $winversion = 6;
6647     } elsif ($ostype =~ m/^win(\d+)$/) {
6648         $winversion = $1;
6649     }
6650
6651     return $winversion;
6652 }
6653
6654 sub resolve_dst_disk_format {
6655         my ($storecfg, $storeid, $src_volname, $format) = @_;
6656         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6657
6658         if (!$format) {
6659             # if no target format is specified, use the source disk format as hint
6660             if ($src_volname) {
6661                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6662                 $format = qemu_img_format($scfg, $src_volname);
6663             } else {
6664                 return $defFormat;
6665             }
6666         }
6667
6668         # test if requested format is supported - else use default
6669         my $supported = grep { $_ eq $format } @$validFormats;
6670         $format = $defFormat if !$supported;
6671         return $format;
6672 }
6673
6674 sub resolve_first_disk {
6675     my $conf = shift;
6676     my @disks = PVE::QemuServer::valid_drive_names();
6677     my $firstdisk;
6678     foreach my $ds (reverse @disks) {
6679         next if !$conf->{$ds};
6680         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6681         next if PVE::QemuServer::drive_is_cdrom($disk);
6682         $firstdisk = $ds;
6683     }
6684     return $firstdisk;
6685 }
6686
6687 sub generate_smbios1_uuid {
6688     my ($uuid, $uuid_str);
6689     UUID::generate($uuid);
6690     UUID::unparse($uuid, $uuid_str);
6691     return "uuid=$uuid_str";
6692 }
6693
6694 # bash completion helper
6695
6696 sub complete_backup_archives {
6697     my ($cmdname, $pname, $cvalue) = @_;
6698
6699     my $cfg = PVE::Storage::config();
6700
6701     my $storeid;
6702
6703     if ($cvalue =~ m/^([^:]+):/) {
6704         $storeid = $1;
6705     }
6706
6707     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6708
6709     my $res = [];
6710     foreach my $id (keys %$data) {
6711         foreach my $item (@{$data->{$id}}) {
6712             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6713             push @$res, $item->{volid} if defined($item->{volid});
6714         }
6715     }
6716
6717     return $res;
6718 }
6719
6720 my $complete_vmid_full = sub {
6721     my ($running) = @_;
6722
6723     my $idlist = vmstatus();
6724
6725     my $res = [];
6726
6727     foreach my $id (keys %$idlist) {
6728         my $d = $idlist->{$id};
6729         if (defined($running)) {
6730             next if $d->{template};
6731             next if $running && $d->{status} ne 'running';
6732             next if !$running && $d->{status} eq 'running';
6733         }
6734         push @$res, $id;
6735
6736     }
6737     return $res;
6738 };
6739
6740 sub complete_vmid {
6741     return &$complete_vmid_full();
6742 }
6743
6744 sub complete_vmid_stopped {
6745     return &$complete_vmid_full(0);
6746 }
6747
6748 sub complete_vmid_running {
6749     return &$complete_vmid_full(1);
6750 }
6751
6752 sub complete_storage {
6753
6754     my $cfg = PVE::Storage::config();
6755     my $ids = $cfg->{ids};
6756
6757     my $res = [];
6758     foreach my $sid (keys %$ids) {
6759         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6760         next if !$ids->{$sid}->{content}->{images};
6761         push @$res, $sid;
6762     }
6763
6764     return $res;
6765 }
6766
6767 1;