PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120
 121     # AMD CPUs
 122     athlon => 'AuthenticAMD',
 123     phenom  => 'AuthenticAMD',
 124     Opteron_G1  => 'AuthenticAMD',
 125     Opteron_G2  => 'AuthenticAMD',
 126     Opteron_G3  => 'AuthenticAMD',
 127     Opteron_G4  => 'AuthenticAMD',
 128     Opteron_G5  => 'AuthenticAMD',
 129
 130     # generic types, use vendor from host node
 131     host => 'default',
 132     kvm32 => 'default',
 133     kvm64 => 'default',
 134     qemu32 => 'default',
 135     qemu64 => 'default',
 136 };
 137
 138 my $cpu_fmt = {
 139     cputype => {
 140         description => "Emulated CPU type.",
 141         type => 'string',
 142         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 143         format_description => 'cputype',
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1000,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-drive',
 456         typetext => 'volume',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         format_description => 'model',
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => {
 738         type => 'string',
 739         format_description => 'image format',
 740         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 741         description => "The drive's backing file's data format.",
 742         optional => 1,
 743     },
 744     size => {
 745         type => 'string',
 746         format => 'disk-size',
 747         format_description => 'DiskSize',
 748         description => "Disk size. This is purely informational and has no effect.",
 749         optional => 1,
 750     },
 751     backup => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making backups.",
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %rerror_fmt = (
 790     rerror => {
 791         type => 'string',
 792         enum => [qw(ignore report stop)],
 793         description => 'Read error action.',
 794         optional => 1,
 795     },
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit) = @_;
 826     $drivedesc_base{$key} = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what speed in $longunit per second.",
 830         optional => 1,
 831     };
 832 };
 833 # throughput: (leaky bucket)
 834 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 835 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 836 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 837 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 838 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 839 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 840 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 841 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 842 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 843
 844 # pools: (pool of IO before throttling starts taking effect)
 845 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 846 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 847 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 848 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 849 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 850 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 851
 852 my $ide_fmt = {
 853     %drivedesc_base,
 854     %rerror_fmt,
 855     %model_fmt,
 856 };
 857
 858 my $idedesc = {
 859     optional => 1,
 860     type => 'string', format => $ide_fmt,
 861     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 862 };
 863 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 864
 865 my $scsi_fmt = {
 866     %drivedesc_base,
 867     %iothread_fmt,
 868     %queues_fmt,
 869 };
 870 my $scsidesc = {
 871     optional => 1,
 872     type => 'string', format => $scsi_fmt,
 873     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 874 };
 875 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 876
 877 my $sata_fmt = {
 878     %drivedesc_base,
 879     %rerror_fmt,
 880 };
 881 my $satadesc = {
 882     optional => 1,
 883     type => 'string', format => $sata_fmt,
 884     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 887
 888 my $virtio_fmt = {
 889     %drivedesc_base,
 890     %iothread_fmt,
 891     %rerror_fmt,
 892 };
 893 my $virtiodesc = {
 894     optional => 1,
 895     type => 'string', format => $virtio_fmt,
 896     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 899
 900 my $alldrive_fmt = {
 901     %drivedesc_base,
 902     %rerror_fmt,
 903     %iothread_fmt,
 904     %model_fmt,
 905     %queues_fmt,
 906 };
 907
 908 my $efidisk_fmt = {
 909     volume => { alias => 'file' },
 910     file => {
 911         type => 'string',
 912         format => 'pve-volume-id-or-qm-path',
 913         default_key => 1,
 914         format_description => 'volume',
 915         description => "The drive's backing volume.",
 916     },
 917     format => {
 918         type => 'string',
 919         format_description => 'image format',
 920         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 921         description => "The drive's backing file's data format.",
 922         optional => 1,
 923     },
 924     size => {
 925         type => 'string',
 926         format => 'disk-size',
 927         format_description => 'DiskSize',
 928         description => "Disk size. This is purely informational and has no effect.",
 929         optional => 1,
 930     },
 931 };
 932
 933 my $efidisk_desc = {
 934     optional => 1,
 935     type => 'string', format => $efidisk_fmt,
 936     description => "Configure a Disk for storing EFI vars",
 937 };
 938
 939 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 940
 941 my $usb_fmt = {
 942     host => {
 943         default_key => 1,
 944         type => 'string', format => 'pve-qm-usb-device',
 945         format_description => 'HOSTUSBDEVICE|spice',
 946         description => <<EODESCR,
 947 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 948
 949  'bus-port(.port)*' (decimal numbers) or
 950  'vendor_id:product_id' (hexadeciaml numbers) or
 951  'spice'
 952
 953 You can use the 'lsusb -t' command to list existing usb devices.
 954
 955 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 956
 957 The value 'spice' can be used to add a usb redirection devices for spice.
 958 EODESCR
 959     },
 960     usb3 => {
 961         optional => 1,
 962         type => 'boolean',
 963         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 964         default => 0,
 965     },
 966 };
 967
 968 my $usbdesc = {
 969     optional => 1,
 970     type => 'string', format => $usb_fmt,
 971     description => "Configure an USB device (n is 0 to 4).",
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 974
 975 # NOTE: the match-groups of this regex are used in parse_hostpci
 976 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 977 my $hostpci_fmt = {
 978     host => {
 979         default_key => 1,
 980         type => 'string',
 981         pattern => qr/$PCIRE(;$PCIRE)*/,
 982         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 983         description => <<EODESCR,
 984 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 985 of PCI virtual functions of the host. HOSTPCIID syntax is:
 986
 987 'bus:dev.func' (hexadecimal numbers)
 988
 989 You can us the 'lspci' command to list existing PCI devices.
 990 EODESCR
 991     },
 992     rombar => {
 993         type => 'boolean',
 994         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 995         optional => 1,
 996         default => 1,
 997     },
 998     pcie => {
 999         type => 'boolean',
1000         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1001         optional => 1,
1002         default => 0,
1003     },
1004     'x-vga' => {
1005         type => 'boolean',
1006         description =>  "Enable vfio-vga device support.",
1007         optional => 1,
1008         default => 0,
1009     },
1010 };
1011 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1012
1013 my $hostpcidesc = {
1014         optional => 1,
1015         type => 'string', format => 'pve-qm-hostpci',
1016         description => "Map host PCI devices into guest.",
1017         verbose_description =>  <<EODESCR,
1018 Map host PCI devices into guest.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer
1021 possible to migrate such machines - use with special care.
1022
1023 CAUTION: Experimental! User reported problems with this option.
1024 EODESCR
1025 };
1026 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1027
1028 my $serialdesc = {
1029         optional => 1,
1030         type => 'string',
1031         pattern => '(/dev/.+|socket)',
1032         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1033         verbose_description =>  <<EODESCR,
1034 Create a serial device inside the VM (n is 0 to 3), and pass through a
1035 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1036 host side (use 'qm terminal' to open a terminal connection).
1037
1038 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1039
1040 CAUTION: Experimental! User reported problems with this option.
1041 EODESCR
1042 };
1043
1044 my $paralleldesc= {
1045         optional => 1,
1046         type => 'string',
1047         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1048         description =>  "Map host parallel devices (n is 0 to 2).",
1049         verbose_description =>  <<EODESCR,
1050 Map host parallel devices (n is 0 to 2).
1051
1052 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1059     $confdesc->{"parallel$i"} = $paralleldesc;
1060 }
1061
1062 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1063     $confdesc->{"serial$i"} = $serialdesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1067     $confdesc->{"hostpci$i"} = $hostpcidesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1071     $drivename_hash->{"ide$i"} = 1;
1072     $confdesc->{"ide$i"} = $idedesc;
1073 }
1074
1075 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1076     $drivename_hash->{"sata$i"} = 1;
1077     $confdesc->{"sata$i"} = $satadesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1081     $drivename_hash->{"scsi$i"} = 1;
1082     $confdesc->{"scsi$i"} = $scsidesc ;
1083 }
1084
1085 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1086     $drivename_hash->{"virtio$i"} = 1;
1087     $confdesc->{"virtio$i"} = $virtiodesc;
1088 }
1089
1090 $drivename_hash->{efidisk0} = 1;
1091 $confdesc->{efidisk0} = $efidisk_desc;
1092
1093 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1094     $confdesc->{"usb$i"} = $usbdesc;
1095 }
1096
1097 my $unuseddesc = {
1098     optional => 1,
1099     type => 'string', format => 'pve-volume-id',
1100     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1101 };
1102
1103 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1104     $confdesc->{"unused$i"} = $unuseddesc;
1105 }
1106
1107 my $kvm_api_version = 0;
1108
1109 sub kvm_version {
1110
1111     return $kvm_api_version if $kvm_api_version;
1112
1113     my $fh = IO::File->new("</dev/kvm") ||
1114         return 0;
1115
1116     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1117         $kvm_api_version = $v;
1118     }
1119
1120     $fh->close();
1121
1122     return  $kvm_api_version;
1123 }
1124
1125 my $kvm_user_version;
1126
1127 sub kvm_user_version {
1128
1129     return $kvm_user_version if $kvm_user_version;
1130
1131     $kvm_user_version = 'unknown';
1132
1133     my $code = sub {
1134         my $line = shift;
1135         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1136             $kvm_user_version = $2;
1137         }
1138     };
1139
1140     eval { run_command("kvm -version", outfunc => $code); };
1141     warn $@ if $@;
1142
1143     return $kvm_user_version;
1144
1145 }
1146
1147 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1148
1149 sub valid_drive_names {
1150     # order is important - used to autoselect boot disk
1151     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1152             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1153             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1154             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1155             'efidisk0');
1156 }
1157
1158 sub is_valid_drivename {
1159     my $dev = shift;
1160
1161     return defined($drivename_hash->{$dev});
1162 }
1163
1164 sub option_exists {
1165     my $key = shift;
1166     return defined($confdesc->{$key});
1167 }
1168
1169 sub nic_models {
1170     return $nic_model_list;
1171 }
1172
1173 sub os_list_description {
1174
1175     return {
1176         other => 'Other',
1177         wxp => 'Windows XP',
1178         w2k => 'Windows 2000',
1179         w2k3 =>, 'Windows 2003',
1180         w2k8 => 'Windows 2008',
1181         wvista => 'Windows Vista',
1182         win7 => 'Windows 7',
1183         win8 => 'Windows 8/2012',
1184         l24 => 'Linux 2.4',
1185         l26 => 'Linux 2.6',
1186     };
1187 }
1188
1189 my $cdrom_path;
1190
1191 sub get_cdrom_path {
1192
1193     return  $cdrom_path if $cdrom_path;
1194
1195     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1196     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1197     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1198 }
1199
1200 sub get_iso_path {
1201     my ($storecfg, $vmid, $cdrom) = @_;
1202
1203     if ($cdrom eq 'cdrom') {
1204         return get_cdrom_path();
1205     } elsif ($cdrom eq 'none') {
1206         return '';
1207     } elsif ($cdrom =~ m|^/|) {
1208         return $cdrom;
1209     } else {
1210         return PVE::Storage::path($storecfg, $cdrom);
1211     }
1212 }
1213
1214 # try to convert old style file names to volume IDs
1215 sub filename_to_volume_id {
1216     my ($vmid, $file, $media) = @_;
1217
1218     if (!($file eq 'none' || $file eq 'cdrom' ||
1219           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1220
1221         return undef if $file =~ m|/|;
1222
1223         if ($media && $media eq 'cdrom') {
1224             $file = "local:iso/$file";
1225         } else {
1226             $file = "local:$vmid/$file";
1227         }
1228     }
1229
1230     return $file;
1231 }
1232
1233 sub verify_media_type {
1234     my ($opt, $vtype, $media) = @_;
1235
1236     return if !$media;
1237
1238     my $etype;
1239     if ($media eq 'disk') {
1240         $etype = 'images';
1241     } elsif ($media eq 'cdrom') {
1242         $etype = 'iso';
1243     } else {
1244         die "internal error";
1245     }
1246
1247     return if ($vtype eq $etype);
1248
1249     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1250 }
1251
1252 sub cleanup_drive_path {
1253     my ($opt, $storecfg, $drive) = @_;
1254
1255     # try to convert filesystem paths to volume IDs
1256
1257     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1258         ($drive->{file} !~ m|^/dev/.+|) &&
1259         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1260         ($drive->{file} !~ m/^\d+$/)) {
1261         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1262         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1263         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1264         verify_media_type($opt, $vtype, $drive->{media});
1265         $drive->{file} = $volid;
1266     }
1267
1268     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1269 }
1270
1271 sub parse_hotplug_features {
1272     my ($data) = @_;
1273
1274     my $res = {};
1275
1276     return $res if $data eq '0';
1277
1278     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1279
1280     foreach my $feature (PVE::Tools::split_list($data)) {
1281         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1282             $res->{$1} = 1;
1283         } else {
1284             die "invalid hotplug feature '$feature'\n";
1285         }
1286     }
1287     return $res;
1288 }
1289
1290 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1291 sub pve_verify_hotplug_features {
1292     my ($value, $noerr) = @_;
1293
1294     return $value if parse_hotplug_features($value);
1295
1296     return undef if $noerr;
1297
1298     die "unable to parse hotplug option\n";
1299 }
1300
1301 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1302 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1303 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1304 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1305 #        [,iothread=on][,serial=serial][,model=model]
1306
1307 sub parse_drive {
1308     my ($key, $data) = @_;
1309
1310     my ($interface, $index);
1311
1312     if ($key =~ m/^([^\d]+)(\d+)$/) {
1313         $interface = $1;
1314         $index = $2;
1315     } else {
1316         return undef;
1317     }
1318
1319     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1320                                      : $confdesc->{$key}->{format};
1321     if (!$desc) {
1322         warn "invalid drive key: $key\n";
1323         return undef;
1324     }
1325     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1326     return undef if !$res;
1327     $res->{interface} = $interface;
1328     $res->{index} = $index;
1329
1330     my $error = 0;
1331     foreach my $opt (qw(bps bps_rd bps_wr)) {
1332         if (my $bps = defined(delete $res->{$opt})) {
1333             if (defined($res->{"m$opt"})) {
1334                 warn "both $opt and m$opt specified\n";
1335                 ++$error;
1336                 next;
1337             }
1338             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1339         }
1340     }
1341     return undef if $error;
1342
1343     return undef if $res->{mbps_rd} && $res->{mbps};
1344     return undef if $res->{mbps_wr} && $res->{mbps};
1345     return undef if $res->{iops_rd} && $res->{iops};
1346     return undef if $res->{iops_wr} && $res->{iops};
1347
1348     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1349         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1350         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1351         return undef if $res->{interface} eq 'virtio';
1352     }
1353
1354     if (my $size = $res->{size}) {
1355         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1356     }
1357
1358     return $res;
1359 }
1360
1361 sub print_drive {
1362     my ($vmid, $drive) = @_;
1363     my $data = { %$drive };
1364     delete $data->{$_} for qw(index interface);
1365     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1366 }
1367
1368 sub scsi_inquiry {
1369     my($fh, $noerr) = @_;
1370
1371     my $SG_IO = 0x2285;
1372     my $SG_GET_VERSION_NUM = 0x2282;
1373
1374     my $versionbuf = "\x00" x 8;
1375     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1376     if (!$ret) {
1377         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1378         return undef;
1379     }
1380     my $version = unpack("I", $versionbuf);
1381     if ($version < 30000) {
1382         die "scsi generic interface too old\n"  if !$noerr;
1383         return undef;
1384     }
1385
1386     my $buf = "\x00" x 36;
1387     my $sensebuf = "\x00" x 8;
1388     my $cmd = pack("C x3 C x1", 0x12, 36);
1389
1390     # see /usr/include/scsi/sg.h
1391     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1392
1393     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1394                       length($sensebuf), 0, length($buf), $buf,
1395                       $cmd, $sensebuf, 6000);
1396
1397     $ret = ioctl($fh, $SG_IO, $packet);
1398     if (!$ret) {
1399         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1400         return undef;
1401     }
1402
1403     my @res = unpack($sg_io_hdr_t, $packet);
1404     if ($res[17] || $res[18]) {
1405         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1406         return undef;
1407     }
1408
1409     my $res = {};
1410     (my $byte0, my $byte1, $res->{vendor},
1411      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1412
1413     $res->{removable} = $byte1 & 128 ? 1 : 0;
1414     $res->{type} = $byte0 & 31;
1415
1416     return $res;
1417 }
1418
1419 sub path_is_scsi {
1420     my ($path) = @_;
1421
1422     my $fh = IO::File->new("+<$path") || return undef;
1423     my $res = scsi_inquiry($fh, 1);
1424     close($fh);
1425
1426     return $res;
1427 }
1428
1429 sub machine_type_is_q35 {
1430     my ($conf) = @_;
1431
1432     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1433 }
1434
1435 sub print_tabletdevice_full {
1436     my ($conf) = @_;
1437
1438     my $q35 = machine_type_is_q35($conf);
1439
1440     # we use uhci for old VMs because tablet driver was buggy in older qemu
1441     my $usbbus = $q35 ? "ehci" : "uhci";
1442
1443     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1444 }
1445
1446 sub print_drivedevice_full {
1447     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1448
1449     my $device = '';
1450     my $maxdev = 0;
1451
1452     if ($drive->{interface} eq 'virtio') {
1453         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1454         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1455         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1456     } elsif ($drive->{interface} eq 'scsi') {
1457
1458         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1459         my $unit = $drive->{index} % $maxdev;
1460         my $devicetype = 'hd';
1461         my $path = '';
1462         if (drive_is_cdrom($drive)) {
1463             $devicetype = 'cd';
1464         } else {
1465             if ($drive->{file} =~ m|^/|) {
1466                 $path = $drive->{file};
1467                 if (my $info = path_is_scsi($path)) {
1468                     if ($info->{type} == 0) {
1469                         $devicetype = 'block';
1470                     } elsif ($info->{type} == 1) { # tape
1471                         $devicetype = 'generic';
1472                     }
1473                 }
1474             } else {
1475                  $path = PVE::Storage::path($storecfg, $drive->{file});
1476             }
1477
1478             if($path =~ m/^iscsi\:\/\//){
1479                 $devicetype = 'generic';
1480             }
1481         }
1482
1483         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1484             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1485         } else {
1486             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1487         }
1488
1489     } elsif ($drive->{interface} eq 'ide'){
1490         $maxdev = 2;
1491         my $controller = int($drive->{index} / $maxdev);
1492         my $unit = $drive->{index} % $maxdev;
1493         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1494
1495         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1496         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1497             $model = URI::Escape::uri_unescape($model);
1498             $device .= ",model=$model";
1499         }
1500     } elsif ($drive->{interface} eq 'sata'){
1501         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1502         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1503         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1504     } elsif ($drive->{interface} eq 'usb') {
1505         die "implement me";
1506         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1507     } else {
1508         die "unsupported interface type";
1509     }
1510
1511     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1512
1513     return $device;
1514 }
1515
1516 sub get_initiator_name {
1517     my $initiator;
1518
1519     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1520     while (defined(my $line = <$fh>)) {
1521         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1522         $initiator = $1;
1523         last;
1524     }
1525     $fh->close();
1526
1527     return $initiator;
1528 }
1529
1530 sub print_drive_full {
1531     my ($storecfg, $vmid, $drive) = @_;
1532
1533     my $path;
1534     my $volid = $drive->{file};
1535     my $format;
1536
1537     if (drive_is_cdrom($drive)) {
1538         $path = get_iso_path($storecfg, $vmid, $volid);
1539     } else {
1540         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1541         if ($storeid) {
1542             $path = PVE::Storage::path($storecfg, $volid);
1543             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1544             $format = qemu_img_format($scfg, $volname);
1545         } else {
1546             $path = $volid;
1547             $format = "raw";
1548         }
1549    }
1550
1551     my $opts = '';
1552     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1553     foreach my $o (@qemu_drive_options) {
1554         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1555     }
1556     if (my $serial = $drive->{serial}) {
1557         $serial = URI::Escape::uri_unescape($serial);
1558         $opts .= ",serial=$serial";
1559     }
1560
1561     $opts .= ",format=$format" if $format && !$drive->{format};
1562
1563     foreach my $o (qw(bps bps_rd bps_wr)) {
1564         my $v = $drive->{"m$o"};
1565         $opts .= ",$o=" . int($v*1024*1024) if $v;
1566     }
1567
1568     my $cache_direct = 0;
1569
1570     if (my $cache = $drive->{cache}) {
1571         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1572     } elsif (!drive_is_cdrom($drive)) {
1573         $opts .= ",cache=none";
1574         $cache_direct = 1;
1575     }
1576
1577     # aio native works only with O_DIRECT
1578     if (!$drive->{aio}) {
1579         if($cache_direct) {
1580             $opts .= ",aio=native";
1581         } else {
1582             $opts .= ",aio=threads";
1583         }
1584     }
1585
1586     if (!drive_is_cdrom($drive)) {
1587         my $detectzeroes;
1588         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1589             $detectzeroes = 'off';
1590         } elsif ($drive->{discard}) {
1591             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1592         } else {
1593             # This used to be our default with discard not being specified:
1594             $detectzeroes = 'on';
1595         }
1596         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1597     }
1598
1599     my $pathinfo = $path ? "file=$path," : '';
1600
1601     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1602 }
1603
1604 sub print_netdevice_full {
1605     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1606
1607     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1608
1609     my $device = $net->{model};
1610     if ($net->{model} eq 'virtio') {
1611          $device = 'virtio-net-pci';
1612      };
1613
1614     my $pciaddr = print_pci_addr("$netid", $bridges);
1615     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1616     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1617         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1618         my $vectors = $net->{queues} * 2 + 2;
1619         $tmpstr .= ",vectors=$vectors,mq=on";
1620     }
1621     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1622
1623     if ($use_old_bios_files) {
1624         my $romfile;
1625         if ($device eq 'virtio-net-pci') {
1626             $romfile = 'pxe-virtio.rom';
1627         } elsif ($device eq 'e1000') {
1628             $romfile = 'pxe-e1000.rom';
1629         } elsif ($device eq 'ne2k') {
1630             $romfile = 'pxe-ne2k_pci.rom';
1631         } elsif ($device eq 'pcnet') {
1632             $romfile = 'pxe-pcnet.rom';
1633         } elsif ($device eq 'rtl8139') {
1634             $romfile = 'pxe-rtl8139.rom';
1635         }
1636         $tmpstr .= ",romfile=$romfile" if $romfile;
1637     }
1638
1639     return $tmpstr;
1640 }
1641
1642 sub print_netdev_full {
1643     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1644
1645     my $i = '';
1646     if ($netid =~ m/^net(\d+)$/) {
1647         $i = int($1);
1648     }
1649
1650     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1651
1652     my $ifname = "tap${vmid}i$i";
1653
1654     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1655     die "interface name '$ifname' is too long (max 15 character)\n"
1656         if length($ifname) >= 16;
1657
1658     my $vhostparam = '';
1659     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1660
1661     my $vmname = $conf->{name} || "vm$vmid";
1662
1663     my $netdev = "";
1664     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1665
1666     if ($net->{bridge}) {
1667         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1668     } else {
1669         $netdev = "type=user,id=$netid,hostname=$vmname";
1670     }
1671
1672     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1673
1674     return $netdev;
1675 }
1676
1677 sub drive_is_cdrom {
1678     my ($drive) = @_;
1679
1680     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1681
1682 }
1683
1684 sub parse_number_sets {
1685     my ($set) = @_;
1686     my $res = [];
1687     foreach my $part (split(/;/, $set)) {
1688         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1689             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1690             push @$res, [ $1, $2 ];
1691         } else {
1692             die "invalid range: $part\n";
1693         }
1694     }
1695     return $res;
1696 }
1697
1698 sub parse_numa {
1699     my ($data) = @_;
1700
1701     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1702     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1703     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1704     return $res;
1705 }
1706
1707 sub parse_hostpci {
1708     my ($value) = @_;
1709
1710     return undef if !$value;
1711
1712     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1713
1714     my @idlist = split(/;/, $res->{host});
1715     delete $res->{host};
1716     foreach my $id (@idlist) {
1717         if ($id =~ /^$PCIRE$/) {
1718             if (defined($2)) {
1719                 push @{$res->{pciid}}, { id => $1, function => $2 };
1720             } else {
1721                 my $pcidevices = lspci($1);
1722                 $res->{pciid} = $pcidevices->{$1};
1723             }
1724         } else {
1725             # should have been caught by parse_property_string already
1726             die "failed to parse PCI id: $id\n";
1727         }
1728     }
1729     return $res;
1730 }
1731
1732 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1733 sub parse_net {
1734     my ($data) = @_;
1735
1736     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1737     if ($@) {
1738         warn $@;
1739         return undef;
1740     }
1741     if (!defined($res->{macaddr})) {
1742         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1743         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1744     }
1745     return $res;
1746 }
1747
1748 sub print_net {
1749     my $net = shift;
1750
1751     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1752 }
1753
1754 sub add_random_macs {
1755     my ($settings) = @_;
1756
1757     foreach my $opt (keys %$settings) {
1758         next if $opt !~ m/^net(\d+)$/;
1759         my $net = parse_net($settings->{$opt});
1760         next if !$net;
1761         $settings->{$opt} = print_net($net);
1762     }
1763 }
1764
1765 sub vm_is_volid_owner {
1766     my ($storecfg, $vmid, $volid) = @_;
1767
1768     if ($volid !~  m|^/|) {
1769         my ($path, $owner);
1770         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1771         if ($owner && ($owner == $vmid)) {
1772             return 1;
1773         }
1774     }
1775
1776     return undef;
1777 }
1778
1779 sub split_flagged_list {
1780     my $text = shift || '';
1781     $text =~ s/[,;]/ /g;
1782     $text =~ s/^\s+//;
1783     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1784 }
1785
1786 sub join_flagged_list {
1787     my ($how, $lst) = @_;
1788     join $how, map { $lst->{$_} . $_ } keys %$lst;
1789 }
1790
1791 sub vmconfig_delete_pending_option {
1792     my ($conf, $key, $force) = @_;
1793
1794     delete $conf->{pending}->{$key};
1795     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1796     $pending_delete_hash->{$key} = $force ? '!' : '';
1797     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1798 }
1799
1800 sub vmconfig_undelete_pending_option {
1801     my ($conf, $key) = @_;
1802
1803     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1804     delete $pending_delete_hash->{$key};
1805
1806     if (%$pending_delete_hash) {
1807         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1808     } else {
1809         delete $conf->{pending}->{delete};
1810     }
1811 }
1812
1813 sub vmconfig_register_unused_drive {
1814     my ($storecfg, $vmid, $conf, $drive) = @_;
1815
1816     if (!drive_is_cdrom($drive)) {
1817         my $volid = $drive->{file};
1818         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1819             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1820         }
1821     }
1822 }
1823
1824 sub vmconfig_cleanup_pending {
1825     my ($conf) = @_;
1826
1827     # remove pending changes when nothing changed
1828     my $changes;
1829     foreach my $opt (keys %{$conf->{pending}}) {
1830         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1831             $changes = 1;
1832             delete $conf->{pending}->{$opt};
1833         }
1834     }
1835
1836     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1837     my $pending_delete_hash = {};
1838     while (my ($opt, $force) = each %$current_delete_hash) {
1839         if (defined($conf->{$opt})) {
1840             $pending_delete_hash->{$opt} = $force;
1841         } else {
1842             $changes = 1;
1843         }
1844     }
1845
1846     if (%$pending_delete_hash) {
1847         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1848     } else {
1849         delete $conf->{pending}->{delete};
1850     }
1851
1852     return $changes;
1853 }
1854
1855 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1856 my $smbios1_fmt = {
1857     uuid => {
1858         type => 'string',
1859         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1860         format_description => 'UUID',
1861         description => "Set SMBIOS1 UUID.",
1862         optional => 1,
1863     },
1864     version => {
1865         type => 'string',
1866         pattern => '\S+',
1867         format_description => 'string',
1868         description => "Set SMBIOS1 version.",
1869         optional => 1,
1870     },
1871     serial => {
1872         type => 'string',
1873         pattern => '\S+',
1874         format_description => 'string',
1875         description => "Set SMBIOS1 serial number.",
1876         optional => 1,
1877     },
1878     manufacturer => {
1879         type => 'string',
1880         pattern => '\S+',
1881         format_description => 'string',
1882         description => "Set SMBIOS1 manufacturer.",
1883         optional => 1,
1884     },
1885     product => {
1886         type => 'string',
1887         pattern => '\S+',
1888         format_description => 'string',
1889         description => "Set SMBIOS1 product ID.",
1890         optional => 1,
1891     },
1892     sku => {
1893         type => 'string',
1894         pattern => '\S+',
1895         format_description => 'string',
1896         description => "Set SMBIOS1 SKU string.",
1897         optional => 1,
1898     },
1899     family => {
1900         type => 'string',
1901         pattern => '\S+',
1902         format_description => 'string',
1903         description => "Set SMBIOS1 family string.",
1904         optional => 1,
1905     },
1906 };
1907
1908 sub parse_smbios1 {
1909     my ($data) = @_;
1910
1911     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1912     warn $@ if $@;
1913     return $res;
1914 }
1915
1916 sub print_smbios1 {
1917     my ($smbios1) = @_;
1918     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1919 }
1920
1921 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1922
1923 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1924 sub verify_bootdisk {
1925     my ($value, $noerr) = @_;
1926
1927     return $value if is_valid_drivename($value);
1928
1929     return undef if $noerr;
1930
1931     die "invalid boot disk '$value'\n";
1932 }
1933
1934 sub parse_watchdog {
1935     my ($value) = @_;
1936
1937     return undef if !$value;
1938
1939     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1940     warn $@ if $@;
1941     return $res;
1942 }
1943
1944 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1945 sub verify_usb_device {
1946     my ($value, $noerr) = @_;
1947
1948     return $value if parse_usb_device($value);
1949
1950     return undef if $noerr;
1951
1952     die "unable to parse usb device\n";
1953 }
1954
1955 # add JSON properties for create and set function
1956 sub json_config_properties {
1957     my $prop = shift;
1958
1959     foreach my $opt (keys %$confdesc) {
1960         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1961         $prop->{$opt} = $confdesc->{$opt};
1962     }
1963
1964     return $prop;
1965 }
1966
1967 sub check_type {
1968     my ($key, $value) = @_;
1969
1970     die "unknown setting '$key'\n" if !$confdesc->{$key};
1971
1972     my $type = $confdesc->{$key}->{type};
1973
1974     if (!defined($value)) {
1975         die "got undefined value\n";
1976     }
1977
1978     if ($value =~ m/[\n\r]/) {
1979         die "property contains a line feed\n";
1980     }
1981
1982     if ($type eq 'boolean') {
1983         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1984         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1985         die "type check ('boolean') failed - got '$value'\n";
1986     } elsif ($type eq 'integer') {
1987         return int($1) if $value =~ m/^(\d+)$/;
1988         die "type check ('integer') failed - got '$value'\n";
1989     } elsif ($type eq 'number') {
1990         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1991         die "type check ('number') failed - got '$value'\n";
1992     } elsif ($type eq 'string') {
1993         if (my $fmt = $confdesc->{$key}->{format}) {
1994             if ($fmt eq 'pve-qm-drive') {
1995                 # special case - we need to pass $key to parse_drive()
1996                 my $drive = parse_drive($key, $value);
1997                 return $value if $drive;
1998                 die "unable to parse drive options\n";
1999             }
2000             PVE::JSONSchema::check_format($fmt, $value);
2001             return $value;
2002         }
2003         $value =~ s/^\"(.*)\"$/$1/;
2004         return $value;
2005     } else {
2006         die "internal error"
2007     }
2008 }
2009
2010 sub check_iommu_support{
2011     #fixme : need to check IOMMU support
2012     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2013
2014     my $iommu=1;
2015     return $iommu;
2016
2017 }
2018
2019 sub touch_config {
2020     my ($vmid) = @_;
2021
2022     my $conf = PVE::QemuConfig->config_file($vmid);
2023     utime undef, undef, $conf;
2024 }
2025
2026 sub destroy_vm {
2027     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2028
2029     my $conffile = PVE::QemuConfig->config_file($vmid);
2030
2031     my $conf = PVE::QemuConfig->load_config($vmid);
2032
2033     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2034
2035     # only remove disks owned by this VM
2036     foreach_drive($conf, sub {
2037         my ($ds, $drive) = @_;
2038
2039         return if drive_is_cdrom($drive);
2040
2041         my $volid = $drive->{file};
2042
2043         return if !$volid || $volid =~ m|^/|;
2044
2045         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2046         return if !$path || !$owner || ($owner != $vmid);
2047
2048         PVE::Storage::vdisk_free($storecfg, $volid);
2049     });
2050
2051     if ($keep_empty_config) {
2052         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2053     } else {
2054         unlink $conffile;
2055     }
2056
2057     # also remove unused disk
2058     eval {
2059         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2060
2061         eval {
2062             PVE::Storage::foreach_volid($dl, sub {
2063                 my ($volid, $sid, $volname, $d) = @_;
2064                 PVE::Storage::vdisk_free($storecfg, $volid);
2065             });
2066         };
2067         warn $@ if $@;
2068
2069     };
2070     warn $@ if $@;
2071 }
2072
2073 sub parse_vm_config {
2074     my ($filename, $raw) = @_;
2075
2076     return undef if !defined($raw);
2077
2078     my $res = {
2079         digest => Digest::SHA::sha1_hex($raw),
2080         snapshots => {},
2081         pending => {},
2082     };
2083
2084     $filename =~ m|/qemu-server/(\d+)\.conf$|
2085         || die "got strange filename '$filename'";
2086
2087     my $vmid = $1;
2088
2089     my $conf = $res;
2090     my $descr;
2091     my $section = '';
2092
2093     my @lines = split(/\n/, $raw);
2094     foreach my $line (@lines) {
2095         next if $line =~ m/^\s*$/;
2096
2097         if ($line =~ m/^\[PENDING\]\s*$/i) {
2098             $section = 'pending';
2099             if (defined($descr)) {
2100                 $descr =~ s/\s+$//;
2101                 $conf->{description} = $descr;
2102             }
2103             $descr = undef;
2104             $conf = $res->{$section} = {};
2105             next;
2106
2107         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2108             $section = $1;
2109             if (defined($descr)) {
2110                 $descr =~ s/\s+$//;
2111                 $conf->{description} = $descr;
2112             }
2113             $descr = undef;
2114             $conf = $res->{snapshots}->{$section} = {};
2115             next;
2116         }
2117
2118         if ($line =~ m/^\#(.*)\s*$/) {
2119             $descr = '' if !defined($descr);
2120             $descr .= PVE::Tools::decode_text($1) . "\n";
2121             next;
2122         }
2123
2124         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2125             $descr = '' if !defined($descr);
2126             $descr .= PVE::Tools::decode_text($2);
2127         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2128             $conf->{snapstate} = $1;
2129         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2130             my $key = $1;
2131             my $value = $2;
2132             $conf->{$key} = $value;
2133         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2134             my $value = $1;
2135             if ($section eq 'pending') {
2136                 $conf->{delete} = $value; # we parse this later
2137             } else {
2138                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2139             }
2140         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2141             my $key = $1;
2142             my $value = $2;
2143             eval { $value = check_type($key, $value); };
2144             if ($@) {
2145                 warn "vm $vmid - unable to parse value of '$key' - $@";
2146             } else {
2147                 my $fmt = $confdesc->{$key}->{format};
2148                 if ($fmt && $fmt eq 'pve-qm-drive') {
2149                     my $v = parse_drive($key, $value);
2150                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2151                         $v->{file} = $volid;
2152                         $value = print_drive($vmid, $v);
2153                     } else {
2154                         warn "vm $vmid - unable to parse value of '$key'\n";
2155                         next;
2156                     }
2157                 }
2158
2159                 if ($key eq 'cdrom') {
2160                     $conf->{ide2} = $value;
2161                 } else {
2162                     $conf->{$key} = $value;
2163                 }
2164             }
2165         }
2166     }
2167
2168     if (defined($descr)) {
2169         $descr =~ s/\s+$//;
2170         $conf->{description} = $descr;
2171     }
2172     delete $res->{snapstate}; # just to be sure
2173
2174     return $res;
2175 }
2176
2177 sub write_vm_config {
2178     my ($filename, $conf) = @_;
2179
2180     delete $conf->{snapstate}; # just to be sure
2181
2182     if ($conf->{cdrom}) {
2183         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2184         $conf->{ide2} = $conf->{cdrom};
2185         delete $conf->{cdrom};
2186     }
2187
2188     # we do not use 'smp' any longer
2189     if ($conf->{sockets}) {
2190         delete $conf->{smp};
2191     } elsif ($conf->{smp}) {
2192         $conf->{sockets} = $conf->{smp};
2193         delete $conf->{cores};
2194         delete $conf->{smp};
2195     }
2196
2197     my $used_volids = {};
2198
2199     my $cleanup_config = sub {
2200         my ($cref, $pending, $snapname) = @_;
2201
2202         foreach my $key (keys %$cref) {
2203             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2204                 $key eq 'snapstate' || $key eq 'pending';
2205             my $value = $cref->{$key};
2206             if ($key eq 'delete') {
2207                 die "propertry 'delete' is only allowed in [PENDING]\n"
2208                     if !$pending;
2209                 # fixme: check syntax?
2210                 next;
2211             }
2212             eval { $value = check_type($key, $value); };
2213             die "unable to parse value of '$key' - $@" if $@;
2214
2215             $cref->{$key} = $value;
2216
2217             if (!$snapname && is_valid_drivename($key)) {
2218                 my $drive = parse_drive($key, $value);
2219                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2220             }
2221         }
2222     };
2223
2224     &$cleanup_config($conf);
2225
2226     &$cleanup_config($conf->{pending}, 1);
2227
2228     foreach my $snapname (keys %{$conf->{snapshots}}) {
2229         die "internal error" if $snapname eq 'pending';
2230         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2231     }
2232
2233     # remove 'unusedX' settings if we re-add a volume
2234     foreach my $key (keys %$conf) {
2235         my $value = $conf->{$key};
2236         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2237             delete $conf->{$key};
2238         }
2239     }
2240
2241     my $generate_raw_config = sub {
2242         my ($conf, $pending) = @_;
2243
2244         my $raw = '';
2245
2246         # add description as comment to top of file
2247         if (defined(my $descr = $conf->{description})) {
2248             if ($descr) {
2249                 foreach my $cl (split(/\n/, $descr)) {
2250                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2251                 }
2252             } else {
2253                 $raw .= "#\n" if $pending;
2254             }
2255         }
2256
2257         foreach my $key (sort keys %$conf) {
2258             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2259             $raw .= "$key: $conf->{$key}\n";
2260         }
2261         return $raw;
2262     };
2263
2264     my $raw = &$generate_raw_config($conf);
2265
2266     if (scalar(keys %{$conf->{pending}})){
2267         $raw .= "\n[PENDING]\n";
2268         $raw .= &$generate_raw_config($conf->{pending}, 1);
2269     }
2270
2271     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2272         $raw .= "\n[$snapname]\n";
2273         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2274     }
2275
2276     return $raw;
2277 }
2278
2279 sub load_defaults {
2280
2281     my $res = {};
2282
2283     # we use static defaults from our JSON schema configuration
2284     foreach my $key (keys %$confdesc) {
2285         if (defined(my $default = $confdesc->{$key}->{default})) {
2286             $res->{$key} = $default;
2287         }
2288     }
2289
2290     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2291     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2292
2293     return $res;
2294 }
2295
2296 sub config_list {
2297     my $vmlist = PVE::Cluster::get_vmlist();
2298     my $res = {};
2299     return $res if !$vmlist || !$vmlist->{ids};
2300     my $ids = $vmlist->{ids};
2301
2302     foreach my $vmid (keys %$ids) {
2303         my $d = $ids->{$vmid};
2304         next if !$d->{node} || $d->{node} ne $nodename;
2305         next if !$d->{type} || $d->{type} ne 'qemu';
2306         $res->{$vmid}->{exists} = 1;
2307     }
2308     return $res;
2309 }
2310
2311 # test if VM uses local resources (to prevent migration)
2312 sub check_local_resources {
2313     my ($conf, $noerr) = @_;
2314
2315     my $loc_res = 0;
2316
2317     $loc_res = 1 if $conf->{hostusb}; # old syntax
2318     $loc_res = 1 if $conf->{hostpci}; # old syntax
2319
2320     foreach my $k (keys %$conf) {
2321         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2322         # sockets are safe: they will recreated be on the target side post-migrate
2323         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2324         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2325     }
2326
2327     die "VM uses local resources\n" if $loc_res && !$noerr;
2328
2329     return $loc_res;
2330 }
2331
2332 # check if used storages are available on all nodes (use by migrate)
2333 sub check_storage_availability {
2334     my ($storecfg, $conf, $node) = @_;
2335
2336     foreach_drive($conf, sub {
2337         my ($ds, $drive) = @_;
2338
2339         my $volid = $drive->{file};
2340         return if !$volid;
2341
2342         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2343         return if !$sid;
2344
2345         # check if storage is available on both nodes
2346         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2347         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2348    });
2349 }
2350
2351 # list nodes where all VM images are available (used by has_feature API)
2352 sub shared_nodes {
2353     my ($conf, $storecfg) = @_;
2354
2355     my $nodelist = PVE::Cluster::get_nodelist();
2356     my $nodehash = { map { $_ => 1 } @$nodelist };
2357     my $nodename = PVE::INotify::nodename();
2358
2359     foreach_drive($conf, sub {
2360         my ($ds, $drive) = @_;
2361
2362         my $volid = $drive->{file};
2363         return if !$volid;
2364
2365         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2366         if ($storeid) {
2367             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2368             if ($scfg->{disable}) {
2369                 $nodehash = {};
2370             } elsif (my $avail = $scfg->{nodes}) {
2371                 foreach my $node (keys %$nodehash) {
2372                     delete $nodehash->{$node} if !$avail->{$node};
2373                 }
2374             } elsif (!$scfg->{shared}) {
2375                 foreach my $node (keys %$nodehash) {
2376                     delete $nodehash->{$node} if $node ne $nodename
2377                 }
2378             }
2379         }
2380     });
2381
2382     return $nodehash
2383 }
2384
2385 sub check_cmdline {
2386     my ($pidfile, $pid) = @_;
2387
2388     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2389     if (defined($fh)) {
2390         my $line = <$fh>;
2391         $fh->close;
2392         return undef if !$line;
2393         my @param = split(/\0/, $line);
2394
2395         my $cmd = $param[0];
2396         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2397
2398         for (my $i = 0; $i < scalar (@param); $i++) {
2399             my $p = $param[$i];
2400             next if !$p;
2401             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2402                 my $p = $param[$i+1];
2403                 return 1 if $p && ($p eq $pidfile);
2404                 return undef;
2405             }
2406         }
2407     }
2408     return undef;
2409 }
2410
2411 sub check_running {
2412     my ($vmid, $nocheck, $node) = @_;
2413
2414     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2415
2416     die "unable to find configuration file for VM $vmid - no such machine\n"
2417         if !$nocheck && ! -f $filename;
2418
2419     my $pidfile = pidfile_name($vmid);
2420
2421     if (my $fd = IO::File->new("<$pidfile")) {
2422         my $st = stat($fd);
2423         my $line = <$fd>;
2424         close($fd);
2425
2426         my $mtime = $st->mtime;
2427         if ($mtime > time()) {
2428             warn "file '$filename' modified in future\n";
2429         }
2430
2431         if ($line =~ m/^(\d+)$/) {
2432             my $pid = $1;
2433             if (check_cmdline($pidfile, $pid)) {
2434                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2435                     return $pid;
2436                 }
2437             }
2438         }
2439     }
2440
2441     return undef;
2442 }
2443
2444 sub vzlist {
2445
2446     my $vzlist = config_list();
2447
2448     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2449
2450     while (defined(my $de = $fd->read)) {
2451         next if $de !~ m/^(\d+)\.pid$/;
2452         my $vmid = $1;
2453         next if !defined($vzlist->{$vmid});
2454         if (my $pid = check_running($vmid)) {
2455             $vzlist->{$vmid}->{pid} = $pid;
2456         }
2457     }
2458
2459     return $vzlist;
2460 }
2461
2462 sub disksize {
2463     my ($storecfg, $conf) = @_;
2464
2465     my $bootdisk = $conf->{bootdisk};
2466     return undef if !$bootdisk;
2467     return undef if !is_valid_drivename($bootdisk);
2468
2469     return undef if !$conf->{$bootdisk};
2470
2471     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2472     return undef if !defined($drive);
2473
2474     return undef if drive_is_cdrom($drive);
2475
2476     my $volid = $drive->{file};
2477     return undef if !$volid;
2478
2479     return $drive->{size};
2480 }
2481
2482 my $last_proc_pid_stat;
2483
2484 # get VM status information
2485 # This must be fast and should not block ($full == false)
2486 # We only query KVM using QMP if $full == true (this can be slow)
2487 sub vmstatus {
2488     my ($opt_vmid, $full) = @_;
2489
2490     my $res = {};
2491
2492     my $storecfg = PVE::Storage::config();
2493
2494     my $list = vzlist();
2495     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2496
2497     my $cpucount = $cpuinfo->{cpus} || 1;
2498
2499     foreach my $vmid (keys %$list) {
2500         next if $opt_vmid && ($vmid ne $opt_vmid);
2501
2502         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2503         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2504
2505         my $d = {};
2506         $d->{pid} = $list->{$vmid}->{pid};
2507
2508         # fixme: better status?
2509         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2510
2511         my $size = disksize($storecfg, $conf);
2512         if (defined($size)) {
2513             $d->{disk} = 0; # no info available
2514             $d->{maxdisk} = $size;
2515         } else {
2516             $d->{disk} = 0;
2517             $d->{maxdisk} = 0;
2518         }
2519
2520         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2521         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2522         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2523
2524         $d->{name} = $conf->{name} || "VM $vmid";
2525         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2526
2527         if ($conf->{balloon}) {
2528             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2529             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2530         }
2531
2532         $d->{uptime} = 0;
2533         $d->{cpu} = 0;
2534         $d->{mem} = 0;
2535
2536         $d->{netout} = 0;
2537         $d->{netin} = 0;
2538
2539         $d->{diskread} = 0;
2540         $d->{diskwrite} = 0;
2541
2542         $d->{template} = PVE::QemuConfig->is_template($conf);
2543
2544         $res->{$vmid} = $d;
2545     }
2546
2547     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2548     foreach my $dev (keys %$netdev) {
2549         next if $dev !~ m/^tap([1-9]\d*)i/;
2550         my $vmid = $1;
2551         my $d = $res->{$vmid};
2552         next if !$d;
2553
2554         $d->{netout} += $netdev->{$dev}->{receive};
2555         $d->{netin} += $netdev->{$dev}->{transmit};
2556
2557         if ($full) {
2558             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2559             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2560         }
2561
2562     }
2563
2564     my $ctime = gettimeofday;
2565
2566     foreach my $vmid (keys %$list) {
2567
2568         my $d = $res->{$vmid};
2569         my $pid = $d->{pid};
2570         next if !$pid;
2571
2572         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2573         next if !$pstat; # not running
2574
2575         my $used = $pstat->{utime} + $pstat->{stime};
2576
2577         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2578
2579         if ($pstat->{vsize}) {
2580             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2581         }
2582
2583         my $old = $last_proc_pid_stat->{$pid};
2584         if (!$old) {
2585             $last_proc_pid_stat->{$pid} = {
2586                 time => $ctime,
2587                 used => $used,
2588                 cpu => 0,
2589             };
2590             next;
2591         }
2592
2593         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2594
2595         if ($dtime > 1000) {
2596             my $dutime = $used -  $old->{used};
2597
2598             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2599             $last_proc_pid_stat->{$pid} = {
2600                 time => $ctime,
2601                 used => $used,
2602                 cpu => $d->{cpu},
2603             };
2604         } else {
2605             $d->{cpu} = $old->{cpu};
2606         }
2607     }
2608
2609     return $res if !$full;
2610
2611     my $qmpclient = PVE::QMPClient->new();
2612
2613     my $ballooncb = sub {
2614         my ($vmid, $resp) = @_;
2615
2616         my $info = $resp->{'return'};
2617         return if !$info->{max_mem};
2618
2619         my $d = $res->{$vmid};
2620
2621         # use memory assigned to VM
2622         $d->{maxmem} = $info->{max_mem};
2623         $d->{balloon} = $info->{actual};
2624
2625         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2626             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2627             $d->{freemem} = $info->{free_mem};
2628         }
2629
2630         $d->{ballooninfo} = $info;
2631     };
2632
2633     my $blockstatscb = sub {
2634         my ($vmid, $resp) = @_;
2635         my $data = $resp->{'return'} || [];
2636         my $totalrdbytes = 0;
2637         my $totalwrbytes = 0;
2638
2639         for my $blockstat (@$data) {
2640             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2641             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2642
2643             $blockstat->{device} =~ s/drive-//;
2644             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2645         }
2646         $res->{$vmid}->{diskread} = $totalrdbytes;
2647         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2648     };
2649
2650     my $statuscb = sub {
2651         my ($vmid, $resp) = @_;
2652
2653         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2654         # this fails if ballon driver is not loaded, so this must be
2655         # the last commnand (following command are aborted if this fails).
2656         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2657
2658         my $status = 'unknown';
2659         if (!defined($status = $resp->{'return'}->{status})) {
2660             warn "unable to get VM status\n";
2661             return;
2662         }
2663
2664         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2665     };
2666
2667     foreach my $vmid (keys %$list) {
2668         next if $opt_vmid && ($vmid ne $opt_vmid);
2669         next if !$res->{$vmid}->{pid}; # not running
2670         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2671     }
2672
2673     $qmpclient->queue_execute(undef, 1);
2674
2675     foreach my $vmid (keys %$list) {
2676         next if $opt_vmid && ($vmid ne $opt_vmid);
2677         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2678     }
2679
2680     return $res;
2681 }
2682
2683 sub foreach_drive {
2684     my ($conf, $func, @param) = @_;
2685
2686     foreach my $ds (valid_drive_names()) {
2687         next if !defined($conf->{$ds});
2688
2689         my $drive = parse_drive($ds, $conf->{$ds});
2690         next if !$drive;
2691
2692         &$func($ds, $drive, @param);
2693     }
2694 }
2695
2696 sub foreach_volid {
2697     my ($conf, $func, @param) = @_;
2698
2699     my $volhash = {};
2700
2701     my $test_volid = sub {
2702         my ($volid, $is_cdrom) = @_;
2703
2704         return if !$volid;
2705
2706         $volhash->{$volid} = $is_cdrom || 0;
2707     };
2708
2709     foreach_drive($conf, sub {
2710         my ($ds, $drive) = @_;
2711         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2712     });
2713
2714     foreach my $snapname (keys %{$conf->{snapshots}}) {
2715         my $snap = $conf->{snapshots}->{$snapname};
2716         &$test_volid($snap->{vmstate}, 0);
2717         foreach_drive($snap, sub {
2718             my ($ds, $drive) = @_;
2719             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2720         });
2721     }
2722
2723     foreach my $volid (keys %$volhash) {
2724         &$func($volid, $volhash->{$volid}, @param);
2725     }
2726 }
2727
2728 sub vga_conf_has_spice {
2729     my ($vga) = @_;
2730
2731     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2732
2733     return $1 || 1;
2734 }
2735
2736 sub config_to_command {
2737     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2738
2739     my $cmd = [];
2740     my $globalFlags = [];
2741     my $machineFlags = [];
2742     my $rtcFlags = [];
2743     my $cpuFlags = [];
2744     my $devices = [];
2745     my $pciaddr = '';
2746     my $bridges = {};
2747     my $kvmver = kvm_user_version();
2748     my $vernum = 0; # unknown
2749     my $ostype = $conf->{ostype};
2750     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2751         $vernum = $1*1000000+$2*1000;
2752     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2753         $vernum = $1*1000000+$2*1000+$3;
2754     }
2755
2756     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2757
2758     my $have_ovz = -f '/proc/vz/vestat';
2759
2760     my $q35 = machine_type_is_q35($conf);
2761     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2762     my $machine_type = $forcemachine || $conf->{machine};
2763     my $use_old_bios_files = undef;
2764     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2765
2766     my $cpuunits = defined($conf->{cpuunits}) ?
2767             $conf->{cpuunits} : $defaults->{cpuunits};
2768
2769     push @$cmd, '/usr/bin/kvm';
2770
2771     push @$cmd, '-id', $vmid;
2772
2773     my $use_virtio = 0;
2774
2775     my $qmpsocket = qmp_socket($vmid);
2776     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2777     push @$cmd, '-mon', "chardev=qmp,mode=control";
2778
2779
2780     push @$cmd, '-pidfile' , pidfile_name($vmid);
2781
2782     push @$cmd, '-daemonize';
2783
2784     if ($conf->{smbios1}) {
2785         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2786     }
2787
2788     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2789         my $ovmfbase;
2790
2791         # prefer the OVMF_CODE variant
2792         if (-f $OVMF_CODE) {
2793             $ovmfbase = $OVMF_CODE;
2794         } elsif (-f $OVMF_IMG) {
2795             $ovmfbase = $OVMF_IMG;
2796         }
2797
2798         die "no uefi base img found\n" if !$ovmfbase;
2799         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2800         my $d = eval { PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0}) };
2801         if (defined($conf->{efidisk0}) && defined($d) && $ovmfbase eq $OVMF_CODE) {
2802             my $format = $d->{format} // 'raw';
2803             my $path;
2804             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2805             if ($storeid) {
2806                 $path = PVE::Storage::path($storecfg, $d->{file});
2807                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2808                 $format = qemu_img_format($scfg, $volname);
2809             } else {
2810                 $path = $d->{file};
2811                 $format = "raw";
2812             }
2813             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2814         } elsif ($ovmfbase eq $OVMF_CODE) {
2815             warn "using uefi without permanent efivars disk\n";
2816             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2817             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2818             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2819         } else {
2820             # if the base img is not OVMF_CODE, we do not have to bother
2821             # to create/use a vars image, since it will not be used anyway
2822             # this can only happen if someone manually deletes the OVMF_CODE image
2823             # or has an old pve-qemu-kvm version installed.
2824             # both should not happen, but we ignore it here
2825         }
2826     }
2827
2828
2829     # add usb controllers
2830     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2831     push @$devices, @usbcontrollers if @usbcontrollers;
2832     my $vga = $conf->{vga};
2833
2834     my $qxlnum = vga_conf_has_spice($vga);
2835     $vga = 'qxl' if $qxlnum;
2836
2837     if (!$vga) {
2838         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2839                                 $conf->{ostype} eq 'win7' ||
2840                                 $conf->{ostype} eq 'w2k8')) {
2841             $vga = 'std';
2842         } else {
2843             $vga = 'cirrus';
2844         }
2845     }
2846
2847     # enable absolute mouse coordinates (needed by vnc)
2848     my $tablet;
2849     if (defined($conf->{tablet})) {
2850         $tablet = $conf->{tablet};
2851     } else {
2852         $tablet = $defaults->{tablet};
2853         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2854         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2855     }
2856
2857     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2858
2859     my $kvm_off = 0;
2860     # host pci devices
2861     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2862         my $d = parse_hostpci($conf->{"hostpci$i"});
2863         next if !$d;
2864
2865         my $pcie = $d->{pcie};
2866         if($pcie){
2867             die "q35 machine model is not enabled" if !$q35;
2868             $pciaddr = print_pcie_addr("hostpci$i");
2869         }else{
2870             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2871         }
2872
2873         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2874         my $xvga = '';
2875         if ($d->{'x-vga'}) {
2876             $xvga = ',x-vga=on';
2877             $kvm_off = 1;
2878             $vga = 'none';
2879             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2880                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2881             }
2882             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2883                 $xvga = "";
2884             }
2885         }
2886         my $pcidevices = $d->{pciid};
2887         my $multifunction = 1 if @$pcidevices > 1;
2888
2889         my $j=0;
2890         foreach my $pcidevice (@$pcidevices) {
2891
2892             my $id = "hostpci$i";
2893             $id .= ".$j" if $multifunction;
2894             my $addr = $pciaddr;
2895             $addr .= ".$j" if $multifunction;
2896             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2897
2898             if($j == 0){
2899                 $devicestr .= "$rombar$xvga";
2900                 $devicestr .= ",multifunction=on" if $multifunction;
2901             }
2902
2903             push @$devices, '-device', $devicestr;
2904             $j++;
2905         }
2906     }
2907
2908     # usb devices
2909     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2910     push @$devices, @usbdevices if @usbdevices;
2911     # serial devices
2912     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2913         if (my $path = $conf->{"serial$i"}) {
2914             if ($path eq 'socket') {
2915                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2916                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2917                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2918             } else {
2919                 die "no such serial device\n" if ! -c $path;
2920                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2921                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2922             }
2923         }
2924     }
2925
2926     # parallel devices
2927     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2928         if (my $path = $conf->{"parallel$i"}) {
2929             die "no such parallel device\n" if ! -c $path;
2930             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2931             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2932             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2933         }
2934     }
2935
2936     my $vmname = $conf->{name} || "vm$vmid";
2937
2938     push @$cmd, '-name', $vmname;
2939
2940     my $sockets = 1;
2941     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2942     $sockets = $conf->{sockets} if  $conf->{sockets};
2943
2944     my $cores = $conf->{cores} || 1;
2945
2946     my $maxcpus = $sockets * $cores;
2947
2948     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2949
2950     my $allowed_vcpus = $cpuinfo->{cpus};
2951
2952     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2953         if ($allowed_vcpus < $maxcpus);
2954
2955     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2956
2957     push @$cmd, '-nodefaults';
2958
2959     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2960
2961     my $bootindex_hash = {};
2962     my $i = 1;
2963     foreach my $o (split(//, $bootorder)) {
2964         $bootindex_hash->{$o} = $i*100;
2965         $i++;
2966     }
2967
2968     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
2969
2970     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2971
2972     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2973
2974     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2975
2976     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2977         my $socket = vnc_socket($vmid);
2978         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2979     } else {
2980         push @$cmd, '-nographic';
2981     }
2982
2983     # time drift fix
2984     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2985
2986     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2987     my $useLocaltime = $conf->{localtime};
2988
2989     if ($ostype) {
2990         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2991
2992         if ($ostype =~ m/^w/) { # windows
2993             $useLocaltime = 1 if !defined($conf->{localtime});
2994
2995             # use time drift fix when acpi is enabled
2996             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2997                 $tdf = 1 if !defined($conf->{tdf});
2998             }
2999         }
3000
3001         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3002             $ostype eq 'wvista') {
3003             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3004             push @$cmd, '-no-hpet';
3005             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3006                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3007                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3008                 push @$cpuFlags , 'hv_time' if !$nokvm;
3009
3010                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3011                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3012                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3013                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3014                 }
3015
3016             } else {
3017                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3018             }
3019         }
3020
3021         if ($ostype eq 'win7' || $ostype eq 'win8') {
3022             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3023         }
3024     }
3025
3026     push @$rtcFlags, 'driftfix=slew' if $tdf;
3027
3028     if ($nokvm) {
3029         push @$machineFlags, 'accel=tcg';
3030     } else {
3031         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3032     }
3033
3034     if ($machine_type) {
3035         push @$machineFlags, "type=${machine_type}";
3036     }
3037
3038     if ($conf->{startdate}) {
3039         push @$rtcFlags, "base=$conf->{startdate}";
3040     } elsif ($useLocaltime) {
3041         push @$rtcFlags, 'base=localtime';
3042     }
3043
3044     my $cpu = $nokvm ? "qemu64" : "kvm64";
3045     if (my $cputype = $conf->{cpu}) {
3046         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3047             or die "Cannot parse cpu description: $cputype\n";
3048         $cpu = $cpuconf->{cputype};
3049         $kvm_off = 1 if $cpuconf->{hidden};
3050     }
3051
3052     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3053
3054     push @$cpuFlags , '-x2apic'
3055         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3056
3057     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3058
3059     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3060
3061     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3062
3063         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3064         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3065     }
3066
3067     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3068
3069     push @$cpuFlags, 'kvm=off' if $kvm_off;
3070
3071     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3072         die "internal error"; # should not happen
3073
3074     push @$cpuFlags, "vendor=${cpu_vendor}"
3075         if $cpu_vendor ne 'default';
3076
3077     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3078
3079     push @$cmd, '-cpu', $cpu;
3080
3081     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3082
3083     push @$cmd, '-S' if $conf->{freeze};
3084
3085     # set keyboard layout
3086     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3087     push @$cmd, '-k', $kb if $kb;
3088
3089     # enable sound
3090     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3091     #push @$cmd, '-soundhw', 'es1370';
3092     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3093
3094     if($conf->{agent}) {
3095         my $qgasocket = qmp_socket($vmid, 1);
3096         my $pciaddr = print_pci_addr("qga0", $bridges);
3097         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3098         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3099         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3100     }
3101
3102     my $spice_port;
3103
3104     if ($qxlnum) {
3105         if ($qxlnum > 1) {
3106             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3107                 for(my $i = 1; $i < $qxlnum; $i++){
3108                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3109                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3110                 }
3111             } else {
3112                 # assume other OS works like Linux
3113                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3114                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3115             }
3116         }
3117
3118         my $pciaddr = print_pci_addr("spice", $bridges);
3119
3120         my $nodename = PVE::INotify::nodename();
3121         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3122         $spice_port = PVE::Tools::next_spice_port($pfamily);
3123
3124         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3125
3126         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3127         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3128         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3129     }
3130
3131     # enable balloon by default, unless explicitly disabled
3132     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3133         $pciaddr = print_pci_addr("balloon0", $bridges);
3134         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3135     }
3136
3137     if ($conf->{watchdog}) {
3138         my $wdopts = parse_watchdog($conf->{watchdog});
3139         $pciaddr = print_pci_addr("watchdog", $bridges);
3140         my $watchdog = $wdopts->{model} || 'i6300esb';
3141         push @$devices, '-device', "$watchdog$pciaddr";
3142         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3143     }
3144
3145     my $vollist = [];
3146     my $scsicontroller = {};
3147     my $ahcicontroller = {};
3148     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3149
3150     # Add iscsi initiator name if available
3151     if (my $initiator = get_initiator_name()) {
3152         push @$devices, '-iscsi', "initiator-name=$initiator";
3153     }
3154
3155     foreach_drive($conf, sub {
3156         my ($ds, $drive) = @_;
3157
3158         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3159             push @$vollist, $drive->{file};
3160         }
3161
3162         $use_virtio = 1 if $ds =~ m/^virtio/;
3163
3164         if (drive_is_cdrom ($drive)) {
3165             if ($bootindex_hash->{d}) {
3166                 $drive->{bootindex} = $bootindex_hash->{d};
3167                 $bootindex_hash->{d} += 1;
3168             }
3169         } else {
3170             if ($bootindex_hash->{c}) {
3171                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3172                 $bootindex_hash->{c} += 1;
3173             }
3174         }
3175
3176         if($drive->{interface} eq 'virtio'){
3177            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3178         }
3179
3180         if ($drive->{interface} eq 'scsi') {
3181
3182             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3183
3184             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3185             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3186
3187             my $iothread = '';
3188             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3189                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3190                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3191             } elsif ($drive->{iothread}) {
3192                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3193             }
3194
3195             my $queues = '';
3196             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3197                 $queues = ",num_queues=$drive->{queues}";
3198             }
3199
3200             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3201             $scsicontroller->{$controller}=1;
3202         }
3203
3204         if ($drive->{interface} eq 'sata') {
3205            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3206            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3207            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3208            $ahcicontroller->{$controller}=1;
3209         }
3210
3211         if ($drive->{interface} eq 'efidisk') {
3212             # this will be added somewhere else
3213             return;
3214         }
3215
3216         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3217         push @$devices, '-drive',$drive_cmd;
3218         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3219     });
3220
3221     for (my $i = 0; $i < $MAX_NETS; $i++) {
3222          next if !$conf->{"net$i"};
3223          my $d = parse_net($conf->{"net$i"});
3224          next if !$d;
3225
3226          $use_virtio = 1 if $d->{model} eq 'virtio';
3227
3228          if ($bootindex_hash->{n}) {
3229             $d->{bootindex} = $bootindex_hash->{n};
3230             $bootindex_hash->{n} += 1;
3231          }
3232
3233          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3234          push @$devices, '-netdev', $netdevfull;
3235
3236          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3237          push @$devices, '-device', $netdevicefull;
3238     }
3239
3240     if (!$q35) {
3241         # add pci bridges
3242         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3243            $bridges->{1} = 1;
3244            $bridges->{2} = 1;
3245         }
3246
3247         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3248
3249         while (my ($k, $v) = each %$bridges) {
3250             $pciaddr = print_pci_addr("pci.$k");
3251             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3252         }
3253     }
3254
3255     # add custom args
3256     if ($conf->{args}) {
3257         my $aa = PVE::Tools::split_args($conf->{args});
3258         push @$cmd, @$aa;
3259     }
3260
3261     push @$cmd, @$devices;
3262     push @$cmd, '-rtc', join(',', @$rtcFlags)
3263         if scalar(@$rtcFlags);
3264     push @$cmd, '-machine', join(',', @$machineFlags)
3265         if scalar(@$machineFlags);
3266     push @$cmd, '-global', join(',', @$globalFlags)
3267         if scalar(@$globalFlags);
3268
3269     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3270 }
3271
3272 sub vnc_socket {
3273     my ($vmid) = @_;
3274     return "${var_run_tmpdir}/$vmid.vnc";
3275 }
3276
3277 sub spice_port {
3278     my ($vmid) = @_;
3279
3280     my $res = vm_mon_cmd($vmid, 'query-spice');
3281
3282     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3283 }
3284
3285 sub qmp_socket {
3286     my ($vmid, $qga) = @_;
3287     my $sockettype = $qga ? 'qga' : 'qmp';
3288     return "${var_run_tmpdir}/$vmid.$sockettype";
3289 }
3290
3291 sub pidfile_name {
3292     my ($vmid) = @_;
3293     return "${var_run_tmpdir}/$vmid.pid";
3294 }
3295
3296 sub vm_devices_list {
3297     my ($vmid) = @_;
3298
3299     my $res = vm_mon_cmd($vmid, 'query-pci');
3300     my $devices = {};
3301     foreach my $pcibus (@$res) {
3302         foreach my $device (@{$pcibus->{devices}}) {
3303             next if !$device->{'qdev_id'};
3304             if ($device->{'pci_bridge'}) {
3305                 $devices->{$device->{'qdev_id'}} = 1;
3306                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3307                     next if !$bridge_device->{'qdev_id'};
3308                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3309                     $devices->{$device->{'qdev_id'}}++;
3310                 }
3311             } else {
3312                 $devices->{$device->{'qdev_id'}} = 1;
3313             }
3314         }
3315     }
3316
3317     my $resblock = vm_mon_cmd($vmid, 'query-block');
3318     foreach my $block (@$resblock) {
3319         if($block->{device} =~ m/^drive-(\S+)/){
3320                 $devices->{$1} = 1;
3321         }
3322     }
3323
3324     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3325     foreach my $mice (@$resmice) {
3326         if ($mice->{name} eq 'QEMU HID Tablet') {
3327             $devices->{tablet} = 1;
3328             last;
3329         }
3330     }
3331
3332     # for usb devices there is no query-usb
3333     # but we can iterate over the entries in
3334     # qom-list path=/machine/peripheral
3335     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3336     foreach my $per (@$resperipheral) {
3337         if ($per->{name} =~ m/^usb\d+$/) {
3338             $devices->{$per->{name}} = 1;
3339         }
3340     }
3341
3342     return $devices;
3343 }
3344
3345 sub vm_deviceplug {
3346     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3347
3348     my $q35 = machine_type_is_q35($conf);
3349
3350     my $devices_list = vm_devices_list($vmid);
3351     return 1 if defined($devices_list->{$deviceid});
3352
3353     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3354
3355     if ($deviceid eq 'tablet') {
3356
3357         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3358
3359     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3360
3361         die "usb hotplug currently not reliable\n";
3362         # since we can't reliably hot unplug all added usb devices
3363         # and usb passthrough disables live migration
3364         # we disable usb hotplugging for now
3365         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3366
3367     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3368
3369         qemu_iothread_add($vmid, $deviceid, $device);
3370
3371         qemu_driveadd($storecfg, $vmid, $device);
3372         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3373
3374         qemu_deviceadd($vmid, $devicefull);
3375         eval { qemu_deviceaddverify($vmid, $deviceid); };
3376         if (my $err = $@) {
3377             eval { qemu_drivedel($vmid, $deviceid); };
3378             warn $@ if $@;
3379             die $err;
3380         }
3381
3382     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3383
3384
3385         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3386         my $pciaddr = print_pci_addr($deviceid);
3387         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3388
3389         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3390
3391         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3392             qemu_iothread_add($vmid, $deviceid, $device);
3393             $devicefull .= ",iothread=iothread-$deviceid";
3394         }
3395
3396         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3397             $devicefull .= ",num_queues=$device->{queues}";
3398         }
3399
3400         qemu_deviceadd($vmid, $devicefull);
3401         qemu_deviceaddverify($vmid, $deviceid);
3402
3403     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3404
3405         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3406         qemu_driveadd($storecfg, $vmid, $device);
3407
3408         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3409         eval { qemu_deviceadd($vmid, $devicefull); };
3410         if (my $err = $@) {
3411             eval { qemu_drivedel($vmid, $deviceid); };
3412             warn $@ if $@;
3413             die $err;
3414         }
3415
3416     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3417
3418         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3419
3420         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3421         my $use_old_bios_files = undef;
3422         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3423
3424         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3425         qemu_deviceadd($vmid, $netdevicefull);
3426         eval { qemu_deviceaddverify($vmid, $deviceid); };
3427         if (my $err = $@) {
3428             eval { qemu_netdevdel($vmid, $deviceid); };
3429             warn $@ if $@;
3430             die $err;
3431         }
3432
3433     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3434
3435         my $bridgeid = $2;
3436         my $pciaddr = print_pci_addr($deviceid);
3437         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3438
3439         qemu_deviceadd($vmid, $devicefull);
3440         qemu_deviceaddverify($vmid, $deviceid);
3441
3442     } else {
3443         die "can't hotplug device '$deviceid'\n";
3444     }
3445
3446     return 1;
3447 }
3448
3449 # fixme: this should raise exceptions on error!
3450 sub vm_deviceunplug {
3451     my ($vmid, $conf, $deviceid) = @_;
3452
3453     my $devices_list = vm_devices_list($vmid);
3454     return 1 if !defined($devices_list->{$deviceid});
3455
3456     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3457
3458     if ($deviceid eq 'tablet') {
3459
3460         qemu_devicedel($vmid, $deviceid);
3461
3462     } elsif ($deviceid =~ m/^usb\d+$/) {
3463
3464         die "usb hotplug currently not reliable\n";
3465         # when unplugging usb devices this way,
3466         # there may be remaining usb controllers/hubs
3467         # so we disable it for now
3468         qemu_devicedel($vmid, $deviceid);
3469         qemu_devicedelverify($vmid, $deviceid);
3470
3471     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3472
3473         qemu_devicedel($vmid, $deviceid);
3474         qemu_devicedelverify($vmid, $deviceid);
3475         qemu_drivedel($vmid, $deviceid);
3476         qemu_iothread_del($conf, $vmid, $deviceid);
3477
3478     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3479
3480         qemu_devicedel($vmid, $deviceid);
3481         qemu_devicedelverify($vmid, $deviceid);
3482         qemu_iothread_del($conf, $vmid, $deviceid);
3483
3484     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3485
3486         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3487         my $device = parse_drive($deviceid, $conf->{$deviceid});
3488         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3489
3490         qemu_devicedel($vmid, $deviceid);
3491         qemu_drivedel($vmid, $deviceid);
3492         qemu_deletescsihw($conf, $vmid, $deviceid);
3493
3494     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3495
3496         qemu_devicedel($vmid, $deviceid);
3497         qemu_devicedelverify($vmid, $deviceid);
3498         qemu_netdevdel($vmid, $deviceid);
3499
3500     } else {
3501         die "can't unplug device '$deviceid'\n";
3502     }
3503
3504     return 1;
3505 }
3506
3507 sub qemu_deviceadd {
3508     my ($vmid, $devicefull) = @_;
3509
3510     $devicefull = "driver=".$devicefull;
3511     my %options =  split(/[=,]/, $devicefull);
3512
3513     vm_mon_cmd($vmid, "device_add" , %options);
3514 }
3515
3516 sub qemu_devicedel {
3517     my ($vmid, $deviceid) = @_;
3518
3519     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3520 }
3521
3522 sub qemu_iothread_add {
3523     my($vmid, $deviceid, $device) = @_;
3524
3525     if ($device->{iothread}) {
3526         my $iothreads = vm_iothreads_list($vmid);
3527         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3528     }
3529 }
3530
3531 sub qemu_iothread_del {
3532     my($conf, $vmid, $deviceid) = @_;
3533
3534     my $device = parse_drive($deviceid, $conf->{$deviceid});
3535     if ($device->{iothread}) {
3536         my $iothreads = vm_iothreads_list($vmid);
3537         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3538     }
3539 }
3540
3541 sub qemu_objectadd {
3542     my($vmid, $objectid, $qomtype) = @_;
3543
3544     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3545
3546     return 1;
3547 }
3548
3549 sub qemu_objectdel {
3550     my($vmid, $objectid) = @_;
3551
3552     vm_mon_cmd($vmid, "object-del", id => $objectid);
3553
3554     return 1;
3555 }
3556
3557 sub qemu_driveadd {
3558     my ($storecfg, $vmid, $device) = @_;
3559
3560     my $drive = print_drive_full($storecfg, $vmid, $device);
3561     $drive =~ s/\\/\\\\/g;
3562     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3563
3564     # If the command succeeds qemu prints: "OK"
3565     return 1 if $ret =~ m/OK/s;
3566
3567     die "adding drive failed: $ret\n";
3568 }
3569
3570 sub qemu_drivedel {
3571     my($vmid, $deviceid) = @_;
3572
3573     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3574     $ret =~ s/^\s+//;
3575
3576     return 1 if $ret eq "";
3577
3578     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3579     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3580
3581     die "deleting drive $deviceid failed : $ret\n";
3582 }
3583
3584 sub qemu_deviceaddverify {
3585     my ($vmid, $deviceid) = @_;
3586
3587     for (my $i = 0; $i <= 5; $i++) {
3588          my $devices_list = vm_devices_list($vmid);
3589          return 1 if defined($devices_list->{$deviceid});
3590          sleep 1;
3591     }
3592
3593     die "error on hotplug device '$deviceid'\n";
3594 }
3595
3596
3597 sub qemu_devicedelverify {
3598     my ($vmid, $deviceid) = @_;
3599
3600     # need to verify that the device is correctly removed as device_del
3601     # is async and empty return is not reliable
3602
3603     for (my $i = 0; $i <= 5; $i++) {
3604          my $devices_list = vm_devices_list($vmid);
3605          return 1 if !defined($devices_list->{$deviceid});
3606          sleep 1;
3607     }
3608
3609     die "error on hot-unplugging device '$deviceid'\n";
3610 }
3611
3612 sub qemu_findorcreatescsihw {
3613     my ($storecfg, $conf, $vmid, $device) = @_;
3614
3615     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3616
3617     my $scsihwid="$controller_prefix$controller";
3618     my $devices_list = vm_devices_list($vmid);
3619
3620     if(!defined($devices_list->{$scsihwid})) {
3621         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3622     }
3623
3624     return 1;
3625 }
3626
3627 sub qemu_deletescsihw {
3628     my ($conf, $vmid, $opt) = @_;
3629
3630     my $device = parse_drive($opt, $conf->{$opt});
3631
3632     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3633         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3634         return 1;
3635     }
3636
3637     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3638
3639     my $devices_list = vm_devices_list($vmid);
3640     foreach my $opt (keys %{$devices_list}) {
3641         if (PVE::QemuServer::is_valid_drivename($opt)) {
3642             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3643             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3644                 return 1;
3645             }
3646         }
3647     }
3648
3649     my $scsihwid="scsihw$controller";
3650
3651     vm_deviceunplug($vmid, $conf, $scsihwid);
3652
3653     return 1;
3654 }
3655
3656 sub qemu_add_pci_bridge {
3657     my ($storecfg, $conf, $vmid, $device) = @_;
3658
3659     my $bridges = {};
3660
3661     my $bridgeid;
3662
3663     print_pci_addr($device, $bridges);
3664
3665     while (my ($k, $v) = each %$bridges) {
3666         $bridgeid = $k;
3667     }
3668     return 1 if !defined($bridgeid) || $bridgeid < 1;
3669
3670     my $bridge = "pci.$bridgeid";
3671     my $devices_list = vm_devices_list($vmid);
3672
3673     if (!defined($devices_list->{$bridge})) {
3674         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3675     }
3676
3677     return 1;
3678 }
3679
3680 sub qemu_set_link_status {
3681     my ($vmid, $device, $up) = @_;
3682
3683     vm_mon_cmd($vmid, "set_link", name => $device,
3684                up => $up ? JSON::true : JSON::false);
3685 }
3686
3687 sub qemu_netdevadd {
3688     my ($vmid, $conf, $device, $deviceid) = @_;
3689
3690     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3691     my %options =  split(/[=,]/, $netdev);
3692
3693     vm_mon_cmd($vmid, "netdev_add",  %options);
3694     return 1;
3695 }
3696
3697 sub qemu_netdevdel {
3698     my ($vmid, $deviceid) = @_;
3699
3700     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3701 }
3702
3703 sub qemu_usb_hotplug {
3704     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3705
3706     return if !$device;
3707
3708     # remove the old one first
3709     vm_deviceunplug($vmid, $conf, $deviceid);
3710
3711     # check if xhci controller is necessary and available
3712     if ($device->{usb3}) {
3713
3714         my $devicelist = vm_devices_list($vmid);
3715
3716         if (!$devicelist->{xhci}) {
3717             my $pciaddr = print_pci_addr("xhci");
3718             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3719         }
3720     }
3721     my $d = parse_usb_device($device->{host});
3722     $d->{usb3} = $device->{usb3};
3723
3724     # add the new one
3725     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3726 }
3727
3728 sub qemu_cpu_hotplug {
3729     my ($vmid, $conf, $vcpus) = @_;
3730
3731     my $sockets = 1;
3732     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3733     $sockets = $conf->{sockets} if  $conf->{sockets};
3734     my $cores = $conf->{cores} || 1;
3735     my $maxcpus = $sockets * $cores;
3736
3737     $vcpus = $maxcpus if !$vcpus;
3738
3739     die "you can't add more vcpus than maxcpus\n"
3740         if $vcpus > $maxcpus;
3741
3742     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3743     die "online cpu unplug is not yet possible\n"
3744         if $vcpus < $currentvcpus;
3745
3746     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3747     die "vcpus in running vm is different than configuration\n"
3748         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3749
3750     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3751         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3752     }
3753 }
3754
3755 sub qemu_block_set_io_throttle {
3756     my ($vmid, $deviceid,
3757         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3758         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3759
3760     return if !check_running($vmid) ;
3761
3762     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3763         bps => int($bps),
3764         bps_rd => int($bps_rd),
3765         bps_wr => int($bps_wr),
3766         iops => int($iops),
3767         iops_rd => int($iops_rd),
3768         iops_wr => int($iops_wr),
3769         bps_max => int($bps_max),
3770         bps_rd_max => int($bps_rd_max),
3771         bps_wr_max => int($bps_wr_max),
3772         iops_max => int($iops_max),
3773         iops_rd_max => int($iops_rd_max),
3774         iops_wr_max => int($iops_wr_max)
3775     );
3776
3777 }
3778
3779 # old code, only used to shutdown old VM after update
3780 sub __read_avail {
3781     my ($fh, $timeout) = @_;
3782
3783     my $sel = new IO::Select;
3784     $sel->add($fh);
3785
3786     my $res = '';
3787     my $buf;
3788
3789     my @ready;
3790     while (scalar (@ready = $sel->can_read($timeout))) {
3791         my $count;
3792         if ($count = $fh->sysread($buf, 8192)) {
3793             if ($buf =~ /^(.*)\(qemu\) $/s) {
3794                 $res .= $1;
3795                 last;
3796             } else {
3797                 $res .= $buf;
3798             }
3799         } else {
3800             if (!defined($count)) {
3801                 die "$!\n";
3802             }
3803             last;
3804         }
3805     }
3806
3807     die "monitor read timeout\n" if !scalar(@ready);
3808
3809     return $res;
3810 }
3811
3812 # old code, only used to shutdown old VM after update
3813 sub vm_monitor_command {
3814     my ($vmid, $cmdstr, $nocheck) = @_;
3815
3816     my $res;
3817
3818     eval {
3819         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3820
3821         my $sname = "${var_run_tmpdir}/$vmid.mon";
3822
3823         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3824             die "unable to connect to VM $vmid socket - $!\n";
3825
3826         my $timeout = 3;
3827
3828         # hack: migrate sometime blocks the monitor (when migrate_downtime
3829         # is set)
3830         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3831             $timeout = 60*60; # 1 hour
3832         }
3833
3834         # read banner;
3835         my $data = __read_avail($sock, $timeout);
3836
3837         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3838             die "got unexpected qemu monitor banner\n";
3839         }
3840
3841         my $sel = new IO::Select;
3842         $sel->add($sock);
3843
3844         if (!scalar(my @ready = $sel->can_write($timeout))) {
3845             die "monitor write error - timeout";
3846         }
3847
3848         my $fullcmd = "$cmdstr\r";
3849
3850         # syslog('info', "VM $vmid monitor command: $cmdstr");
3851
3852         my $b;
3853         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3854             die "monitor write error - $!";
3855         }
3856
3857         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3858
3859         $timeout = 20;
3860
3861         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3862             $timeout = 60*60; # 1 hour
3863         } elsif ($cmdstr =~ m/^(eject|change)/) {
3864             $timeout = 60; # note: cdrom mount command is slow
3865         }
3866         if ($res = __read_avail($sock, $timeout)) {
3867
3868             my @lines = split("\r?\n", $res);
3869
3870             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3871
3872             $res = join("\n", @lines);
3873             $res .= "\n";
3874         }
3875     };
3876
3877     my $err = $@;
3878
3879     if ($err) {
3880         syslog("err", "VM $vmid monitor command failed - $err");
3881         die $err;
3882     }
3883
3884     return $res;
3885 }
3886
3887 sub qemu_block_resize {
3888     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3889
3890     my $running = check_running($vmid);
3891
3892     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3893
3894     return if !$running;
3895
3896     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3897
3898 }
3899
3900 sub qemu_volume_snapshot {
3901     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3902
3903     my $running = check_running($vmid);
3904
3905     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3906         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3907     } else {
3908         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3909     }
3910 }
3911
3912 sub qemu_volume_snapshot_delete {
3913     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3914
3915     my $running = check_running($vmid);
3916
3917     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3918
3919     return if !$running;
3920
3921     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3922 }
3923
3924 sub set_migration_caps {
3925     my ($vmid) = @_;
3926
3927     my $cap_ref = [];
3928
3929     my $enabled_cap = {
3930         "auto-converge" => 1,
3931         "xbzrle" => 1,
3932         "x-rdma-pin-all" => 0,
3933         "zero-blocks" => 0,
3934         "compress" => 0
3935     };
3936
3937     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3938
3939     for my $supported_capability (@$supported_capabilities) {
3940         push @$cap_ref, {
3941             capability => $supported_capability->{capability},
3942             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3943         };
3944     }
3945
3946     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3947 }
3948
3949 my $fast_plug_option = {
3950     'lock' => 1,
3951     'name' => 1,
3952     'onboot' => 1,
3953     'shares' => 1,
3954     'startup' => 1,
3955     'description' => 1,
3956     'protection' => 1,
3957 };
3958
3959 # hotplug changes in [PENDING]
3960 # $selection hash can be used to only apply specified options, for
3961 # example: { cores => 1 } (only apply changed 'cores')
3962 # $errors ref is used to return error messages
3963 sub vmconfig_hotplug_pending {
3964     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3965
3966     my $defaults = load_defaults();
3967
3968     # commit values which do not have any impact on running VM first
3969     # Note: those option cannot raise errors, we we do not care about
3970     # $selection and always apply them.
3971
3972     my $add_error = sub {
3973         my ($opt, $msg) = @_;
3974         $errors->{$opt} = "hotplug problem - $msg";
3975     };
3976
3977     my $changes = 0;
3978     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3979         if ($fast_plug_option->{$opt}) {
3980             $conf->{$opt} = $conf->{pending}->{$opt};
3981             delete $conf->{pending}->{$opt};
3982             $changes = 1;
3983         }
3984     }
3985
3986     if ($changes) {
3987         PVE::QemuConfig->write_config($vmid, $conf);
3988         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3989     }
3990
3991     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3992
3993     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3994     while (my ($opt, $force) = each %$pending_delete_hash) {
3995         next if $selection && !$selection->{$opt};
3996         eval {
3997             if ($opt eq 'hotplug') {
3998                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3999             } elsif ($opt eq 'tablet') {
4000                 die "skip\n" if !$hotplug_features->{usb};
4001                 if ($defaults->{tablet}) {
4002                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4003                 } else {
4004                     vm_deviceunplug($vmid, $conf, $opt);
4005                 }
4006             } elsif ($opt =~ m/^usb\d+/) {
4007                 die "skip\n";
4008                 # since we cannot reliably hot unplug usb devices
4009                 # we are disabling it
4010                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4011                 vm_deviceunplug($vmid, $conf, $opt);
4012             } elsif ($opt eq 'vcpus') {
4013                 die "skip\n" if !$hotplug_features->{cpu};
4014                 qemu_cpu_hotplug($vmid, $conf, undef);
4015             } elsif ($opt eq 'balloon') {
4016                 # enable balloon device is not hotpluggable
4017                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4018             } elsif ($fast_plug_option->{$opt}) {
4019                 # do nothing
4020             } elsif ($opt =~ m/^net(\d+)$/) {
4021                 die "skip\n" if !$hotplug_features->{network};
4022                 vm_deviceunplug($vmid, $conf, $opt);
4023             } elsif (is_valid_drivename($opt)) {
4024                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4025                 vm_deviceunplug($vmid, $conf, $opt);
4026                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4027             } elsif ($opt =~ m/^memory$/) {
4028                 die "skip\n" if !$hotplug_features->{memory};
4029                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4030             } elsif ($opt eq 'cpuunits') {
4031                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4032             } elsif ($opt eq 'cpulimit') {
4033                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4034             } else {
4035                 die "skip\n";
4036             }
4037         };
4038         if (my $err = $@) {
4039             &$add_error($opt, $err) if $err ne "skip\n";
4040         } else {
4041             # save new config if hotplug was successful
4042             delete $conf->{$opt};
4043             vmconfig_undelete_pending_option($conf, $opt);
4044             PVE::QemuConfig->write_config($vmid, $conf);
4045             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4046         }
4047     }
4048
4049     foreach my $opt (keys %{$conf->{pending}}) {
4050         next if $selection && !$selection->{$opt};
4051         my $value = $conf->{pending}->{$opt};
4052         eval {
4053             if ($opt eq 'hotplug') {
4054                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4055             } elsif ($opt eq 'tablet') {
4056                 die "skip\n" if !$hotplug_features->{usb};
4057                 if ($value == 1) {
4058                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4059                 } elsif ($value == 0) {
4060                     vm_deviceunplug($vmid, $conf, $opt);
4061                 }
4062             } elsif ($opt =~ m/^usb\d+$/) {
4063                 die "skip\n";
4064                 # since we cannot reliably hot unplug usb devices
4065                 # we are disabling it
4066                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4067                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4068                 die "skip\n" if !$d;
4069                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4070             } elsif ($opt eq 'vcpus') {
4071                 die "skip\n" if !$hotplug_features->{cpu};
4072                 qemu_cpu_hotplug($vmid, $conf, $value);
4073             } elsif ($opt eq 'balloon') {
4074                 # enable/disable balloning device is not hotpluggable
4075                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4076                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4077                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4078
4079                 # allow manual ballooning if shares is set to zero
4080                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4081                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4082                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4083                 }
4084             } elsif ($opt =~ m/^net(\d+)$/) {
4085                 # some changes can be done without hotplug
4086                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4087                                     $vmid, $opt, $value);
4088             } elsif (is_valid_drivename($opt)) {
4089                 # some changes can be done without hotplug
4090                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4091                                      $vmid, $opt, $value, 1);
4092             } elsif ($opt =~ m/^memory$/) { #dimms
4093                 die "skip\n" if !$hotplug_features->{memory};
4094                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4095             } elsif ($opt eq 'cpuunits') {
4096                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4097             } elsif ($opt eq 'cpulimit') {
4098                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4099                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4100             } else {
4101                 die "skip\n";  # skip non-hot-pluggable options
4102             }
4103         };
4104         if (my $err = $@) {
4105             &$add_error($opt, $err) if $err ne "skip\n";
4106         } else {
4107             # save new config if hotplug was successful
4108             $conf->{$opt} = $value;
4109             delete $conf->{pending}->{$opt};
4110             PVE::QemuConfig->write_config($vmid, $conf);
4111             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4112         }
4113     }
4114 }
4115
4116 sub try_deallocate_drive {
4117     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4118
4119     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4120         my $volid = $drive->{file};
4121         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4122             my $sid = PVE::Storage::parse_volume_id($volid);
4123             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4124
4125             # check if the disk is really unused
4126             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4127                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4128             PVE::Storage::vdisk_free($storecfg, $volid);
4129             return 1;
4130         } else {
4131             # If vm is not owner of this disk remove from config
4132             return 1;
4133         }
4134     }
4135
4136     return undef;
4137 }
4138
4139 sub vmconfig_delete_or_detach_drive {
4140     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4141
4142     my $drive = parse_drive($opt, $conf->{$opt});
4143
4144     my $rpcenv = PVE::RPCEnvironment::get();
4145     my $authuser = $rpcenv->get_user();
4146
4147     if ($force) {
4148         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4149         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4150     } else {
4151         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4152     }
4153 }
4154
4155 sub vmconfig_apply_pending {
4156     my ($vmid, $conf, $storecfg) = @_;
4157
4158     # cold plug
4159
4160     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4161     while (my ($opt, $force) = each %$pending_delete_hash) {
4162         die "internal error" if $opt =~ m/^unused/;
4163         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4164         if (!defined($conf->{$opt})) {
4165             vmconfig_undelete_pending_option($conf, $opt);
4166             PVE::QemuConfig->write_config($vmid, $conf);
4167         } elsif (is_valid_drivename($opt)) {
4168             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4169             vmconfig_undelete_pending_option($conf, $opt);
4170             delete $conf->{$opt};
4171             PVE::QemuConfig->write_config($vmid, $conf);
4172         } else {
4173             vmconfig_undelete_pending_option($conf, $opt);
4174             delete $conf->{$opt};
4175             PVE::QemuConfig->write_config($vmid, $conf);
4176         }
4177     }
4178
4179     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4180
4181     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4182         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4183
4184         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4185             # skip if nothing changed
4186         } elsif (is_valid_drivename($opt)) {
4187             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4188                 if defined($conf->{$opt});
4189             $conf->{$opt} = $conf->{pending}->{$opt};
4190         } else {
4191             $conf->{$opt} = $conf->{pending}->{$opt};
4192         }
4193
4194         delete $conf->{pending}->{$opt};
4195         PVE::QemuConfig->write_config($vmid, $conf);
4196     }
4197 }
4198
4199 my $safe_num_ne = sub {
4200     my ($a, $b) = @_;
4201
4202     return 0 if !defined($a) && !defined($b);
4203     return 1 if !defined($a);
4204     return 1 if !defined($b);
4205
4206     return $a != $b;
4207 };
4208
4209 my $safe_string_ne = sub {
4210     my ($a, $b) = @_;
4211
4212     return 0 if !defined($a) && !defined($b);
4213     return 1 if !defined($a);
4214     return 1 if !defined($b);
4215
4216     return $a ne $b;
4217 };
4218
4219 sub vmconfig_update_net {
4220     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4221
4222     my $newnet = parse_net($value);
4223
4224     if ($conf->{$opt}) {
4225         my $oldnet = parse_net($conf->{$opt});
4226
4227         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4228             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4229             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4230             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4231
4232             # for non online change, we try to hot-unplug
4233             die "skip\n" if !$hotplug;
4234             vm_deviceunplug($vmid, $conf, $opt);
4235         } else {
4236
4237             die "internal error" if $opt !~ m/net(\d+)/;
4238             my $iface = "tap${vmid}i$1";
4239
4240             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4241                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4242                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4243                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4244                 PVE::Network::tap_unplug($iface);
4245                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4246             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4247                 # Rate can be applied on its own but any change above needs to
4248                 # include the rate in tap_plug since OVS resets everything.
4249                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4250             }
4251
4252             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4253                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4254             }
4255
4256             return 1;
4257         }
4258     }
4259
4260     if ($hotplug) {
4261         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4262     } else {
4263         die "skip\n";
4264     }
4265 }
4266
4267 sub vmconfig_update_disk {
4268     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4269
4270     # fixme: do we need force?
4271
4272     my $drive = parse_drive($opt, $value);
4273
4274     if ($conf->{$opt}) {
4275
4276         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4277
4278             my $media = $drive->{media} || 'disk';
4279             my $oldmedia = $old_drive->{media} || 'disk';
4280             die "unable to change media type\n" if $media ne $oldmedia;
4281
4282             if (!drive_is_cdrom($old_drive)) {
4283
4284                 if ($drive->{file} ne $old_drive->{file}) {
4285
4286                     die "skip\n" if !$hotplug;
4287
4288                     # unplug and register as unused
4289                     vm_deviceunplug($vmid, $conf, $opt);
4290                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4291
4292                 } else {
4293                     # update existing disk
4294
4295                     # skip non hotpluggable value
4296                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4297                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4298                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4299                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4300                         die "skip\n";
4301                     }
4302
4303                     # apply throttle
4304                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4305                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4306                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4307                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4308                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4309                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4310                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4311                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4312                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4313                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4314                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4315                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4316
4317                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4318                                                    ($drive->{mbps} || 0)*1024*1024,
4319                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4320                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4321                                                    $drive->{iops} || 0,
4322                                                    $drive->{iops_rd} || 0,
4323                                                    $drive->{iops_wr} || 0,
4324                                                    ($drive->{mbps_max} || 0)*1024*1024,
4325                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4326                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4327                                                    $drive->{iops_max} || 0,
4328                                                    $drive->{iops_rd_max} || 0,
4329                                                    $drive->{iops_wr_max} || 0);
4330
4331                     }
4332
4333                     return 1;
4334                 }
4335
4336             } else { # cdrom
4337
4338                 if ($drive->{file} eq 'none') {
4339                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4340                 } else {
4341                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4342                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4343                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4344                 }
4345
4346                 return 1;
4347             }
4348         }
4349     }
4350
4351     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4352     # hotplug new disks
4353     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4354     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4355 }
4356
4357 sub vm_start {
4358     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4359         $forcemachine, $spice_ticket) = @_;
4360
4361     PVE::QemuConfig->lock_config($vmid, sub {
4362         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4363
4364         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4365
4366         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4367
4368         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4369
4370         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4371             vmconfig_apply_pending($vmid, $conf, $storecfg);
4372             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4373         }
4374
4375         my $defaults = load_defaults();
4376
4377         # set environment variable useful inside network script
4378         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4379
4380         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4381
4382         my $migrate_port = 0;
4383         my $migrate_uri;
4384         if ($statefile) {
4385             if ($statefile eq 'tcp') {
4386                 my $localip = "localhost";
4387                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4388                 my $nodename = PVE::INotify::nodename();
4389                 if ($datacenterconf->{migration_unsecure}) {
4390                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4391                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4392                 }
4393                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4394                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4395                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4396                 push @$cmd, '-incoming', $migrate_uri;
4397                 push @$cmd, '-S';
4398
4399             } elsif ($statefile eq 'unix') {
4400                 # should be default for secure migrations as a ssh TCP forward
4401                 # tunnel is not deterministic reliable ready and fails regurarly
4402                 # to set up in time, so use UNIX socket forwards
4403                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4404                 unlink $socket_addr;
4405
4406                 $migrate_uri = "unix:$socket_addr";
4407
4408                 push @$cmd, '-incoming', $migrate_uri;
4409                 push @$cmd, '-S';
4410
4411             } else {
4412                 push @$cmd, '-loadstate', $statefile;
4413             }
4414         } elsif ($paused) {
4415             push @$cmd, '-S';
4416         }
4417
4418         # host pci devices
4419         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4420           my $d = parse_hostpci($conf->{"hostpci$i"});
4421           next if !$d;
4422           my $pcidevices = $d->{pciid};
4423           foreach my $pcidevice (@$pcidevices) {
4424                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4425
4426                 my $info = pci_device_info("0000:$pciid");
4427                 die "IOMMU not present\n" if !check_iommu_support();
4428                 die "no pci device info for device '$pciid'\n" if !$info;
4429                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4430                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4431           }
4432         }
4433
4434         PVE::Storage::activate_volumes($storecfg, $vollist);
4435
4436         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4437             my $cmd = [];
4438             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4439             eval  { run_command($cmd); };
4440         }
4441
4442         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4443                                                   : $defaults->{cpuunits};
4444
4445         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4446
4447         my %properties = (
4448             Slice => 'qemu.slice',
4449             KillMode => 'none',
4450             CPUShares => $cpuunits
4451         );
4452
4453         if (my $cpulimit = $conf->{cpulimit}) {
4454             $properties{CPUQuota} = int($cpulimit * 100);
4455         }
4456         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4457
4458         if ($conf->{hugepages}) {
4459
4460             my $code = sub {
4461                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4462                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4463
4464                 PVE::QemuServer::Memory::hugepages_mount();
4465                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4466
4467                 eval  {
4468                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4469                     run_command($cmd, %run_params);
4470                 };
4471
4472                 if (my $err = $@) {
4473                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4474                     die $err;
4475                 }
4476
4477                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4478             };
4479             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4480
4481         } else {
4482             eval  {
4483                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4484                 run_command($cmd, %run_params);
4485             };
4486         }
4487
4488         if (my $err = $@) {
4489             # deactivate volumes if start fails
4490             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4491             die "start failed: $err";
4492         }
4493
4494         print "migration listens on $migrate_uri\n" if $migrate_uri;
4495
4496         if ($statefile && $statefile ne 'tcp')  {
4497             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4498             warn $@ if $@;
4499         }
4500
4501         if ($migratedfrom) {
4502
4503             eval {
4504                 set_migration_caps($vmid);
4505             };
4506             warn $@ if $@;
4507
4508             if ($spice_port) {
4509                 print "spice listens on port $spice_port\n";
4510                 if ($spice_ticket) {
4511                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4512                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4513                 }
4514             }
4515
4516         } else {
4517
4518             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4519                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4520                     if $conf->{balloon};
4521             }
4522
4523             foreach my $opt (keys %$conf) {
4524                 next if $opt !~  m/^net\d+$/;
4525                 my $nicconf = parse_net($conf->{$opt});
4526                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4527             }
4528         }
4529
4530         vm_mon_cmd_nocheck($vmid, 'qom-set',
4531                     path => "machine/peripheral/balloon0",
4532                     property => "guest-stats-polling-interval",
4533                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4534
4535     });
4536 }
4537
4538 sub vm_mon_cmd {
4539     my ($vmid, $execute, %params) = @_;
4540
4541     my $cmd = { execute => $execute, arguments => \%params };
4542     vm_qmp_command($vmid, $cmd);
4543 }
4544
4545 sub vm_mon_cmd_nocheck {
4546     my ($vmid, $execute, %params) = @_;
4547
4548     my $cmd = { execute => $execute, arguments => \%params };
4549     vm_qmp_command($vmid, $cmd, 1);
4550 }
4551
4552 sub vm_qmp_command {
4553     my ($vmid, $cmd, $nocheck) = @_;
4554
4555     my $res;
4556
4557     my $timeout;
4558     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4559         $timeout = $cmd->{arguments}->{timeout};
4560         delete $cmd->{arguments}->{timeout};
4561     }
4562
4563     eval {
4564         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4565         my $sname = qmp_socket($vmid);
4566         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4567             my $qmpclient = PVE::QMPClient->new();
4568
4569             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4570         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4571             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4572                 if scalar(%{$cmd->{arguments}});
4573             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4574         } else {
4575             die "unable to open monitor socket\n";
4576         }
4577     };
4578     if (my $err = $@) {
4579         syslog("err", "VM $vmid qmp command failed - $err");
4580         die $err;
4581     }
4582
4583     return $res;
4584 }
4585
4586 sub vm_human_monitor_command {
4587     my ($vmid, $cmdline) = @_;
4588
4589     my $res;
4590
4591     my $cmd = {
4592         execute => 'human-monitor-command',
4593         arguments => { 'command-line' => $cmdline},
4594     };
4595
4596     return vm_qmp_command($vmid, $cmd);
4597 }
4598
4599 sub vm_commandline {
4600     my ($storecfg, $vmid) = @_;
4601
4602     my $conf = PVE::QemuConfig->load_config($vmid);
4603
4604     my $defaults = load_defaults();
4605
4606     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4607
4608     return join(' ', @$cmd);
4609 }
4610
4611 sub vm_reset {
4612     my ($vmid, $skiplock) = @_;
4613
4614     PVE::QemuConfig->lock_config($vmid, sub {
4615
4616         my $conf = PVE::QemuConfig->load_config($vmid);
4617
4618         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4619
4620         vm_mon_cmd($vmid, "system_reset");
4621     });
4622 }
4623
4624 sub get_vm_volumes {
4625     my ($conf) = @_;
4626
4627     my $vollist = [];
4628     foreach_volid($conf, sub {
4629         my ($volid, $is_cdrom) = @_;
4630
4631         return if $volid =~ m|^/|;
4632
4633         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4634         return if !$sid;
4635
4636         push @$vollist, $volid;
4637     });
4638
4639     return $vollist;
4640 }
4641
4642 sub vm_stop_cleanup {
4643     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4644
4645     eval {
4646
4647         if (!$keepActive) {
4648             my $vollist = get_vm_volumes($conf);
4649             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4650         }
4651
4652         foreach my $ext (qw(mon qmp pid vnc qga)) {
4653             unlink "/var/run/qemu-server/${vmid}.$ext";
4654         }
4655
4656         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4657     };
4658     warn $@ if $@; # avoid errors - just warn
4659 }
4660
4661 # Note: use $nockeck to skip tests if VM configuration file exists.
4662 # We need that when migration VMs to other nodes (files already moved)
4663 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4664 sub vm_stop {
4665     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4666
4667     $force = 1 if !defined($force) && !$shutdown;
4668
4669     if ($migratedfrom){
4670         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4671         kill 15, $pid if $pid;
4672         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4673         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4674         return;
4675     }
4676
4677     PVE::QemuConfig->lock_config($vmid, sub {
4678
4679         my $pid = check_running($vmid, $nocheck);
4680         return if !$pid;
4681
4682         my $conf;
4683         if (!$nocheck) {
4684             $conf = PVE::QemuConfig->load_config($vmid);
4685             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4686             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4687                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4688                 $timeout = $opts->{down} if $opts->{down};
4689             }
4690         }
4691
4692         $timeout = 60 if !defined($timeout);
4693
4694         eval {
4695             if ($shutdown) {
4696                 if (defined($conf) && $conf->{agent}) {
4697                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4698                 } else {
4699                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4700                 }
4701             } else {
4702                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4703             }
4704         };
4705         my $err = $@;
4706
4707         if (!$err) {
4708             my $count = 0;
4709             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4710                 $count++;
4711                 sleep 1;
4712             }
4713
4714             if ($count >= $timeout) {
4715                 if ($force) {
4716                     warn "VM still running - terminating now with SIGTERM\n";
4717                     kill 15, $pid;
4718                 } else {
4719                     die "VM quit/powerdown failed - got timeout\n";
4720                 }
4721             } else {
4722                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4723                 return;
4724             }
4725         } else {
4726             if ($force) {
4727                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4728                 kill 15, $pid;
4729             } else {
4730                 die "VM quit/powerdown failed\n";
4731             }
4732         }
4733
4734         # wait again
4735         $timeout = 10;
4736
4737         my $count = 0;
4738         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4739             $count++;
4740             sleep 1;
4741         }
4742
4743         if ($count >= $timeout) {
4744             warn "VM still running - terminating now with SIGKILL\n";
4745             kill 9, $pid;
4746             sleep 1;
4747         }
4748
4749         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4750    });
4751 }
4752
4753 sub vm_suspend {
4754     my ($vmid, $skiplock) = @_;
4755
4756     PVE::QemuConfig->lock_config($vmid, sub {
4757
4758         my $conf = PVE::QemuConfig->load_config($vmid);
4759
4760         PVE::QemuConfig->check_lock($conf)
4761             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4762
4763         vm_mon_cmd($vmid, "stop");
4764     });
4765 }
4766
4767 sub vm_resume {
4768     my ($vmid, $skiplock, $nocheck) = @_;
4769
4770     PVE::QemuConfig->lock_config($vmid, sub {
4771
4772         if (!$nocheck) {
4773
4774             my $conf = PVE::QemuConfig->load_config($vmid);
4775
4776             PVE::QemuConfig->check_lock($conf)
4777                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4778
4779             vm_mon_cmd($vmid, "cont");
4780
4781         } else {
4782             vm_mon_cmd_nocheck($vmid, "cont");
4783         }
4784     });
4785 }
4786
4787 sub vm_sendkey {
4788     my ($vmid, $skiplock, $key) = @_;
4789
4790     PVE::QemuConfig->lock_config($vmid, sub {
4791
4792         my $conf = PVE::QemuConfig->load_config($vmid);
4793
4794         # there is no qmp command, so we use the human monitor command
4795         vm_human_monitor_command($vmid, "sendkey $key");
4796     });
4797 }
4798
4799 sub vm_destroy {
4800     my ($storecfg, $vmid, $skiplock) = @_;
4801
4802     PVE::QemuConfig->lock_config($vmid, sub {
4803
4804         my $conf = PVE::QemuConfig->load_config($vmid);
4805
4806         if (!check_running($vmid)) {
4807             destroy_vm($storecfg, $vmid, undef, $skiplock);
4808         } else {
4809             die "VM $vmid is running - destroy failed\n";
4810         }
4811     });
4812 }
4813
4814 # pci helpers
4815
4816 sub file_write {
4817     my ($filename, $buf) = @_;
4818
4819     my $fh = IO::File->new($filename, "w");
4820     return undef if !$fh;
4821
4822     my $res = print $fh $buf;
4823
4824     $fh->close();
4825
4826     return $res;
4827 }
4828
4829 sub pci_device_info {
4830     my ($name) = @_;
4831
4832     my $res;
4833
4834     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4835     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4836
4837     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4838     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4839
4840     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4841     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4842
4843     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4844     return undef if !defined($product) || $product !~ s/^0x//;
4845
4846     $res = {
4847         name => $name,
4848         vendor => $vendor,
4849         product => $product,
4850         domain => $domain,
4851         bus => $bus,
4852         slot => $slot,
4853         func => $func,
4854         irq => $irq,
4855         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4856     };
4857
4858     return $res;
4859 }
4860
4861 sub pci_dev_reset {
4862     my ($dev) = @_;
4863
4864     my $name = $dev->{name};
4865
4866     my $fn = "$pcisysfs/devices/$name/reset";
4867
4868     return file_write($fn, "1");
4869 }
4870
4871 sub pci_dev_bind_to_vfio {
4872     my ($dev) = @_;
4873
4874     my $name = $dev->{name};
4875
4876     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4877
4878     if (!-d $vfio_basedir) {
4879         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4880     }
4881     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4882
4883     my $testdir = "$vfio_basedir/$name";
4884     return 1 if -d $testdir;
4885
4886     my $data = "$dev->{vendor} $dev->{product}";
4887     return undef if !file_write("$vfio_basedir/new_id", $data);
4888
4889     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4890     if (!file_write($fn, $name)) {
4891         return undef if -f $fn;
4892     }
4893
4894     $fn = "$vfio_basedir/bind";
4895     if (! -d $testdir) {
4896         return undef if !file_write($fn, $name);
4897     }
4898
4899     return -d $testdir;
4900 }
4901
4902 sub pci_dev_group_bind_to_vfio {
4903     my ($pciid) = @_;
4904
4905     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4906
4907     if (!-d $vfio_basedir) {
4908         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4909     }
4910     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4911
4912     # get IOMMU group devices
4913     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4914       my @devs = grep /^0000:/, readdir($D);
4915     closedir($D);
4916
4917     foreach my $pciid (@devs) {
4918         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4919
4920         # pci bridges, switches or root ports are not supported
4921         # they have a pci_bus subdirectory so skip them
4922         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4923
4924         my $info = pci_device_info($1);
4925         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4926     }
4927
4928     return 1;
4929 }
4930
4931 # vzdump restore implementaion
4932
4933 sub tar_archive_read_firstfile {
4934     my $archive = shift;
4935
4936     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4937
4938     # try to detect archive type first
4939     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4940         die "unable to open file '$archive'\n";
4941     my $firstfile = <$fh>;
4942     kill 15, $pid;
4943     close $fh;
4944
4945     die "ERROR: archive contaions no data\n" if !$firstfile;
4946     chomp $firstfile;
4947
4948     return $firstfile;
4949 }
4950
4951 sub tar_restore_cleanup {
4952     my ($storecfg, $statfile) = @_;
4953
4954     print STDERR "starting cleanup\n";
4955
4956     if (my $fd = IO::File->new($statfile, "r")) {
4957         while (defined(my $line = <$fd>)) {
4958             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4959                 my $volid = $2;
4960                 eval {
4961                     if ($volid =~ m|^/|) {
4962                         unlink $volid || die 'unlink failed\n';
4963                     } else {
4964                         PVE::Storage::vdisk_free($storecfg, $volid);
4965                     }
4966                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4967                 };
4968                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4969             } else {
4970                 print STDERR "unable to parse line in statfile - $line";
4971             }
4972         }
4973         $fd->close();
4974     }
4975 }
4976
4977 sub restore_archive {
4978     my ($archive, $vmid, $user, $opts) = @_;
4979
4980     my $format = $opts->{format};
4981     my $comp;
4982
4983     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4984         $format = 'tar' if !$format;
4985         $comp = 'gzip';
4986     } elsif ($archive =~ m/\.tar$/) {
4987         $format = 'tar' if !$format;
4988     } elsif ($archive =~ m/.tar.lzo$/) {
4989         $format = 'tar' if !$format;
4990         $comp = 'lzop';
4991     } elsif ($archive =~ m/\.vma$/) {
4992         $format = 'vma' if !$format;
4993     } elsif ($archive =~ m/\.vma\.gz$/) {
4994         $format = 'vma' if !$format;
4995         $comp = 'gzip';
4996     } elsif ($archive =~ m/\.vma\.lzo$/) {
4997         $format = 'vma' if !$format;
4998         $comp = 'lzop';
4999     } else {
5000         $format = 'vma' if !$format; # default
5001     }
5002
5003     # try to detect archive format
5004     if ($format eq 'tar') {
5005         return restore_tar_archive($archive, $vmid, $user, $opts);
5006     } else {
5007         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5008     }
5009 }
5010
5011 sub restore_update_config_line {
5012     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5013
5014     return if $line =~ m/^\#qmdump\#/;
5015     return if $line =~ m/^\#vzdump\#/;
5016     return if $line =~ m/^lock:/;
5017     return if $line =~ m/^unused\d+:/;
5018     return if $line =~ m/^parent:/;
5019     return if $line =~ m/^template:/; # restored VM is never a template
5020
5021     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5022     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5023         # try to convert old 1.X settings
5024         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5025         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5026             my ($model, $macaddr) = split(/\=/, $devconfig);
5027             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5028             my $net = {
5029                 model => $model,
5030                 bridge => "vmbr$ind",
5031                 macaddr => $macaddr,
5032             };
5033             my $netstr = print_net($net);
5034
5035             print $outfd "net$cookie->{netcount}: $netstr\n";
5036             $cookie->{netcount}++;
5037         }
5038     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5039         my ($id, $netstr) = ($1, $2);
5040         my $net = parse_net($netstr);
5041         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5042         $netstr = print_net($net);
5043         print $outfd "$id: $netstr\n";
5044     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5045         my $virtdev = $1;
5046         my $value = $3;
5047         my $di = parse_drive($virtdev, $value);
5048         if (defined($di->{backup}) && !$di->{backup}) {
5049             print $outfd "#$line";
5050         } elsif ($map->{$virtdev}) {
5051             delete $di->{format}; # format can change on restore
5052             $di->{file} = $map->{$virtdev};
5053             $value = print_drive($vmid, $di);
5054             print $outfd "$virtdev: $value\n";
5055         } else {
5056             print $outfd $line;
5057         }
5058     } else {
5059         print $outfd $line;
5060     }
5061 }
5062
5063 sub scan_volids {
5064     my ($cfg, $vmid) = @_;
5065
5066     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5067
5068     my $volid_hash = {};
5069     foreach my $storeid (keys %$info) {
5070         foreach my $item (@{$info->{$storeid}}) {
5071             next if !($item->{volid} && $item->{size});
5072             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5073             $volid_hash->{$item->{volid}} = $item;
5074         }
5075     }
5076
5077     return $volid_hash;
5078 }
5079
5080 sub is_volume_in_use {
5081     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5082
5083     my $path = PVE::Storage::path($storecfg, $volid);
5084
5085     my $scan_config = sub {
5086         my ($cref, $snapname) = @_;
5087
5088         foreach my $key (keys %$cref) {
5089             my $value = $cref->{$key};
5090             if (is_valid_drivename($key)) {
5091                 next if $skip_drive && $key eq $skip_drive;
5092                 my $drive = parse_drive($key, $value);
5093                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5094                 return 1 if $volid eq $drive->{file};
5095                 if ($drive->{file} =~ m!^/!) {
5096                     return 1 if $drive->{file} eq $path;
5097                 } else {
5098                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5099                     next if !$storeid;
5100                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5101                     next if !$scfg;
5102                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5103                 }
5104             }
5105         }
5106
5107         return 0;
5108     };
5109
5110     return 1 if &$scan_config($conf);
5111
5112     undef $skip_drive;
5113
5114     foreach my $snapname (keys %{$conf->{snapshots}}) {
5115         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5116     }
5117
5118     return 0;
5119 }
5120
5121 sub update_disksize {
5122     my ($vmid, $conf, $volid_hash) = @_;
5123
5124     my $changes;
5125
5126     my $used = {};
5127
5128     # Note: it is allowed to define multiple storages with same path (alias), so
5129     # we need to check both 'volid' and real 'path' (two different volid can point
5130     # to the same path).
5131
5132     my $usedpath = {};
5133
5134     # update size info
5135     foreach my $opt (keys %$conf) {
5136         if (is_valid_drivename($opt)) {
5137             my $drive = parse_drive($opt, $conf->{$opt});
5138             my $volid = $drive->{file};
5139             next if !$volid;
5140
5141             $used->{$volid} = 1;
5142             if ($volid_hash->{$volid} &&
5143                 (my $path = $volid_hash->{$volid}->{path})) {
5144                 $usedpath->{$path} = 1;
5145             }
5146
5147             next if drive_is_cdrom($drive);
5148             next if !$volid_hash->{$volid};
5149
5150             $drive->{size} = $volid_hash->{$volid}->{size};
5151             my $new = print_drive($vmid, $drive);
5152             if ($new ne $conf->{$opt}) {
5153                 $changes = 1;
5154                 $conf->{$opt} = $new;
5155             }
5156         }
5157     }
5158
5159     # remove 'unusedX' entry if volume is used
5160     foreach my $opt (keys %$conf) {
5161         next if $opt !~ m/^unused\d+$/;
5162         my $volid = $conf->{$opt};
5163         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5164         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5165             $changes = 1;
5166             delete $conf->{$opt};
5167         }
5168     }
5169
5170     foreach my $volid (sort keys %$volid_hash) {
5171         next if $volid =~ m/vm-$vmid-state-/;
5172         next if $used->{$volid};
5173         my $path = $volid_hash->{$volid}->{path};
5174         next if !$path; # just to be sure
5175         next if $usedpath->{$path};
5176         $changes = 1;
5177         PVE::QemuConfig->add_unused_volume($conf, $volid);
5178         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5179     }
5180
5181     return $changes;
5182 }
5183
5184 sub rescan {
5185     my ($vmid, $nolock) = @_;
5186
5187     my $cfg = PVE::Storage::config();
5188
5189     my $volid_hash = scan_volids($cfg, $vmid);
5190
5191     my $updatefn =  sub {
5192         my ($vmid) = @_;
5193
5194         my $conf = PVE::QemuConfig->load_config($vmid);
5195
5196         PVE::QemuConfig->check_lock($conf);
5197
5198         my $vm_volids = {};
5199         foreach my $volid (keys %$volid_hash) {
5200             my $info = $volid_hash->{$volid};
5201             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5202         }
5203
5204         my $changes = update_disksize($vmid, $conf, $vm_volids);
5205
5206         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5207     };
5208
5209     if (defined($vmid)) {
5210         if ($nolock) {
5211             &$updatefn($vmid);
5212         } else {
5213             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5214         }
5215     } else {
5216         my $vmlist = config_list();
5217         foreach my $vmid (keys %$vmlist) {
5218             if ($nolock) {
5219                 &$updatefn($vmid);
5220             } else {
5221                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5222             }
5223         }
5224     }
5225 }
5226
5227 sub restore_vma_archive {
5228     my ($archive, $vmid, $user, $opts, $comp) = @_;
5229
5230     my $input = $archive eq '-' ? "<&STDIN" : undef;
5231     my $readfrom = $archive;
5232
5233     my $uncomp = '';
5234     if ($comp) {
5235         $readfrom = '-';
5236         my $qarchive = PVE::Tools::shellquote($archive);
5237         if ($comp eq 'gzip') {
5238             $uncomp = "zcat $qarchive|";
5239         } elsif ($comp eq 'lzop') {
5240             $uncomp = "lzop -d -c $qarchive|";
5241         } else {
5242             die "unknown compression method '$comp'\n";
5243         }
5244
5245     }
5246
5247     my $tmpdir = "/var/tmp/vzdumptmp$$";
5248     rmtree $tmpdir;
5249
5250     # disable interrupts (always do cleanups)
5251     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5252         warn "got interrupt - ignored\n";
5253     };
5254
5255     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5256     POSIX::mkfifo($mapfifo, 0600);
5257     my $fifofh;
5258
5259     my $openfifo = sub {
5260         open($fifofh, '>', $mapfifo) || die $!;
5261     };
5262
5263     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5264
5265     my $oldtimeout;
5266     my $timeout = 5;
5267
5268     my $devinfo = {};
5269
5270     my $rpcenv = PVE::RPCEnvironment::get();
5271
5272     my $conffile = PVE::QemuConfig->config_file($vmid);
5273     my $tmpfn = "$conffile.$$.tmp";
5274
5275     # Note: $oldconf is undef if VM does not exists
5276     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5277     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5278
5279     my $print_devmap = sub {
5280         my $virtdev_hash = {};
5281
5282         my $cfgfn = "$tmpdir/qemu-server.conf";
5283
5284         # we can read the config - that is already extracted
5285         my $fh = IO::File->new($cfgfn, "r") ||
5286             "unable to read qemu-server.conf - $!\n";
5287
5288         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5289         if (-f $fwcfgfn) {
5290             my $pve_firewall_dir = '/etc/pve/firewall';
5291             mkdir $pve_firewall_dir; # make sure the dir exists
5292             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5293         }
5294
5295         while (defined(my $line = <$fh>)) {
5296             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5297                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5298                 die "archive does not contain data for drive '$virtdev'\n"
5299                     if !$devinfo->{$devname};
5300                 if (defined($opts->{storage})) {
5301                     $storeid = $opts->{storage} || 'local';
5302                 } elsif (!$storeid) {
5303                     $storeid = 'local';
5304                 }
5305                 $format = 'raw' if !$format;
5306                 $devinfo->{$devname}->{devname} = $devname;
5307                 $devinfo->{$devname}->{virtdev} = $virtdev;
5308                 $devinfo->{$devname}->{format} = $format;
5309                 $devinfo->{$devname}->{storeid} = $storeid;
5310
5311                 # check permission on storage
5312                 my $pool = $opts->{pool}; # todo: do we need that?
5313                 if ($user ne 'root@pam') {
5314                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5315                 }
5316
5317                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5318             }
5319         }
5320
5321         foreach my $devname (keys %$devinfo) {
5322             die "found no device mapping information for device '$devname'\n"
5323                 if !$devinfo->{$devname}->{virtdev};
5324         }
5325
5326         my $cfg = PVE::Storage::config();
5327
5328         # create empty/temp config
5329         if ($oldconf) {
5330             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5331             foreach_drive($oldconf, sub {
5332                 my ($ds, $drive) = @_;
5333
5334                 return if drive_is_cdrom($drive);
5335
5336                 my $volid = $drive->{file};
5337
5338                 return if !$volid || $volid =~ m|^/|;
5339
5340                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5341                 return if !$path || !$owner || ($owner != $vmid);
5342
5343                 # Note: only delete disk we want to restore
5344                 # other volumes will become unused
5345                 if ($virtdev_hash->{$ds}) {
5346                     PVE::Storage::vdisk_free($cfg, $volid);
5347                 }
5348             });
5349
5350             # delete vmstate files
5351             # since after the restore we have no snapshots anymore
5352             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5353                 my $snap = $oldconf->{snapshots}->{$snapname};
5354                 if ($snap->{vmstate}) {
5355                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5356                     if (my $err = $@) {
5357                         warn $err;
5358                     }
5359                 }
5360             }
5361         }
5362
5363         my $map = {};
5364         foreach my $virtdev (sort keys %$virtdev_hash) {
5365             my $d = $virtdev_hash->{$virtdev};
5366             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5367             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5368
5369             # test if requested format is supported
5370             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5371             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5372             $d->{format} = $defFormat if !$supported;
5373
5374             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5375                                                   $d->{format}, undef, $alloc_size);
5376             print STDERR "new volume ID is '$volid'\n";
5377             $d->{volid} = $volid;
5378             my $path = PVE::Storage::path($cfg, $volid);
5379
5380             PVE::Storage::activate_volumes($cfg,[$volid]);
5381
5382             my $write_zeros = 1;
5383             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5384                 $write_zeros = 0;
5385             }
5386
5387             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5388
5389             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5390             $map->{$virtdev} = $volid;
5391         }
5392
5393         $fh->seek(0, 0) || die "seek failed - $!\n";
5394
5395         my $outfd = new IO::File ($tmpfn, "w") ||
5396             die "unable to write config for VM $vmid\n";
5397
5398         my $cookie = { netcount => 0 };
5399         while (defined(my $line = <$fh>)) {
5400             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5401         }
5402
5403         $fh->close();
5404         $outfd->close();
5405     };
5406
5407     eval {
5408         # enable interrupts
5409         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5410             die "interrupted by signal\n";
5411         };
5412         local $SIG{ALRM} = sub { die "got timeout\n"; };
5413
5414         $oldtimeout = alarm($timeout);
5415
5416         my $parser = sub {
5417             my $line = shift;
5418
5419             print "$line\n";
5420
5421             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5422                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5423                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5424             } elsif ($line =~ m/^CTIME: /) {
5425                 # we correctly received the vma config, so we can disable
5426                 # the timeout now for disk allocation (set to 10 minutes, so
5427                 # that we always timeout if something goes wrong)
5428                 alarm(600);
5429                 &$print_devmap();
5430                 print $fifofh "done\n";
5431                 my $tmp = $oldtimeout || 0;
5432                 $oldtimeout = undef;
5433                 alarm($tmp);
5434                 close($fifofh);
5435             }
5436         };
5437
5438         print "restore vma archive: $cmd\n";
5439         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5440     };
5441     my $err = $@;
5442
5443     alarm($oldtimeout) if $oldtimeout;
5444
5445     my $vollist = [];
5446     foreach my $devname (keys %$devinfo) {
5447         my $volid = $devinfo->{$devname}->{volid};
5448         push @$vollist, $volid if $volid;
5449     }
5450
5451     my $cfg = PVE::Storage::config();
5452     PVE::Storage::deactivate_volumes($cfg, $vollist);
5453
5454     unlink $mapfifo;
5455
5456     if ($err) {
5457         rmtree $tmpdir;
5458         unlink $tmpfn;
5459
5460         foreach my $devname (keys %$devinfo) {
5461             my $volid = $devinfo->{$devname}->{volid};
5462             next if !$volid;
5463             eval {
5464                 if ($volid =~ m|^/|) {
5465                     unlink $volid || die 'unlink failed\n';
5466                 } else {
5467                     PVE::Storage::vdisk_free($cfg, $volid);
5468                 }
5469                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5470             };
5471             print STDERR "unable to cleanup '$volid' - $@" if $@;
5472         }
5473         die $err;
5474     }
5475
5476     rmtree $tmpdir;
5477
5478     rename($tmpfn, $conffile) ||
5479         die "unable to commit configuration file '$conffile'\n";
5480
5481     PVE::Cluster::cfs_update(); # make sure we read new file
5482
5483     eval { rescan($vmid, 1); };
5484     warn $@ if $@;
5485 }
5486
5487 sub restore_tar_archive {
5488     my ($archive, $vmid, $user, $opts) = @_;
5489
5490     if ($archive ne '-') {
5491         my $firstfile = tar_archive_read_firstfile($archive);
5492         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5493             if $firstfile ne 'qemu-server.conf';
5494     }
5495
5496     my $storecfg = PVE::Storage::config();
5497
5498     # destroy existing data - keep empty config
5499     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5500     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5501
5502     my $tocmd = "/usr/lib/qemu-server/qmextract";
5503
5504     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5505     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5506     $tocmd .= ' --prealloc' if $opts->{prealloc};
5507     $tocmd .= ' --info' if $opts->{info};
5508
5509     # tar option "xf" does not autodetect compression when read from STDIN,
5510     # so we pipe to zcat
5511     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5512         PVE::Tools::shellquote("--to-command=$tocmd");
5513
5514     my $tmpdir = "/var/tmp/vzdumptmp$$";
5515     mkpath $tmpdir;
5516
5517     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5518     local $ENV{VZDUMP_VMID} = $vmid;
5519     local $ENV{VZDUMP_USER} = $user;
5520
5521     my $conffile = PVE::QemuConfig->config_file($vmid);
5522     my $tmpfn = "$conffile.$$.tmp";
5523
5524     # disable interrupts (always do cleanups)
5525     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5526         print STDERR "got interrupt - ignored\n";
5527     };
5528
5529     eval {
5530         # enable interrupts
5531         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5532             die "interrupted by signal\n";
5533         };
5534
5535         if ($archive eq '-') {
5536             print "extracting archive from STDIN\n";
5537             run_command($cmd, input => "<&STDIN");
5538         } else {
5539             print "extracting archive '$archive'\n";
5540             run_command($cmd);
5541         }
5542
5543         return if $opts->{info};
5544
5545         # read new mapping
5546         my $map = {};
5547         my $statfile = "$tmpdir/qmrestore.stat";
5548         if (my $fd = IO::File->new($statfile, "r")) {
5549             while (defined (my $line = <$fd>)) {
5550                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5551                     $map->{$1} = $2 if $1;
5552                 } else {
5553                     print STDERR "unable to parse line in statfile - $line\n";
5554                 }
5555             }
5556             $fd->close();
5557         }
5558
5559         my $confsrc = "$tmpdir/qemu-server.conf";
5560
5561         my $srcfd = new IO::File($confsrc, "r") ||
5562             die "unable to open file '$confsrc'\n";
5563
5564         my $outfd = new IO::File ($tmpfn, "w") ||
5565             die "unable to write config for VM $vmid\n";
5566
5567         my $cookie = { netcount => 0 };
5568         while (defined (my $line = <$srcfd>)) {
5569             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5570         }
5571
5572         $srcfd->close();
5573         $outfd->close();
5574     };
5575     my $err = $@;
5576
5577     if ($err) {
5578
5579         unlink $tmpfn;
5580
5581         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5582
5583         die $err;
5584     }
5585
5586     rmtree $tmpdir;
5587
5588     rename $tmpfn, $conffile ||
5589         die "unable to commit configuration file '$conffile'\n";
5590
5591     PVE::Cluster::cfs_update(); # make sure we read new file
5592
5593     eval { rescan($vmid, 1); };
5594     warn $@ if $@;
5595 };
5596
5597 sub foreach_writable_storage {
5598     my ($conf, $func) = @_;
5599
5600     my $sidhash = {};
5601
5602     foreach my $ds (keys %$conf) {
5603         next if !is_valid_drivename($ds);
5604
5605         my $drive = parse_drive($ds, $conf->{$ds});
5606         next if !$drive;
5607         next if drive_is_cdrom($drive);
5608
5609         my $volid = $drive->{file};
5610
5611         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5612         $sidhash->{$sid} = $sid if $sid;
5613     }
5614
5615     foreach my $sid (sort keys %$sidhash) {
5616         &$func($sid);
5617     }
5618 }
5619
5620 sub do_snapshots_with_qemu {
5621     my ($storecfg, $volid) = @_;
5622
5623     my $storage_name = PVE::Storage::parse_volume_id($volid);
5624
5625     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5626         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5627         return 1;
5628     }
5629
5630     if ($volid =~ m/\.(qcow2|qed)$/){
5631         return 1;
5632     }
5633
5634     return undef;
5635 }
5636
5637 sub qga_check_running {
5638     my ($vmid) = @_;
5639
5640     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5641     if ($@) {
5642         warn "Qemu Guest Agent are not running - $@";
5643         return 0;
5644     }
5645     return 1;
5646 }
5647
5648 sub template_create {
5649     my ($vmid, $conf, $disk) = @_;
5650
5651     my $storecfg = PVE::Storage::config();
5652
5653     foreach_drive($conf, sub {
5654         my ($ds, $drive) = @_;
5655
5656         return if drive_is_cdrom($drive);
5657         return if $disk && $ds ne $disk;
5658
5659         my $volid = $drive->{file};
5660         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5661
5662         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5663         $drive->{file} = $voliddst;
5664         $conf->{$ds} = print_drive($vmid, $drive);
5665         PVE::QemuConfig->write_config($vmid, $conf);
5666     });
5667 }
5668
5669 sub qemu_img_convert {
5670     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5671
5672     my $storecfg = PVE::Storage::config();
5673     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5674     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5675
5676     if ($src_storeid && $dst_storeid) {
5677
5678         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5679
5680         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5681         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5682
5683         my $src_format = qemu_img_format($src_scfg, $src_volname);
5684         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5685
5686         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5687         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5688
5689         my $cmd = [];
5690         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5691         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5692         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5693         if ($is_zero_initialized) {
5694             push @$cmd, "zeroinit:$dst_path";
5695         } else {
5696             push @$cmd, $dst_path;
5697         }
5698
5699         my $parser = sub {
5700             my $line = shift;
5701             if($line =~ m/\((\S+)\/100\%\)/){
5702                 my $percent = $1;
5703                 my $transferred = int($size * $percent / 100);
5704                 my $remaining = $size - $transferred;
5705
5706                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5707             }
5708
5709         };
5710
5711         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5712         my $err = $@;
5713         die "copy failed: $err" if $err;
5714     }
5715 }
5716
5717 sub qemu_img_format {
5718     my ($scfg, $volname) = @_;
5719
5720     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5721         return $1;
5722     } else {
5723         return "raw";
5724     }
5725 }
5726
5727 sub qemu_drive_mirror {
5728     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5729
5730     my $storecfg = PVE::Storage::config();
5731     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5732
5733     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5734
5735     my $format = qemu_img_format($dst_scfg, $dst_volname);
5736
5737     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5738
5739     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5740
5741     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5742     $opts->{format} = $format if $format;
5743
5744     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5745
5746     my $finish_job = sub {
5747         while (1) {
5748             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5749             my $stat = @$stats[0];
5750             last if !$stat;
5751             sleep 1;
5752         }
5753     };
5754
5755     eval {
5756     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5757         while (1) {
5758             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5759             my $stat = @$stats[0];
5760             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5761             die "error job is not mirroring" if $stat->{type} ne "mirror";
5762
5763             my $busy = $stat->{busy};
5764             my $ready = $stat->{ready};
5765
5766             if (my $total = $stat->{len}) {
5767                 my $transferred = $stat->{offset} || 0;
5768                 my $remaining = $total - $transferred;
5769                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5770
5771                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5772             }
5773
5774
5775             if ($stat->{ready} eq 'true') {
5776
5777                 last if $vmiddst != $vmid;
5778
5779                 # try to switch the disk if source and destination are on the same guest
5780                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5781                 if (!$@) {
5782                     &$finish_job();
5783                     last;
5784                 }
5785                 die $@ if $@ !~ m/cannot be completed/;
5786             }
5787             sleep 1;
5788         }
5789
5790
5791     };
5792     my $err = $@;
5793
5794     my $cancel_job = sub {
5795         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5796         &$finish_job();
5797     };
5798
5799     if ($err) {
5800         eval { &$cancel_job(); };
5801         die "mirroring error: $err";
5802     }
5803
5804     if ($vmiddst != $vmid) {
5805         # if we clone a disk for a new target vm, we don't switch the disk
5806         &$cancel_job(); # so we call block-job-cancel
5807     }
5808 }
5809
5810 sub clone_disk {
5811     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5812         $newvmid, $storage, $format, $full, $newvollist) = @_;
5813
5814     my $newvolid;
5815
5816     if (!$full) {
5817         print "create linked clone of drive $drivename ($drive->{file})\n";
5818         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5819         push @$newvollist, $newvolid;
5820     } else {
5821         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5822         $storeid = $storage if $storage;
5823
5824         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5825         if (!$format) {
5826             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5827             $format = qemu_img_format($scfg, $volname);
5828         }
5829
5830         # test if requested format is supported - else use default
5831         my $supported = grep { $_ eq $format } @$validFormats;
5832         $format = $defFormat if !$supported;
5833
5834         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5835
5836         print "create full clone of drive $drivename ($drive->{file})\n";
5837         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5838         push @$newvollist, $newvolid;
5839
5840         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5841
5842         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5843         if (!$running || $snapname) {
5844             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5845         } else {
5846             #qemu 2.6
5847             die "drive-mirror is not working currently when iothread is enabled" if $drive->{iothread};
5848
5849             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5850         }
5851     }
5852
5853     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5854
5855     my $disk = $drive;
5856     $disk->{format} = undef;
5857     $disk->{file} = $newvolid;
5858     $disk->{size} = $size;
5859
5860     return $disk;
5861 }
5862
5863 # this only works if VM is running
5864 sub get_current_qemu_machine {
5865     my ($vmid) = @_;
5866
5867     my $cmd = { execute => 'query-machines', arguments => {} };
5868     my $res = vm_qmp_command($vmid, $cmd);
5869
5870     my ($current, $default);
5871     foreach my $e (@$res) {
5872         $default = $e->{name} if $e->{'is-default'};
5873         $current = $e->{name} if $e->{'is-current'};
5874     }
5875
5876     # fallback to the default machine if current is not supported by qemu
5877     return $current || $default || 'pc';
5878 }
5879
5880 sub qemu_machine_feature_enabled {
5881     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5882
5883     my $current_major;
5884     my $current_minor;
5885
5886     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5887
5888         $current_major = $3;
5889         $current_minor = $4;
5890
5891     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5892
5893         $current_major = $1;
5894         $current_minor = $2;
5895     }
5896
5897     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5898
5899
5900 }
5901
5902 sub qemu_machine_pxe {
5903     my ($vmid, $conf, $machine) = @_;
5904
5905     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5906
5907     foreach my $opt (keys %$conf) {
5908         next if $opt !~ m/^net(\d+)$/;
5909         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5910         next if !$net;
5911         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5912         return $machine.".pxe" if $romfile =~ m/pxe/;
5913         last;
5914     }
5915
5916     return $machine;
5917 }
5918
5919 sub qemu_use_old_bios_files {
5920     my ($machine_type) = @_;
5921
5922     return if !$machine_type;
5923
5924     my $use_old_bios_files = undef;
5925
5926     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5927         $machine_type = $1;
5928         $use_old_bios_files = 1;
5929     } else {
5930         my $kvmver = kvm_user_version();
5931         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5932         # load new efi bios files on migration. So this hack is required to allow
5933         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5934         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5935         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5936     }
5937
5938     return ($use_old_bios_files, $machine_type);
5939 }
5940
5941 sub lspci {
5942
5943     my $devices = {};
5944
5945     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5946             my (undef, $id, $function) = @_;
5947             my $res = { id => $id, function => $function};
5948             push @{$devices->{$id}}, $res;
5949     });
5950
5951     return $devices;
5952 }
5953
5954 sub vm_iothreads_list {
5955     my ($vmid) = @_;
5956
5957     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5958
5959     my $iothreads = {};
5960     foreach my $iothread (@$res) {
5961         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5962     }
5963
5964     return $iothreads;
5965 }
5966
5967 sub scsihw_infos {
5968     my ($conf, $drive) = @_;
5969
5970     my $maxdev = 0;
5971
5972     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5973         $maxdev = 7;
5974     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5975         $maxdev = 1;
5976     } else {
5977         $maxdev = 256;
5978     }
5979
5980     my $controller = int($drive->{index} / $maxdev);
5981     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5982
5983     return ($maxdev, $controller, $controller_prefix);
5984 }
5985
5986 # bash completion helper
5987
5988 sub complete_backup_archives {
5989     my ($cmdname, $pname, $cvalue) = @_;
5990
5991     my $cfg = PVE::Storage::config();
5992
5993     my $storeid;
5994
5995     if ($cvalue =~ m/^([^:]+):/) {
5996         $storeid = $1;
5997     }
5998
5999     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6000
6001     my $res = [];
6002     foreach my $id (keys %$data) {
6003         foreach my $item (@{$data->{$id}}) {
6004             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6005             push @$res, $item->{volid} if defined($item->{volid});
6006         }
6007     }
6008
6009     return $res;
6010 }
6011
6012 my $complete_vmid_full = sub {
6013     my ($running) = @_;
6014
6015     my $idlist = vmstatus();
6016
6017     my $res = [];
6018
6019     foreach my $id (keys %$idlist) {
6020         my $d = $idlist->{$id};
6021         if (defined($running)) {
6022             next if $d->{template};
6023             next if $running && $d->{status} ne 'running';
6024             next if !$running && $d->{status} eq 'running';
6025         }
6026         push @$res, $id;
6027
6028     }
6029     return $res;
6030 };
6031
6032 sub complete_vmid {
6033     return &$complete_vmid_full();
6034 }
6035
6036 sub complete_vmid_stopped {
6037     return &$complete_vmid_full(0);
6038 }
6039
6040 sub complete_vmid_running {
6041     return &$complete_vmid_full(1);
6042 }
6043
6044 sub complete_storage {
6045
6046     my $cfg = PVE::Storage::config();
6047     my $ids = $cfg->{ids};
6048
6049     my $res = [];
6050     foreach my $sid (keys %$ids) {
6051         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6052         next if !$ids->{$sid}->{content}->{images};
6053         push @$res, $sid;
6054     }
6055
6056     return $res;
6057 }
6058
6059 1;