PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::Memory;
  34 use Time::HiRes qw(gettimeofday);
  35 use File::Copy qw(copy);
  36 use URI::Escape;
  37
  38 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  39
  40 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  41
  42 # Note about locking: we use flock on the config file protect
  43 # against concurent actions.
  44 # Aditionaly, we have a 'lock' setting in the config file. This
  45 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  46 # allowed when such lock is set. But you can ignore this kind of
  47 # lock with the --skiplock flag.
  48
  49 cfs_register_file('/qemu-server/',
  50                   \&parse_vm_config,
  51                   \&write_vm_config);
  52
  53 PVE::JSONSchema::register_standard_option('skiplock', {
  54     description => "Ignore locks - only root is allowed to use this option.",
  55     type => 'boolean',
  56     optional => 1,
  57 });
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 #no warnings 'redefine';
  73
  74 sub cgroups_write {
  75    my ($controller, $vmid, $option, $value) = @_;
  76
  77    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  78    PVE::ProcFSTools::write_proc_entry($path, $value);
  79
  80 }
  81
  82 my $nodename = PVE::INotify::nodename();
  83
  84 mkdir "/etc/pve/nodes/$nodename";
  85 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  86 mkdir $confdir;
  87
  88 my $var_run_tmpdir = "/var/run/qemu-server";
  89 mkdir $var_run_tmpdir;
  90
  91 my $lock_dir = "/var/lock/qemu-server";
  92 mkdir $lock_dir;
  93
  94 my $pcisysfs = "/sys/bus/pci";
  95
  96 my $cpu_vendor_list = {
  97     # Intel CPUs
  98     486 => 'GenuineIntel',
  99     pentium => 'GenuineIntel',
 100     pentium2  => 'GenuineIntel',
 101     pentium3  => 'GenuineIntel',
 102     coreduo => 'GenuineIntel',
 103     core2duo => 'GenuineIntel',
 104     Conroe  => 'GenuineIntel',
 105     Penryn  => 'GenuineIntel',
 106     Nehalem  => 'GenuineIntel',
 107     Westmere => 'GenuineIntel',
 108     SandyBridge => 'GenuineIntel',
 109     IvyBridge => 'GenuineIntel',
 110     Haswell => 'GenuineIntel',
 111     'Haswell-noTSX' => 'GenuineIntel',
 112     Broadwell => 'GenuineIntel',
 113     'Broadwell-noTSX' => 'GenuineIntel',
 114
 115     # AMD CPUs
 116     athlon => 'AuthenticAMD',
 117     phenom  => 'AuthenticAMD',
 118     Opteron_G1  => 'AuthenticAMD',
 119     Opteron_G2  => 'AuthenticAMD',
 120     Opteron_G3  => 'AuthenticAMD',
 121     Opteron_G4  => 'AuthenticAMD',
 122     Opteron_G5  => 'AuthenticAMD',
 123
 124     # generic types, use vendor from host node
 125     host => 'default',
 126     kvm32 => 'default',
 127     kvm64 => 'default',
 128     qemu32 => 'default',
 129     qemu64 => 'default',
 130 };
 131
 132 my $cpu_fmt = {
 133     cputype => {
 134         description => "Emulated CPU type.",
 135         type => 'string',
 136         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 137         format_description => 'cputype',
 138         default => 'kvm64',
 139         default_key => 1,
 140     },
 141     hidden => {
 142         description => "Do not identify as a KVM virtual machine.",
 143         type => 'boolean',
 144         optional => 1,
 145         default => 0
 146     },
 147 };
 148
 149 my $watchdog_fmt = {
 150     model => {
 151         default_key => 1,
 152         type => 'string',
 153         enum => [qw(i6300esb ib700)],
 154         description => "Watchdog type to emulate.",
 155         default => 'i6300esb',
 156         optional => 1,
 157     },
 158     action => {
 159         type => 'string',
 160         enum => [qw(reset shutdown poweroff pause debug none)],
 161         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 162         optional => 1,
 163     },
 164 };
 165 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 166
 167 my $confdesc = {
 168     onboot => {
 169         optional => 1,
 170         type => 'boolean',
 171         description => "Specifies whether a VM will be started during system bootup.",
 172         default => 0,
 173     },
 174     autostart => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Automatic restart after crash (currently ignored).",
 178         default => 0,
 179     },
 180     hotplug => {
 181         optional => 1,
 182         type => 'string', format => 'pve-hotplug-features',
 183         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 184         default => 'network,disk,usb',
 185     },
 186     reboot => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 190         default => 1,
 191     },
 192     lock => {
 193         optional => 1,
 194         type => 'string',
 195         description => "Lock/unlock the VM.",
 196         enum => [qw(migrate backup snapshot rollback)],
 197     },
 198     cpulimit => {
 199         optional => 1,
 200         type => 'number',
 201         description => "Limit of CPU usage.",
 202         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 203         minimum => 0,
 204         maximum => 128,
 205         default => 0,
 206     },
 207     cpuunits => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "CPU weight for a VM.",
 211         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 212         minimum => 0,
 213         maximum => 500000,
 214         default => 1000,
 215     },
 216     memory => {
 217         optional => 1,
 218         type => 'integer',
 219         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 220         minimum => 16,
 221         default => 512,
 222     },
 223     balloon => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 227         minimum => 0,
 228     },
 229     shares => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 233         minimum => 0,
 234         maximum => 50000,
 235         default => 1000,
 236     },
 237     keyboard => {
 238         optional => 1,
 239         type => 'string',
 240         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 241         enum => PVE::Tools::kvmkeymaplist(),
 242         default => 'en-us',
 243     },
 244     name => {
 245         optional => 1,
 246         type => 'string', format => 'dns-name',
 247         description => "Set a name for the VM. Only used on the configuration web interface.",
 248     },
 249     scsihw => {
 250         optional => 1,
 251         type => 'string',
 252         description => "SCSI controller model",
 253         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 254         default => 'lsi',
 255     },
 256     description => {
 257         optional => 1,
 258         type => 'string',
 259         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 260     },
 261     ostype => {
 262         optional => 1,
 263         type => 'string',
 264         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 265         description => "Specify guest operating system.",
 266         verbose_description => <<EODESC,
 267 Specify guest operating system. This is used to enable special
 268 optimization/features for specific operating systems:
 269
 270 [horizontal]
 271 other;; unspecified OS
 272 wxp;; Microsoft Windows XP
 273 w2k;; Microsoft Windows 2000
 274 w2k3;; Microsoft Windows 2003
 275 w2k8;; Microsoft Windows 2008
 276 wvista;; Microsoft Windows Vista
 277 win7;; Microsoft Windows 7
 278 win8;; Microsoft Windows 8/2012
 279 l24;; Linux 2.4 Kernel
 280 l26;; Linux 2.6/3.X Kernel
 281 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 282 EODESC
 283     },
 284     boot => {
 285         optional => 1,
 286         type => 'string',
 287         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 288         pattern => '[acdn]{1,4}',
 289         default => 'cdn',
 290     },
 291     bootdisk => {
 292         optional => 1,
 293         type => 'string', format => 'pve-qm-bootdisk',
 294         description => "Enable booting from specified disk.",
 295         pattern => '(ide|sata|scsi|virtio)\d+',
 296     },
 297     smp => {
 298         optional => 1,
 299         type => 'integer',
 300         description => "The number of CPUs. Please use option -sockets instead.",
 301         minimum => 1,
 302         default => 1,
 303     },
 304     sockets => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPU sockets.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     cores => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of cores per socket.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     numa => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Enable/disable NUMA.",
 322         default => 0,
 323     },
 324     hugepages => {
 325         optional => 1,
 326         type => 'string',
 327         description => "Enable/disable hugepages memory.",
 328         enum => [qw(any 2 1024)],
 329     },
 330     vcpus => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Number of hotplugged vcpus.",
 334         minimum => 1,
 335         default => 0,
 336     },
 337     acpi => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Enable/disable ACPI.",
 341         default => 1,
 342     },
 343     agent => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable Qemu GuestAgent.",
 347         default => 0,
 348     },
 349     kvm => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable KVM hardware virtualization.",
 353         default => 1,
 354     },
 355     tdf => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable time drift fix.",
 359         default => 0,
 360     },
 361     localtime => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 365     },
 366     freeze => {
 367         optional => 1,
 368         type => 'boolean',
 369         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 370     },
 371     vga => {
 372         optional => 1,
 373         type => 'string',
 374         description => "Select the VGA type.",
 375         verbose_description => "Select the VGA type. If you want to use high resolution" .
 376             " modes (>= 1280x1024x16) then you should use the options " .
 377             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 378             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 379             "display sever. For win* OS you can select how many independent " .
 380             "displays you want, Linux guests can add displays them self. " .
 381             "You can also run without any graphic card, using a serial device" .
 382             " as terminal.",
 383         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 384     },
 385     watchdog => {
 386         optional => 1,
 387         type => 'string', format => 'pve-qm-watchdog',
 388         description => "Create a virtual hardware watchdog device.",
 389         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 390             " (by a guest action), the watchdog must be periodically polled " .
 391             "by an agent inside the guest or else the watchdog will reset " .
 392             "the guest (or execute the respective action specified)",
 393     },
 394     startdate => {
 395         optional => 1,
 396         type => 'string',
 397         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 398         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 399         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 400         default => 'now',
 401     },
 402     startup =>  get_standard_option('pve-startup-order'),
 403     template => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Enable/disable Template.",
 407         default => 0,
 408     },
 409     args => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Arbitrary arguments passed to kvm.",
 413         verbose_description => <<EODESCR,
 414 Arbitrary arguments passed to kvm, for example:
 415
 416 args: -no-reboot -no-hpet
 417
 418 NOTE: this option is for experts only.
 419 EODESCR
 420     },
 421     tablet => {
 422         optional => 1,
 423         type => 'boolean',
 424         default => 1,
 425         description => "Enable/disable the USB tablet device.",
 426         verbose_description => "Enable/disable the USB tablet device. This device is " .
 427             "usually needed to allow absolute mouse positioning with VNC. " .
 428             "Else the mouse runs out of sync with normal VNC clients. " .
 429             "If you're running lots of console-only guests on one host, " .
 430             "you may consider disabling this to save some context switches. " .
 431             "This is turned off by default if you use spice (-vga=qxl).",
 432     },
 433     migrate_speed => {
 434         optional => 1,
 435         type => 'integer',
 436         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 437         minimum => 0,
 438         default => 0,
 439     },
 440     migrate_downtime => {
 441         optional => 1,
 442         type => 'number',
 443         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 444         minimum => 0,
 445         default => 0.1,
 446     },
 447     cdrom => {
 448         optional => 1,
 449         type => 'string', format => 'pve-qm-drive',
 450         typetext => 'volume',
 451         description => "This is an alias for option -ide2",
 452     },
 453     cpu => {
 454         optional => 1,
 455         description => "Emulated CPU type.",
 456         type => 'string',
 457         format => $cpu_fmt,
 458     },
 459     parent => get_standard_option('pve-snapshot-name', {
 460         optional => 1,
 461         description => "Parent snapshot name. This is used internally, and should not be modified.",
 462     }),
 463     snaptime => {
 464         optional => 1,
 465         description => "Timestamp for snapshots.",
 466         type => 'integer',
 467         minimum => 0,
 468     },
 469     vmstate => {
 470         optional => 1,
 471         type => 'string', format => 'pve-volume-id',
 472         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 473     },
 474     machine => {
 475         description => "Specific the Qemu machine type.",
 476         type => 'string',
 477         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 478         maxLength => 40,
 479         optional => 1,
 480     },
 481     smbios1 => {
 482         description => "Specify SMBIOS type 1 fields.",
 483         type => 'string', format => 'pve-qm-smbios1',
 484         maxLength => 256,
 485         optional => 1,
 486     },
 487     protection => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 491         default => 0,
 492     },
 493     bios => {
 494         optional => 1,
 495         type => 'string',
 496         enum => [ qw(seabios ovmf) ],
 497         description => "Select BIOS implementation.",
 498         default => 'seabios',
 499     },
 500 };
 501
 502 # what about other qemu settings ?
 503 #cpu => 'string',
 504 #machine => 'string',
 505 #fda => 'file',
 506 #fdb => 'file',
 507 #mtdblock => 'file',
 508 #sd => 'file',
 509 #pflash => 'file',
 510 #snapshot => 'bool',
 511 #bootp => 'file',
 512 ##tftp => 'dir',
 513 ##smb => 'dir',
 514 #kernel => 'file',
 515 #append => 'string',
 516 #initrd => 'file',
 517 ##soundhw => 'string',
 518
 519 while (my ($k, $v) = each %$confdesc) {
 520     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 521 }
 522
 523 my $MAX_IDE_DISKS = 4;
 524 my $MAX_SCSI_DISKS = 14;
 525 my $MAX_VIRTIO_DISKS = 16;
 526 my $MAX_SATA_DISKS = 6;
 527 my $MAX_USB_DEVICES = 5;
 528 my $MAX_NETS = 32;
 529 my $MAX_UNUSED_DISKS = 8;
 530 my $MAX_HOSTPCI_DEVICES = 4;
 531 my $MAX_SERIAL_PORTS = 4;
 532 my $MAX_PARALLEL_PORTS = 3;
 533 my $MAX_NUMA = 8;
 534
 535 my $numa_fmt = {
 536     cpus => {
 537         type => "string",
 538         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 539         description => "CPUs accessing this NUMA node.",
 540         format_description => "id[-id];...",
 541     },
 542     memory => {
 543         type => "number",
 544         description => "Amount of memory this NUMA node provides.",
 545         optional => 1,
 546     },
 547     hostnodes => {
 548         type => "string",
 549         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 550         description => "Host NUMA nodes to use.",
 551         format_description => "id[-id];...",
 552         optional => 1,
 553     },
 554     policy => {
 555         type => 'string',
 556         enum => [qw(preferred bind interleave)],
 557         description => "NUMA allocation policy.",
 558         optional => 1,
 559     },
 560 };
 561 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 562 my $numadesc = {
 563     optional => 1,
 564     type => 'string', format => $numa_fmt,
 565     description => "NUMA topology.",
 566 };
 567 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 568
 569 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 570     $confdesc->{"numa$i"} = $numadesc;
 571 }
 572
 573 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 574                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 575                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 576 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 577
 578 my $net_fmt_bridge_descr = <<__EOD__;
 579 Bridge to attach the network device to. The Proxmox VE standard bridge
 580 is called 'vmbr0'.
 581
 582 If you do not specify a bridge, we create a kvm user (NATed) network
 583 device, which provides DHCP and DNS services. The following addresses
 584 are used:
 585
 586  10.0.2.2   Gateway
 587  10.0.2.3   DNS Server
 588  10.0.2.4   SMB Server
 589
 590 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 591 __EOD__
 592
 593 my $net_fmt = {
 594     macaddr => {
 595         type => 'string',
 596         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 597         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 598         format_description => "XX:XX:XX:XX:XX:XX",
 599         optional => 1,
 600     },
 601     model => {
 602         type => 'string',
 603         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 604         format_description => 'model',
 605         enum => $nic_model_list,
 606         default_key => 1,
 607     },
 608     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 609     bridge => {
 610         type => 'string',
 611         description => $net_fmt_bridge_descr,
 612         format_description => 'bridge',
 613         optional => 1,
 614     },
 615     queues => {
 616         type => 'integer',
 617         minimum => 0, maximum => 16,
 618         description => 'Number of packet queues to be used on the device.',
 619         optional => 1,
 620     },
 621     rate => {
 622         type => 'number',
 623         minimum => 0,
 624         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 625         optional => 1,
 626     },
 627     tag => {
 628         type => 'integer',
 629         minimum => 1, maximum => 4094,
 630         description => 'VLAN tag to apply to packets on this interface.',
 631         optional => 1,
 632     },
 633     trunks => {
 634         type => 'string',
 635         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 636         description => 'VLAN trunks to pass through this interface.',
 637         format_description => 'vlanid[;vlanid...]',
 638         optional => 1,
 639     },
 640     firewall => {
 641         type => 'boolean',
 642         description => 'Whether this interface should be protected by the firewall.',
 643         optional => 1,
 644     },
 645     link_down => {
 646         type => 'boolean',
 647         description => 'Whether this interface should be disconnected (like pulling the plug).',
 648         optional => 1,
 649     },
 650 };
 651
 652 my $netdesc = {
 653     optional => 1,
 654     type => 'string', format => $net_fmt,
 655     description => "Specify network devices.",
 656 };
 657
 658 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 659
 660 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 661     $confdesc->{"net$i"} = $netdesc;
 662 }
 663
 664 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 665 sub verify_volume_id_or_qm_path {
 666     my ($volid, $noerr) = @_;
 667
 668     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 669         return $volid;
 670     }
 671
 672     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 673     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 674     if ($@) {
 675         return undef if $noerr;
 676         die $@;
 677     }
 678     return $volid;
 679 }
 680
 681 my $drivename_hash;
 682
 683 my %drivedesc_base = (
 684     volume => { alias => 'file' },
 685     file => {
 686         type => 'string',
 687         format => 'pve-volume-id-or-qm-path',
 688         default_key => 1,
 689         format_description => 'volume',
 690         description => "The drive's backing volume.",
 691     },
 692     media => {
 693         type => 'string',
 694         enum => [qw(cdrom disk)],
 695         description => "The drive's media type.",
 696         default => 'disk',
 697         optional => 1
 698     },
 699     cyls => {
 700         type => 'integer',
 701         description => "Force the drive's physical geometry to have a specific cylinder count.",
 702         optional => 1
 703     },
 704     heads => {
 705         type => 'integer',
 706         description => "Force the drive's physical geometry to have a specific head count.",
 707         optional => 1
 708     },
 709     secs => {
 710         type => 'integer',
 711         description => "Force the drive's physical geometry to have a specific sector count.",
 712         optional => 1
 713     },
 714     trans => {
 715         type => 'string',
 716         enum => [qw(none lba auto)],
 717         description => "Force disk geometry bios translation mode.",
 718         optional => 1,
 719     },
 720     snapshot => {
 721         type => 'boolean',
 722         description => "Whether the drive should be included when making snapshots.",
 723         optional => 1,
 724     },
 725     cache => {
 726         type => 'string',
 727         enum => [qw(none writethrough writeback unsafe directsync)],
 728         description => "The drive's cache mode",
 729         optional => 1,
 730     },
 731     format => {
 732         type => 'string',
 733         format_description => 'image format',
 734         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 735         description => "The drive's backing file's data format.",
 736         optional => 1,
 737     },
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     werror => {
 751         type => 'string',
 752         enum => [qw(enospc ignore report stop)],
 753         description => 'Write error action.',
 754         optional => 1,
 755     },
 756     aio => {
 757         type => 'string',
 758         enum => [qw(native threads)],
 759         description => 'AIO type to use.',
 760         optional => 1,
 761     },
 762     discard => {
 763         type => 'string',
 764         enum => [qw(ignore on)],
 765         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 766         optional => 1,
 767     },
 768     detect_zeroes => {
 769         type => 'boolean',
 770         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 771         optional => 1,
 772     },
 773     serial => {
 774         type => 'string',
 775         format => 'urlencoded',
 776         format_description => 'serial',
 777         maxLength => 20*3, # *3 since it's %xx url enoded
 778         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 779         optional => 1,
 780     }
 781 );
 782
 783 my %rerror_fmt = (
 784     rerror => {
 785         type => 'string',
 786         enum => [qw(ignore report stop)],
 787         description => 'Read error action.',
 788         optional => 1,
 789     },
 790 );
 791
 792 my %iothread_fmt = ( iothread => {
 793         type => 'boolean',
 794         description => "Whether to use iothreads for this drive",
 795         optional => 1,
 796 });
 797
 798 my %model_fmt = (
 799     model => {
 800         type => 'string',
 801         format => 'urlencoded',
 802         format_description => 'model',
 803         maxLength => 40*3, # *3 since it's %xx url enoded
 804         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 805         optional => 1,
 806     },
 807 );
 808
 809 my %queues_fmt = (
 810     queues => {
 811         type => 'integer',
 812         description => "Number of queues.",
 813         minimum => 2,
 814         optional => 1
 815     }
 816 );
 817
 818 my $add_throttle_desc = sub {
 819     my ($key, $type, $what, $unit, $longunit) = @_;
 820     $drivedesc_base{$key} = {
 821         type => $type,
 822         format_description => $unit,
 823         description => "Maximum $what speed in $longunit per second.",
 824         optional => 1,
 825     };
 826 };
 827 # throughput: (leaky bucket)
 828 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 829 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 830 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 831 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 832 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 833 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 834 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 835 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 836 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 837
 838 # pools: (pool of IO before throttling starts taking effect)
 839 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 840 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 841 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 842 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 843 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 844 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 845
 846 my $ide_fmt = {
 847     %drivedesc_base,
 848     %rerror_fmt,
 849     %model_fmt,
 850 };
 851
 852 my $idedesc = {
 853     optional => 1,
 854     type => 'string', format => $ide_fmt,
 855     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 856 };
 857 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 858
 859 my $scsi_fmt = {
 860     %drivedesc_base,
 861     %iothread_fmt,
 862     %queues_fmt,
 863 };
 864 my $scsidesc = {
 865     optional => 1,
 866     type => 'string', format => $scsi_fmt,
 867     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 868 };
 869 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 870
 871 my $sata_fmt = {
 872     %drivedesc_base,
 873     %rerror_fmt,
 874 };
 875 my $satadesc = {
 876     optional => 1,
 877     type => 'string', format => $sata_fmt,
 878     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 881
 882 my $virtio_fmt = {
 883     %drivedesc_base,
 884     %iothread_fmt,
 885     %rerror_fmt,
 886 };
 887 my $virtiodesc = {
 888     optional => 1,
 889     type => 'string', format => $virtio_fmt,
 890     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 891 };
 892 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 893
 894 my $alldrive_fmt = {
 895     %drivedesc_base,
 896     %rerror_fmt,
 897     %iothread_fmt,
 898     %model_fmt,
 899     %queues_fmt,
 900 };
 901
 902 my $usb_fmt = {
 903     host => {
 904         default_key => 1,
 905         type => 'string', format => 'pve-qm-usb-device',
 906         format_description => 'HOSTUSBDEVICE|spice',
 907         description => <<EODESCR,
 908 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 909
 910  'bus-port(.port)*' (decimal numbers) or
 911  'vendor_id:product_id' (hexadeciaml numbers) or
 912  'spice'
 913
 914 You can use the 'lsusb -t' command to list existing usb devices.
 915
 916 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 917
 918 The value 'spice' can be used to add a usb redirection devices for spice.
 919 EODESCR
 920     },
 921     usb3 => {
 922         optional => 1,
 923         type => 'boolean',
 924         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 925         default => 0,
 926     },
 927 };
 928
 929 my $usbdesc = {
 930     optional => 1,
 931     type => 'string', format => $usb_fmt,
 932     description => "Configure an USB device (n is 0 to 4).",
 933 };
 934 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 935
 936 # NOTE: the match-groups of this regex are used in parse_hostpci
 937 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 938 my $hostpci_fmt = {
 939     host => {
 940         default_key => 1,
 941         type => 'string',
 942         pattern => qr/$PCIRE(;$PCIRE)*/,
 943         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 944         description => <<EODESCR,
 945 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 946 of PCI virtual functions of the host. HOSTPCIID syntax is:
 947
 948 'bus:dev.func' (hexadecimal numbers)
 949
 950 You can us the 'lspci' command to list existing PCI devices.
 951 EODESCR
 952     },
 953     rombar => {
 954         type => 'boolean',
 955         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 956         optional => 1,
 957         default => 1,
 958     },
 959     pcie => {
 960         type => 'boolean',
 961         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 962         optional => 1,
 963         default => 0,
 964     },
 965     'x-vga' => {
 966         type => 'boolean',
 967         description =>  "Enable vfio-vga device support.",
 968         optional => 1,
 969         default => 0,
 970     },
 971 };
 972 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 973
 974 my $hostpcidesc = {
 975         optional => 1,
 976         type => 'string', format => 'pve-qm-hostpci',
 977         description => "Map host PCI devices into guest.",
 978         verbose_description =>  <<EODESCR,
 979 Map host PCI devices into guest.
 980
 981 NOTE: This option allows direct access to host hardware. So it is no longer
 982 possible to migrate such machines - use with special care.
 983
 984 CAUTION: Experimental! User reported problems with this option.
 985 EODESCR
 986 };
 987 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 988
 989 my $serialdesc = {
 990         optional => 1,
 991         type => 'string',
 992         pattern => '(/dev/.+|socket)',
 993         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 994         verbose_description =>  <<EODESCR,
 995 Create a serial device inside the VM (n is 0 to 3), and pass through a
 996 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 997 host side (use 'qm terminal' to open a terminal connection).
 998
 999 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1000
1001 CAUTION: Experimental! User reported problems with this option.
1002 EODESCR
1003 };
1004
1005 my $paralleldesc= {
1006         optional => 1,
1007         type => 'string',
1008         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1009         description =>  "Map host parallel devices (n is 0 to 2).",
1010         verbose_description =>  <<EODESCR,
1011 Map host parallel devices (n is 0 to 2).
1012
1013 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1014
1015 CAUTION: Experimental! User reported problems with this option.
1016 EODESCR
1017 };
1018
1019 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1020     $confdesc->{"parallel$i"} = $paralleldesc;
1021 }
1022
1023 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1024     $confdesc->{"serial$i"} = $serialdesc;
1025 }
1026
1027 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1028     $confdesc->{"hostpci$i"} = $hostpcidesc;
1029 }
1030
1031 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1032     $drivename_hash->{"ide$i"} = 1;
1033     $confdesc->{"ide$i"} = $idedesc;
1034 }
1035
1036 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1037     $drivename_hash->{"sata$i"} = 1;
1038     $confdesc->{"sata$i"} = $satadesc;
1039 }
1040
1041 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1042     $drivename_hash->{"scsi$i"} = 1;
1043     $confdesc->{"scsi$i"} = $scsidesc ;
1044 }
1045
1046 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1047     $drivename_hash->{"virtio$i"} = 1;
1048     $confdesc->{"virtio$i"} = $virtiodesc;
1049 }
1050
1051 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1052     $confdesc->{"usb$i"} = $usbdesc;
1053 }
1054
1055 my $unuseddesc = {
1056     optional => 1,
1057     type => 'string', format => 'pve-volume-id',
1058     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1059 };
1060
1061 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1062     $confdesc->{"unused$i"} = $unuseddesc;
1063 }
1064
1065 my $kvm_api_version = 0;
1066
1067 sub kvm_version {
1068
1069     return $kvm_api_version if $kvm_api_version;
1070
1071     my $fh = IO::File->new("</dev/kvm") ||
1072         return 0;
1073
1074     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1075         $kvm_api_version = $v;
1076     }
1077
1078     $fh->close();
1079
1080     return  $kvm_api_version;
1081 }
1082
1083 my $kvm_user_version;
1084
1085 sub kvm_user_version {
1086
1087     return $kvm_user_version if $kvm_user_version;
1088
1089     $kvm_user_version = 'unknown';
1090
1091     my $code = sub {
1092         my $line = shift;
1093         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1094             $kvm_user_version = $2;
1095         }
1096     };
1097
1098     eval { run_command("kvm -version", outfunc => $code); };
1099     warn $@ if $@;
1100
1101     return $kvm_user_version;
1102
1103 }
1104
1105 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1106
1107 sub valid_drive_names {
1108     # order is important - used to autoselect boot disk
1109     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1110             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1111             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1112             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1113 }
1114
1115 sub is_valid_drivename {
1116     my $dev = shift;
1117
1118     return defined($drivename_hash->{$dev});
1119 }
1120
1121 sub option_exists {
1122     my $key = shift;
1123     return defined($confdesc->{$key});
1124 }
1125
1126 sub nic_models {
1127     return $nic_model_list;
1128 }
1129
1130 sub os_list_description {
1131
1132     return {
1133         other => 'Other',
1134         wxp => 'Windows XP',
1135         w2k => 'Windows 2000',
1136         w2k3 =>, 'Windows 2003',
1137         w2k8 => 'Windows 2008',
1138         wvista => 'Windows Vista',
1139         win7 => 'Windows 7',
1140         win8 => 'Windows 8/2012',
1141         l24 => 'Linux 2.4',
1142         l26 => 'Linux 2.6',
1143     };
1144 }
1145
1146 my $cdrom_path;
1147
1148 sub get_cdrom_path {
1149
1150     return  $cdrom_path if $cdrom_path;
1151
1152     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1153     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1154     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1155 }
1156
1157 sub get_iso_path {
1158     my ($storecfg, $vmid, $cdrom) = @_;
1159
1160     if ($cdrom eq 'cdrom') {
1161         return get_cdrom_path();
1162     } elsif ($cdrom eq 'none') {
1163         return '';
1164     } elsif ($cdrom =~ m|^/|) {
1165         return $cdrom;
1166     } else {
1167         return PVE::Storage::path($storecfg, $cdrom);
1168     }
1169 }
1170
1171 # try to convert old style file names to volume IDs
1172 sub filename_to_volume_id {
1173     my ($vmid, $file, $media) = @_;
1174
1175     if (!($file eq 'none' || $file eq 'cdrom' ||
1176           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1177
1178         return undef if $file =~ m|/|;
1179
1180         if ($media && $media eq 'cdrom') {
1181             $file = "local:iso/$file";
1182         } else {
1183             $file = "local:$vmid/$file";
1184         }
1185     }
1186
1187     return $file;
1188 }
1189
1190 sub verify_media_type {
1191     my ($opt, $vtype, $media) = @_;
1192
1193     return if !$media;
1194
1195     my $etype;
1196     if ($media eq 'disk') {
1197         $etype = 'images';
1198     } elsif ($media eq 'cdrom') {
1199         $etype = 'iso';
1200     } else {
1201         die "internal error";
1202     }
1203
1204     return if ($vtype eq $etype);
1205
1206     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1207 }
1208
1209 sub cleanup_drive_path {
1210     my ($opt, $storecfg, $drive) = @_;
1211
1212     # try to convert filesystem paths to volume IDs
1213
1214     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1215         ($drive->{file} !~ m|^/dev/.+|) &&
1216         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1217         ($drive->{file} !~ m/^\d+$/)) {
1218         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1219         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1220         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1221         verify_media_type($opt, $vtype, $drive->{media});
1222         $drive->{file} = $volid;
1223     }
1224
1225     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1226 }
1227
1228 sub parse_hotplug_features {
1229     my ($data) = @_;
1230
1231     my $res = {};
1232
1233     return $res if $data eq '0';
1234
1235     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1236
1237     foreach my $feature (PVE::Tools::split_list($data)) {
1238         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1239             $res->{$1} = 1;
1240         } else {
1241             die "invalid hotplug feature '$feature'\n";
1242         }
1243     }
1244     return $res;
1245 }
1246
1247 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1248 sub pve_verify_hotplug_features {
1249     my ($value, $noerr) = @_;
1250
1251     return $value if parse_hotplug_features($value);
1252
1253     return undef if $noerr;
1254
1255     die "unable to parse hotplug option\n";
1256 }
1257
1258 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1259 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1260 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1261 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1262 #        [,iothread=on][,serial=serial][,model=model]
1263
1264 sub parse_drive {
1265     my ($key, $data) = @_;
1266
1267     my ($interface, $index);
1268
1269     if ($key =~ m/^([^\d]+)(\d+)$/) {
1270         $interface = $1;
1271         $index = $2;
1272     } else {
1273         return undef;
1274     }
1275
1276     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1277                                      : $confdesc->{$key}->{format};
1278     if (!$desc) {
1279         warn "invalid drive key: $key\n";
1280         return undef;
1281     }
1282     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1283     return undef if !$res;
1284     $res->{interface} = $interface;
1285     $res->{index} = $index;
1286
1287     my $error = 0;
1288     foreach my $opt (qw(bps bps_rd bps_wr)) {
1289         if (my $bps = defined(delete $res->{$opt})) {
1290             if (defined($res->{"m$opt"})) {
1291                 warn "both $opt and m$opt specified\n";
1292                 ++$error;
1293                 next;
1294             }
1295             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1296         }
1297     }
1298     return undef if $error;
1299
1300     return undef if $res->{mbps_rd} && $res->{mbps};
1301     return undef if $res->{mbps_wr} && $res->{mbps};
1302     return undef if $res->{iops_rd} && $res->{iops};
1303     return undef if $res->{iops_wr} && $res->{iops};
1304
1305     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1306         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1307         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1308         return undef if $res->{interface} eq 'virtio';
1309     }
1310
1311     if (my $size = $res->{size}) {
1312         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1313     }
1314
1315     return $res;
1316 }
1317
1318 sub print_drive {
1319     my ($vmid, $drive) = @_;
1320     my $data = { %$drive };
1321     delete $data->{$_} for qw(index interface);
1322     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1323 }
1324
1325 sub scsi_inquiry {
1326     my($fh, $noerr) = @_;
1327
1328     my $SG_IO = 0x2285;
1329     my $SG_GET_VERSION_NUM = 0x2282;
1330
1331     my $versionbuf = "\x00" x 8;
1332     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333     if (!$ret) {
1334         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335         return undef;
1336     }
1337     my $version = unpack("I", $versionbuf);
1338     if ($version < 30000) {
1339         die "scsi generic interface too old\n"  if !$noerr;
1340         return undef;
1341     }
1342
1343     my $buf = "\x00" x 36;
1344     my $sensebuf = "\x00" x 8;
1345     my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347     # see /usr/include/scsi/sg.h
1348     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351                       length($sensebuf), 0, length($buf), $buf,
1352                       $cmd, $sensebuf, 6000);
1353
1354     $ret = ioctl($fh, $SG_IO, $packet);
1355     if (!$ret) {
1356         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357         return undef;
1358     }
1359
1360     my @res = unpack($sg_io_hdr_t, $packet);
1361     if ($res[17] || $res[18]) {
1362         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363         return undef;
1364     }
1365
1366     my $res = {};
1367     (my $byte0, my $byte1, $res->{vendor},
1368      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370     $res->{removable} = $byte1 & 128 ? 1 : 0;
1371     $res->{type} = $byte0 & 31;
1372
1373     return $res;
1374 }
1375
1376 sub path_is_scsi {
1377     my ($path) = @_;
1378
1379     my $fh = IO::File->new("+<$path") || return undef;
1380     my $res = scsi_inquiry($fh, 1);
1381     close($fh);
1382
1383     return $res;
1384 }
1385
1386 sub machine_type_is_q35 {
1387     my ($conf) = @_;
1388
1389     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1390 }
1391
1392 sub print_tabletdevice_full {
1393     my ($conf) = @_;
1394
1395     my $q35 = machine_type_is_q35($conf);
1396
1397     # we use uhci for old VMs because tablet driver was buggy in older qemu
1398     my $usbbus = $q35 ? "ehci" : "uhci";
1399
1400     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1401 }
1402
1403 sub print_drivedevice_full {
1404     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1405
1406     my $device = '';
1407     my $maxdev = 0;
1408
1409     if ($drive->{interface} eq 'virtio') {
1410         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1411         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1412         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1413     } elsif ($drive->{interface} eq 'scsi') {
1414
1415         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1416         my $unit = $drive->{index} % $maxdev;
1417         my $devicetype = 'hd';
1418         my $path = '';
1419         if (drive_is_cdrom($drive)) {
1420             $devicetype = 'cd';
1421         } else {
1422             if ($drive->{file} =~ m|^/|) {
1423                 $path = $drive->{file};
1424                 if (my $info = path_is_scsi($path)) {
1425                     if ($info->{type} == 0) {
1426                         $devicetype = 'block';
1427                     } elsif ($info->{type} == 1) { # tape
1428                         $devicetype = 'generic';
1429                     }
1430                 }
1431             } else {
1432                  $path = PVE::Storage::path($storecfg, $drive->{file});
1433             }
1434
1435             if($path =~ m/^iscsi\:\/\//){
1436                 $devicetype = 'generic';
1437             }
1438         }
1439
1440         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1441             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1442         } else {
1443             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1444         }
1445
1446     } elsif ($drive->{interface} eq 'ide'){
1447         $maxdev = 2;
1448         my $controller = int($drive->{index} / $maxdev);
1449         my $unit = $drive->{index} % $maxdev;
1450         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1451
1452         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1453         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1454             $model = URI::Escape::uri_unescape($model);
1455             $device .= ",model=$model";
1456         }
1457     } elsif ($drive->{interface} eq 'sata'){
1458         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1459         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1460         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1461     } elsif ($drive->{interface} eq 'usb') {
1462         die "implement me";
1463         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1464     } else {
1465         die "unsupported interface type";
1466     }
1467
1468     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1469
1470     return $device;
1471 }
1472
1473 sub get_initiator_name {
1474     my $initiator;
1475
1476     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1477     while (defined(my $line = <$fh>)) {
1478         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1479         $initiator = $1;
1480         last;
1481     }
1482     $fh->close();
1483
1484     return $initiator;
1485 }
1486
1487 sub print_drive_full {
1488     my ($storecfg, $vmid, $drive) = @_;
1489
1490     my $path;
1491     my $volid = $drive->{file};
1492     my $format;
1493
1494     if (drive_is_cdrom($drive)) {
1495         $path = get_iso_path($storecfg, $vmid, $volid);
1496     } else {
1497         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1498         if ($storeid) {
1499             $path = PVE::Storage::path($storecfg, $volid);
1500             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1501             $format = qemu_img_format($scfg, $volname);
1502         } else {
1503             $path = $volid;
1504             $format = "raw";
1505         }
1506    }
1507
1508     my $opts = '';
1509     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1510     foreach my $o (@qemu_drive_options) {
1511         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1512     }
1513     if (my $serial = $drive->{serial}) {
1514         $serial = URI::Escape::uri_unescape($serial);
1515         $opts .= ",serial=$serial";
1516     }
1517
1518     $opts .= ",format=$format" if $format && !$drive->{format};
1519
1520     foreach my $o (qw(bps bps_rd bps_wr)) {
1521         my $v = $drive->{"m$o"};
1522         $opts .= ",$o=" . int($v*1024*1024) if $v;
1523     }
1524
1525     my $cache_direct = 0;
1526
1527     if (my $cache = $drive->{cache}) {
1528         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1529     } elsif (!drive_is_cdrom($drive)) {
1530         $opts .= ",cache=none";
1531         $cache_direct = 1;
1532     }
1533
1534     # aio native works only with O_DIRECT
1535     if (!$drive->{aio}) {
1536         if($cache_direct) {
1537             $opts .= ",aio=native";
1538         } else {
1539             $opts .= ",aio=threads";
1540         }
1541     }
1542
1543     if (!drive_is_cdrom($drive)) {
1544         my $detectzeroes;
1545         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1546             $detectzeroes = 'off';
1547         } elsif ($drive->{discard}) {
1548             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1549         } else {
1550             # This used to be our default with discard not being specified:
1551             $detectzeroes = 'on';
1552         }
1553         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1554     }
1555
1556     my $pathinfo = $path ? "file=$path," : '';
1557
1558     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1559 }
1560
1561 sub print_netdevice_full {
1562     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1563
1564     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1565
1566     my $device = $net->{model};
1567     if ($net->{model} eq 'virtio') {
1568          $device = 'virtio-net-pci';
1569      };
1570
1571     my $pciaddr = print_pci_addr("$netid", $bridges);
1572     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1573     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1574         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1575         my $vectors = $net->{queues} * 2 + 2;
1576         $tmpstr .= ",vectors=$vectors,mq=on";
1577     }
1578     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1579
1580     if ($use_old_bios_files) {
1581         my $romfile;
1582         if ($device eq 'virtio-net-pci') {
1583             $romfile = 'pxe-virtio.rom';
1584         } elsif ($device eq 'e1000') {
1585             $romfile = 'pxe-e1000.rom';
1586         } elsif ($device eq 'ne2k') {
1587             $romfile = 'pxe-ne2k_pci.rom';
1588         } elsif ($device eq 'pcnet') {
1589             $romfile = 'pxe-pcnet.rom';
1590         } elsif ($device eq 'rtl8139') {
1591             $romfile = 'pxe-rtl8139.rom';
1592         }
1593         $tmpstr .= ",romfile=$romfile" if $romfile;
1594     }
1595
1596     return $tmpstr;
1597 }
1598
1599 sub print_netdev_full {
1600     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1601
1602     my $i = '';
1603     if ($netid =~ m/^net(\d+)$/) {
1604         $i = int($1);
1605     }
1606
1607     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1608
1609     my $ifname = "tap${vmid}i$i";
1610
1611     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1612     die "interface name '$ifname' is too long (max 15 character)\n"
1613         if length($ifname) >= 16;
1614
1615     my $vhostparam = '';
1616     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1617
1618     my $vmname = $conf->{name} || "vm$vmid";
1619
1620     my $netdev = "";
1621     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1622
1623     if ($net->{bridge}) {
1624         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1625     } else {
1626         $netdev = "type=user,id=$netid,hostname=$vmname";
1627     }
1628
1629     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1630
1631     return $netdev;
1632 }
1633
1634 sub drive_is_cdrom {
1635     my ($drive) = @_;
1636
1637     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1638
1639 }
1640
1641 sub parse_number_sets {
1642     my ($set) = @_;
1643     my $res = [];
1644     foreach my $part (split(/;/, $set)) {
1645         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1646             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1647             push @$res, [ $1, $2 ];
1648         } else {
1649             die "invalid range: $part\n";
1650         }
1651     }
1652     return $res;
1653 }
1654
1655 sub parse_numa {
1656     my ($data) = @_;
1657
1658     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1659     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1660     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1661     return $res;
1662 }
1663
1664 sub parse_hostpci {
1665     my ($value) = @_;
1666
1667     return undef if !$value;
1668
1669     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1670
1671     my @idlist = split(/;/, $res->{host});
1672     delete $res->{host};
1673     foreach my $id (@idlist) {
1674         if ($id =~ /^$PCIRE$/) {
1675             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1676         } else {
1677             # should have been caught by parse_property_string already
1678             die "failed to parse PCI id: $id\n";
1679         }
1680     }
1681     return $res;
1682 }
1683
1684 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1685 sub parse_net {
1686     my ($data) = @_;
1687
1688     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1689     if ($@) {
1690         warn $@;
1691         return undef;
1692     }
1693     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1694     return $res;
1695 }
1696
1697 sub print_net {
1698     my $net = shift;
1699
1700     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1701 }
1702
1703 sub add_random_macs {
1704     my ($settings) = @_;
1705
1706     foreach my $opt (keys %$settings) {
1707         next if $opt !~ m/^net(\d+)$/;
1708         my $net = parse_net($settings->{$opt});
1709         next if !$net;
1710         $settings->{$opt} = print_net($net);
1711     }
1712 }
1713
1714 sub vm_is_volid_owner {
1715     my ($storecfg, $vmid, $volid) = @_;
1716
1717     if ($volid !~  m|^/|) {
1718         my ($path, $owner);
1719         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1720         if ($owner && ($owner == $vmid)) {
1721             return 1;
1722         }
1723     }
1724
1725     return undef;
1726 }
1727
1728 sub split_flagged_list {
1729     my $text = shift || '';
1730     $text =~ s/[,;]/ /g;
1731     $text =~ s/^\s+//;
1732     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1733 }
1734
1735 sub join_flagged_list {
1736     my ($how, $lst) = @_;
1737     join $how, map { $lst->{$_} . $_ } keys %$lst;
1738 }
1739
1740 sub vmconfig_delete_pending_option {
1741     my ($conf, $key, $force) = @_;
1742
1743     delete $conf->{pending}->{$key};
1744     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1745     $pending_delete_hash->{$key} = $force ? '!' : '';
1746     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1747 }
1748
1749 sub vmconfig_undelete_pending_option {
1750     my ($conf, $key) = @_;
1751
1752     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1753     delete $pending_delete_hash->{$key};
1754
1755     if (%$pending_delete_hash) {
1756         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1757     } else {
1758         delete $conf->{pending}->{delete};
1759     }
1760 }
1761
1762 sub vmconfig_register_unused_drive {
1763     my ($storecfg, $vmid, $conf, $drive) = @_;
1764
1765     if (!drive_is_cdrom($drive)) {
1766         my $volid = $drive->{file};
1767         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1768             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1769         }
1770     }
1771 }
1772
1773 sub vmconfig_cleanup_pending {
1774     my ($conf) = @_;
1775
1776     # remove pending changes when nothing changed
1777     my $changes;
1778     foreach my $opt (keys %{$conf->{pending}}) {
1779         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1780             $changes = 1;
1781             delete $conf->{pending}->{$opt};
1782         }
1783     }
1784
1785     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1786     my $pending_delete_hash = {};
1787     while (my ($opt, $force) = each %$current_delete_hash) {
1788         if (defined($conf->{$opt})) {
1789             $pending_delete_hash->{$opt} = $force;
1790         } else {
1791             $changes = 1;
1792         }
1793     }
1794
1795     if (%$pending_delete_hash) {
1796         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1797     } else {
1798         delete $conf->{pending}->{delete};
1799     }
1800
1801     return $changes;
1802 }
1803
1804 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1805 my $smbios1_fmt = {
1806     uuid => {
1807         type => 'string',
1808         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1809         format_description => 'UUID',
1810         description => "Set SMBIOS1 UUID.",
1811         optional => 1,
1812     },
1813     version => {
1814         type => 'string',
1815         pattern => '\S+',
1816         format_description => 'string',
1817         description => "Set SMBIOS1 version.",
1818         optional => 1,
1819     },
1820     serial => {
1821         type => 'string',
1822         pattern => '\S+',
1823         format_description => 'string',
1824         description => "Set SMBIOS1 serial number.",
1825         optional => 1,
1826     },
1827     manufacturer => {
1828         type => 'string',
1829         pattern => '\S+',
1830         format_description => 'string',
1831         description => "Set SMBIOS1 manufacturer.",
1832         optional => 1,
1833     },
1834     product => {
1835         type => 'string',
1836         pattern => '\S+',
1837         format_description => 'string',
1838         description => "Set SMBIOS1 product ID.",
1839         optional => 1,
1840     },
1841     sku => {
1842         type => 'string',
1843         pattern => '\S+',
1844         format_description => 'string',
1845         description => "Set SMBIOS1 SKU string.",
1846         optional => 1,
1847     },
1848     family => {
1849         type => 'string',
1850         pattern => '\S+',
1851         format_description => 'string',
1852         description => "Set SMBIOS1 family string.",
1853         optional => 1,
1854     },
1855 };
1856
1857 sub parse_smbios1 {
1858     my ($data) = @_;
1859
1860     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1861     warn $@ if $@;
1862     return $res;
1863 }
1864
1865 sub print_smbios1 {
1866     my ($smbios1) = @_;
1867     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1868 }
1869
1870 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1871
1872 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1873 sub verify_bootdisk {
1874     my ($value, $noerr) = @_;
1875
1876     return $value if is_valid_drivename($value);
1877
1878     return undef if $noerr;
1879
1880     die "invalid boot disk '$value'\n";
1881 }
1882
1883 sub parse_watchdog {
1884     my ($value) = @_;
1885
1886     return undef if !$value;
1887
1888     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1889     warn $@ if $@;
1890     return $res;
1891 }
1892
1893 sub parse_usb_device {
1894     my ($value) = @_;
1895
1896     return undef if !$value;
1897
1898     my $res = {};
1899     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1900         $res->{vendorid} = $2;
1901         $res->{productid} = $4;
1902     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1903         $res->{hostbus} = $1;
1904         $res->{hostport} = $2;
1905     } elsif ($value =~ m/^spice$/i) {
1906         $res->{spice} = 1;
1907     } else {
1908         return undef;
1909     }
1910
1911     return $res;
1912 }
1913
1914 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1915 sub verify_usb_device {
1916     my ($value, $noerr) = @_;
1917
1918     return $value if parse_usb_device($value);
1919
1920     return undef if $noerr;
1921
1922     die "unable to parse usb device\n";
1923 }
1924
1925 # add JSON properties for create and set function
1926 sub json_config_properties {
1927     my $prop = shift;
1928
1929     foreach my $opt (keys %$confdesc) {
1930         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1931         $prop->{$opt} = $confdesc->{$opt};
1932     }
1933
1934     return $prop;
1935 }
1936
1937 sub check_type {
1938     my ($key, $value) = @_;
1939
1940     die "unknown setting '$key'\n" if !$confdesc->{$key};
1941
1942     my $type = $confdesc->{$key}->{type};
1943
1944     if (!defined($value)) {
1945         die "got undefined value\n";
1946     }
1947
1948     if ($value =~ m/[\n\r]/) {
1949         die "property contains a line feed\n";
1950     }
1951
1952     if ($type eq 'boolean') {
1953         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1954         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1955         die "type check ('boolean') failed - got '$value'\n";
1956     } elsif ($type eq 'integer') {
1957         return int($1) if $value =~ m/^(\d+)$/;
1958         die "type check ('integer') failed - got '$value'\n";
1959     } elsif ($type eq 'number') {
1960         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1961         die "type check ('number') failed - got '$value'\n";
1962     } elsif ($type eq 'string') {
1963         if (my $fmt = $confdesc->{$key}->{format}) {
1964             if ($fmt eq 'pve-qm-drive') {
1965                 # special case - we need to pass $key to parse_drive()
1966                 my $drive = parse_drive($key, $value);
1967                 return $value if $drive;
1968                 die "unable to parse drive options\n";
1969             }
1970             PVE::JSONSchema::check_format($fmt, $value);
1971             return $value;
1972         }
1973         $value =~ s/^\"(.*)\"$/$1/;
1974         return $value;
1975     } else {
1976         die "internal error"
1977     }
1978 }
1979
1980 sub check_iommu_support{
1981     #fixme : need to check IOMMU support
1982     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1983
1984     my $iommu=1;
1985     return $iommu;
1986
1987 }
1988
1989 sub touch_config {
1990     my ($vmid) = @_;
1991
1992     my $conf = PVE::QemuConfig->config_file($vmid);
1993     utime undef, undef, $conf;
1994 }
1995
1996 sub destroy_vm {
1997     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1998
1999     my $conffile = PVE::QemuConfig->config_file($vmid);
2000
2001     my $conf = PVE::QemuConfig->load_config($vmid);
2002
2003     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2004
2005     # only remove disks owned by this VM
2006     foreach_drive($conf, sub {
2007         my ($ds, $drive) = @_;
2008
2009         return if drive_is_cdrom($drive);
2010
2011         my $volid = $drive->{file};
2012
2013         return if !$volid || $volid =~ m|^/|;
2014
2015         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2016         return if !$path || !$owner || ($owner != $vmid);
2017
2018         PVE::Storage::vdisk_free($storecfg, $volid);
2019     });
2020
2021     if ($keep_empty_config) {
2022         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2023     } else {
2024         unlink $conffile;
2025     }
2026
2027     # also remove unused disk
2028     eval {
2029         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2030
2031         eval {
2032             PVE::Storage::foreach_volid($dl, sub {
2033                 my ($volid, $sid, $volname, $d) = @_;
2034                 PVE::Storage::vdisk_free($storecfg, $volid);
2035             });
2036         };
2037         warn $@ if $@;
2038
2039     };
2040     warn $@ if $@;
2041 }
2042
2043 sub parse_vm_config {
2044     my ($filename, $raw) = @_;
2045
2046     return undef if !defined($raw);
2047
2048     my $res = {
2049         digest => Digest::SHA::sha1_hex($raw),
2050         snapshots => {},
2051         pending => {},
2052     };
2053
2054     $filename =~ m|/qemu-server/(\d+)\.conf$|
2055         || die "got strange filename '$filename'";
2056
2057     my $vmid = $1;
2058
2059     my $conf = $res;
2060     my $descr;
2061     my $section = '';
2062
2063     my @lines = split(/\n/, $raw);
2064     foreach my $line (@lines) {
2065         next if $line =~ m/^\s*$/;
2066
2067         if ($line =~ m/^\[PENDING\]\s*$/i) {
2068             $section = 'pending';
2069             if (defined($descr)) {
2070                 $descr =~ s/\s+$//;
2071                 $conf->{description} = $descr;
2072             }
2073             $descr = undef;
2074             $conf = $res->{$section} = {};
2075             next;
2076
2077         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2078             $section = $1;
2079             if (defined($descr)) {
2080                 $descr =~ s/\s+$//;
2081                 $conf->{description} = $descr;
2082             }
2083             $descr = undef;
2084             $conf = $res->{snapshots}->{$section} = {};
2085             next;
2086         }
2087
2088         if ($line =~ m/^\#(.*)\s*$/) {
2089             $descr = '' if !defined($descr);
2090             $descr .= PVE::Tools::decode_text($1) . "\n";
2091             next;
2092         }
2093
2094         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2095             $descr = '' if !defined($descr);
2096             $descr .= PVE::Tools::decode_text($2);
2097         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2098             $conf->{snapstate} = $1;
2099         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2100             my $key = $1;
2101             my $value = $2;
2102             $conf->{$key} = $value;
2103         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2104             my $value = $1;
2105             if ($section eq 'pending') {
2106                 $conf->{delete} = $value; # we parse this later
2107             } else {
2108                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2109             }
2110         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2111             my $key = $1;
2112             my $value = $2;
2113             eval { $value = check_type($key, $value); };
2114             if ($@) {
2115                 warn "vm $vmid - unable to parse value of '$key' - $@";
2116             } else {
2117                 my $fmt = $confdesc->{$key}->{format};
2118                 if ($fmt && $fmt eq 'pve-qm-drive') {
2119                     my $v = parse_drive($key, $value);
2120                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2121                         $v->{file} = $volid;
2122                         $value = print_drive($vmid, $v);
2123                     } else {
2124                         warn "vm $vmid - unable to parse value of '$key'\n";
2125                         next;
2126                     }
2127                 }
2128
2129                 if ($key eq 'cdrom') {
2130                     $conf->{ide2} = $value;
2131                 } else {
2132                     $conf->{$key} = $value;
2133                 }
2134             }
2135         }
2136     }
2137
2138     if (defined($descr)) {
2139         $descr =~ s/\s+$//;
2140         $conf->{description} = $descr;
2141     }
2142     delete $res->{snapstate}; # just to be sure
2143
2144     return $res;
2145 }
2146
2147 sub write_vm_config {
2148     my ($filename, $conf) = @_;
2149
2150     delete $conf->{snapstate}; # just to be sure
2151
2152     if ($conf->{cdrom}) {
2153         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2154         $conf->{ide2} = $conf->{cdrom};
2155         delete $conf->{cdrom};
2156     }
2157
2158     # we do not use 'smp' any longer
2159     if ($conf->{sockets}) {
2160         delete $conf->{smp};
2161     } elsif ($conf->{smp}) {
2162         $conf->{sockets} = $conf->{smp};
2163         delete $conf->{cores};
2164         delete $conf->{smp};
2165     }
2166
2167     my $used_volids = {};
2168
2169     my $cleanup_config = sub {
2170         my ($cref, $pending, $snapname) = @_;
2171
2172         foreach my $key (keys %$cref) {
2173             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2174                 $key eq 'snapstate' || $key eq 'pending';
2175             my $value = $cref->{$key};
2176             if ($key eq 'delete') {
2177                 die "propertry 'delete' is only allowed in [PENDING]\n"
2178                     if !$pending;
2179                 # fixme: check syntax?
2180                 next;
2181             }
2182             eval { $value = check_type($key, $value); };
2183             die "unable to parse value of '$key' - $@" if $@;
2184
2185             $cref->{$key} = $value;
2186
2187             if (!$snapname && is_valid_drivename($key)) {
2188                 my $drive = parse_drive($key, $value);
2189                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2190             }
2191         }
2192     };
2193
2194     &$cleanup_config($conf);
2195
2196     &$cleanup_config($conf->{pending}, 1);
2197
2198     foreach my $snapname (keys %{$conf->{snapshots}}) {
2199         die "internal error" if $snapname eq 'pending';
2200         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2201     }
2202
2203     # remove 'unusedX' settings if we re-add a volume
2204     foreach my $key (keys %$conf) {
2205         my $value = $conf->{$key};
2206         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2207             delete $conf->{$key};
2208         }
2209     }
2210
2211     my $generate_raw_config = sub {
2212         my ($conf, $pending) = @_;
2213
2214         my $raw = '';
2215
2216         # add description as comment to top of file
2217         if (defined(my $descr = $conf->{description})) {
2218             if ($descr) {
2219                 foreach my $cl (split(/\n/, $descr)) {
2220                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2221                 }
2222             } else {
2223                 $raw .= "#\n" if $pending;
2224             }
2225         }
2226
2227         foreach my $key (sort keys %$conf) {
2228             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2229             $raw .= "$key: $conf->{$key}\n";
2230         }
2231         return $raw;
2232     };
2233
2234     my $raw = &$generate_raw_config($conf);
2235
2236     if (scalar(keys %{$conf->{pending}})){
2237         $raw .= "\n[PENDING]\n";
2238         $raw .= &$generate_raw_config($conf->{pending}, 1);
2239     }
2240
2241     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2242         $raw .= "\n[$snapname]\n";
2243         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2244     }
2245
2246     return $raw;
2247 }
2248
2249 sub load_defaults {
2250
2251     my $res = {};
2252
2253     # we use static defaults from our JSON schema configuration
2254     foreach my $key (keys %$confdesc) {
2255         if (defined(my $default = $confdesc->{$key}->{default})) {
2256             $res->{$key} = $default;
2257         }
2258     }
2259
2260     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2261     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2262
2263     return $res;
2264 }
2265
2266 sub config_list {
2267     my $vmlist = PVE::Cluster::get_vmlist();
2268     my $res = {};
2269     return $res if !$vmlist || !$vmlist->{ids};
2270     my $ids = $vmlist->{ids};
2271
2272     foreach my $vmid (keys %$ids) {
2273         my $d = $ids->{$vmid};
2274         next if !$d->{node} || $d->{node} ne $nodename;
2275         next if !$d->{type} || $d->{type} ne 'qemu';
2276         $res->{$vmid}->{exists} = 1;
2277     }
2278     return $res;
2279 }
2280
2281 # test if VM uses local resources (to prevent migration)
2282 sub check_local_resources {
2283     my ($conf, $noerr) = @_;
2284
2285     my $loc_res = 0;
2286
2287     $loc_res = 1 if $conf->{hostusb}; # old syntax
2288     $loc_res = 1 if $conf->{hostpci}; # old syntax
2289
2290     foreach my $k (keys %$conf) {
2291         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2292         # sockets are safe: they will recreated be on the target side post-migrate
2293         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2294         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2295     }
2296
2297     die "VM uses local resources\n" if $loc_res && !$noerr;
2298
2299     return $loc_res;
2300 }
2301
2302 # check if used storages are available on all nodes (use by migrate)
2303 sub check_storage_availability {
2304     my ($storecfg, $conf, $node) = @_;
2305
2306     foreach_drive($conf, sub {
2307         my ($ds, $drive) = @_;
2308
2309         my $volid = $drive->{file};
2310         return if !$volid;
2311
2312         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2313         return if !$sid;
2314
2315         # check if storage is available on both nodes
2316         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2317         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2318    });
2319 }
2320
2321 # list nodes where all VM images are available (used by has_feature API)
2322 sub shared_nodes {
2323     my ($conf, $storecfg) = @_;
2324
2325     my $nodelist = PVE::Cluster::get_nodelist();
2326     my $nodehash = { map { $_ => 1 } @$nodelist };
2327     my $nodename = PVE::INotify::nodename();
2328
2329     foreach_drive($conf, sub {
2330         my ($ds, $drive) = @_;
2331
2332         my $volid = $drive->{file};
2333         return if !$volid;
2334
2335         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2336         if ($storeid) {
2337             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2338             if ($scfg->{disable}) {
2339                 $nodehash = {};
2340             } elsif (my $avail = $scfg->{nodes}) {
2341                 foreach my $node (keys %$nodehash) {
2342                     delete $nodehash->{$node} if !$avail->{$node};
2343                 }
2344             } elsif (!$scfg->{shared}) {
2345                 foreach my $node (keys %$nodehash) {
2346                     delete $nodehash->{$node} if $node ne $nodename
2347                 }
2348             }
2349         }
2350     });
2351
2352     return $nodehash
2353 }
2354
2355 sub check_cmdline {
2356     my ($pidfile, $pid) = @_;
2357
2358     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2359     if (defined($fh)) {
2360         my $line = <$fh>;
2361         $fh->close;
2362         return undef if !$line;
2363         my @param = split(/\0/, $line);
2364
2365         my $cmd = $param[0];
2366         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2367
2368         for (my $i = 0; $i < scalar (@param); $i++) {
2369             my $p = $param[$i];
2370             next if !$p;
2371             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2372                 my $p = $param[$i+1];
2373                 return 1 if $p && ($p eq $pidfile);
2374                 return undef;
2375             }
2376         }
2377     }
2378     return undef;
2379 }
2380
2381 sub check_running {
2382     my ($vmid, $nocheck, $node) = @_;
2383
2384     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2385
2386     die "unable to find configuration file for VM $vmid - no such machine\n"
2387         if !$nocheck && ! -f $filename;
2388
2389     my $pidfile = pidfile_name($vmid);
2390
2391     if (my $fd = IO::File->new("<$pidfile")) {
2392         my $st = stat($fd);
2393         my $line = <$fd>;
2394         close($fd);
2395
2396         my $mtime = $st->mtime;
2397         if ($mtime > time()) {
2398             warn "file '$filename' modified in future\n";
2399         }
2400
2401         if ($line =~ m/^(\d+)$/) {
2402             my $pid = $1;
2403             if (check_cmdline($pidfile, $pid)) {
2404                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2405                     return $pid;
2406                 }
2407             }
2408         }
2409     }
2410
2411     return undef;
2412 }
2413
2414 sub vzlist {
2415
2416     my $vzlist = config_list();
2417
2418     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2419
2420     while (defined(my $de = $fd->read)) {
2421         next if $de !~ m/^(\d+)\.pid$/;
2422         my $vmid = $1;
2423         next if !defined($vzlist->{$vmid});
2424         if (my $pid = check_running($vmid)) {
2425             $vzlist->{$vmid}->{pid} = $pid;
2426         }
2427     }
2428
2429     return $vzlist;
2430 }
2431
2432 sub disksize {
2433     my ($storecfg, $conf) = @_;
2434
2435     my $bootdisk = $conf->{bootdisk};
2436     return undef if !$bootdisk;
2437     return undef if !is_valid_drivename($bootdisk);
2438
2439     return undef if !$conf->{$bootdisk};
2440
2441     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2442     return undef if !defined($drive);
2443
2444     return undef if drive_is_cdrom($drive);
2445
2446     my $volid = $drive->{file};
2447     return undef if !$volid;
2448
2449     return $drive->{size};
2450 }
2451
2452 my $last_proc_pid_stat;
2453
2454 # get VM status information
2455 # This must be fast and should not block ($full == false)
2456 # We only query KVM using QMP if $full == true (this can be slow)
2457 sub vmstatus {
2458     my ($opt_vmid, $full) = @_;
2459
2460     my $res = {};
2461
2462     my $storecfg = PVE::Storage::config();
2463
2464     my $list = vzlist();
2465     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2466
2467     my $cpucount = $cpuinfo->{cpus} || 1;
2468
2469     foreach my $vmid (keys %$list) {
2470         next if $opt_vmid && ($vmid ne $opt_vmid);
2471
2472         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2473         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2474
2475         my $d = {};
2476         $d->{pid} = $list->{$vmid}->{pid};
2477
2478         # fixme: better status?
2479         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2480
2481         my $size = disksize($storecfg, $conf);
2482         if (defined($size)) {
2483             $d->{disk} = 0; # no info available
2484             $d->{maxdisk} = $size;
2485         } else {
2486             $d->{disk} = 0;
2487             $d->{maxdisk} = 0;
2488         }
2489
2490         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2491         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2492         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2493
2494         $d->{name} = $conf->{name} || "VM $vmid";
2495         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2496
2497         if ($conf->{balloon}) {
2498             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2499             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2500         }
2501
2502         $d->{uptime} = 0;
2503         $d->{cpu} = 0;
2504         $d->{mem} = 0;
2505
2506         $d->{netout} = 0;
2507         $d->{netin} = 0;
2508
2509         $d->{diskread} = 0;
2510         $d->{diskwrite} = 0;
2511
2512         $d->{template} = PVE::QemuConfig->is_template($conf);
2513
2514         $res->{$vmid} = $d;
2515     }
2516
2517     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2518     foreach my $dev (keys %$netdev) {
2519         next if $dev !~ m/^tap([1-9]\d*)i/;
2520         my $vmid = $1;
2521         my $d = $res->{$vmid};
2522         next if !$d;
2523
2524         $d->{netout} += $netdev->{$dev}->{receive};
2525         $d->{netin} += $netdev->{$dev}->{transmit};
2526
2527         if ($full) {
2528             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2529             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2530         }
2531
2532     }
2533
2534     my $ctime = gettimeofday;
2535
2536     foreach my $vmid (keys %$list) {
2537
2538         my $d = $res->{$vmid};
2539         my $pid = $d->{pid};
2540         next if !$pid;
2541
2542         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2543         next if !$pstat; # not running
2544
2545         my $used = $pstat->{utime} + $pstat->{stime};
2546
2547         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2548
2549         if ($pstat->{vsize}) {
2550             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2551         }
2552
2553         my $old = $last_proc_pid_stat->{$pid};
2554         if (!$old) {
2555             $last_proc_pid_stat->{$pid} = {
2556                 time => $ctime,
2557                 used => $used,
2558                 cpu => 0,
2559             };
2560             next;
2561         }
2562
2563         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2564
2565         if ($dtime > 1000) {
2566             my $dutime = $used -  $old->{used};
2567
2568             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2569             $last_proc_pid_stat->{$pid} = {
2570                 time => $ctime,
2571                 used => $used,
2572                 cpu => $d->{cpu},
2573             };
2574         } else {
2575             $d->{cpu} = $old->{cpu};
2576         }
2577     }
2578
2579     return $res if !$full;
2580
2581     my $qmpclient = PVE::QMPClient->new();
2582
2583     my $ballooncb = sub {
2584         my ($vmid, $resp) = @_;
2585
2586         my $info = $resp->{'return'};
2587         return if !$info->{max_mem};
2588
2589         my $d = $res->{$vmid};
2590
2591         # use memory assigned to VM
2592         $d->{maxmem} = $info->{max_mem};
2593         $d->{balloon} = $info->{actual};
2594
2595         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2596             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2597             $d->{freemem} = $info->{free_mem};
2598         }
2599
2600         $d->{ballooninfo} = $info;
2601     };
2602
2603     my $blockstatscb = sub {
2604         my ($vmid, $resp) = @_;
2605         my $data = $resp->{'return'} || [];
2606         my $totalrdbytes = 0;
2607         my $totalwrbytes = 0;
2608
2609         for my $blockstat (@$data) {
2610             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2611             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2612
2613             $blockstat->{device} =~ s/drive-//;
2614             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2615         }
2616         $res->{$vmid}->{diskread} = $totalrdbytes;
2617         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2618     };
2619
2620     my $statuscb = sub {
2621         my ($vmid, $resp) = @_;
2622
2623         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2624         # this fails if ballon driver is not loaded, so this must be
2625         # the last commnand (following command are aborted if this fails).
2626         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2627
2628         my $status = 'unknown';
2629         if (!defined($status = $resp->{'return'}->{status})) {
2630             warn "unable to get VM status\n";
2631             return;
2632         }
2633
2634         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2635     };
2636
2637     foreach my $vmid (keys %$list) {
2638         next if $opt_vmid && ($vmid ne $opt_vmid);
2639         next if !$res->{$vmid}->{pid}; # not running
2640         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2641     }
2642
2643     $qmpclient->queue_execute(undef, 1);
2644
2645     foreach my $vmid (keys %$list) {
2646         next if $opt_vmid && ($vmid ne $opt_vmid);
2647         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2648     }
2649
2650     return $res;
2651 }
2652
2653 sub foreach_drive {
2654     my ($conf, $func, @param) = @_;
2655
2656     foreach my $ds (valid_drive_names()) {
2657         next if !defined($conf->{$ds});
2658
2659         my $drive = parse_drive($ds, $conf->{$ds});
2660         next if !$drive;
2661
2662         &$func($ds, $drive, @param);
2663     }
2664 }
2665
2666 sub foreach_volid {
2667     my ($conf, $func, @param) = @_;
2668
2669     my $volhash = {};
2670
2671     my $test_volid = sub {
2672         my ($volid, $is_cdrom) = @_;
2673
2674         return if !$volid;
2675
2676         $volhash->{$volid} = $is_cdrom || 0;
2677     };
2678
2679     foreach_drive($conf, sub {
2680         my ($ds, $drive) = @_;
2681         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2682     });
2683
2684     foreach my $snapname (keys %{$conf->{snapshots}}) {
2685         my $snap = $conf->{snapshots}->{$snapname};
2686         &$test_volid($snap->{vmstate}, 0);
2687         foreach_drive($snap, sub {
2688             my ($ds, $drive) = @_;
2689             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2690         });
2691     }
2692
2693     foreach my $volid (keys %$volhash) {
2694         &$func($volid, $volhash->{$volid}, @param);
2695     }
2696 }
2697
2698 sub vga_conf_has_spice {
2699     my ($vga) = @_;
2700
2701     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2702
2703     return $1 || 1;
2704 }
2705
2706 sub config_to_command {
2707     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2708
2709     my $cmd = [];
2710     my $globalFlags = [];
2711     my $machineFlags = [];
2712     my $rtcFlags = [];
2713     my $cpuFlags = [];
2714     my $devices = [];
2715     my $pciaddr = '';
2716     my $bridges = {};
2717     my $kvmver = kvm_user_version();
2718     my $vernum = 0; # unknown
2719     my $ostype = $conf->{ostype};
2720     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2721         $vernum = $1*1000000+$2*1000;
2722     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2723         $vernum = $1*1000000+$2*1000+$3;
2724     }
2725
2726     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2727
2728     my $have_ovz = -f '/proc/vz/vestat';
2729
2730     my $q35 = machine_type_is_q35($conf);
2731     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2732     my $machine_type = $forcemachine || $conf->{machine};
2733     my $use_old_bios_files = undef;
2734     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2735
2736     my $cpuunits = defined($conf->{cpuunits}) ?
2737             $conf->{cpuunits} : $defaults->{cpuunits};
2738
2739     push @$cmd, '/usr/bin/kvm';
2740
2741     push @$cmd, '-id', $vmid;
2742
2743     my $use_virtio = 0;
2744
2745     my $qmpsocket = qmp_socket($vmid);
2746     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2747     push @$cmd, '-mon', "chardev=qmp,mode=control";
2748
2749
2750     push @$cmd, '-pidfile' , pidfile_name($vmid);
2751
2752     push @$cmd, '-daemonize';
2753
2754     if ($conf->{smbios1}) {
2755         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2756     }
2757
2758     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2759         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2760         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2761         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2762         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2763         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2764         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2765     }
2766
2767     if ($q35) {
2768         # the q35 chipset support native usb2, so we enable usb controller
2769         # by default for this machine type
2770         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2771     } else {
2772         $pciaddr = print_pci_addr("piix3", $bridges);
2773         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2774
2775         my $use_usb2 = 0;
2776         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2777             next if !$conf->{"usb$i"};
2778             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2779             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2780             $use_usb2 = 1;
2781         }
2782         # include usb device config
2783         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2784     }
2785
2786     # add usb3 controller if needed
2787
2788     my $use_usb3 = 0;
2789     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2790         next if !$conf->{"usb$i"};
2791         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2792         next if !$d || !$d->{usb3};
2793         $use_usb3 = 1;
2794     }
2795
2796     $pciaddr = print_pci_addr("xhci", $bridges);
2797     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2798
2799     my $vga = $conf->{vga};
2800
2801     my $qxlnum = vga_conf_has_spice($vga);
2802     $vga = 'qxl' if $qxlnum;
2803
2804     if (!$vga) {
2805         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2806                                 $conf->{ostype} eq 'win7' ||
2807                                 $conf->{ostype} eq 'w2k8')) {
2808             $vga = 'std';
2809         } else {
2810             $vga = 'cirrus';
2811         }
2812     }
2813
2814     # enable absolute mouse coordinates (needed by vnc)
2815     my $tablet;
2816     if (defined($conf->{tablet})) {
2817         $tablet = $conf->{tablet};
2818     } else {
2819         $tablet = $defaults->{tablet};
2820         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2821         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2822     }
2823
2824     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2825
2826     my $kvm_off = 0;
2827     # host pci devices
2828     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2829         my $d = parse_hostpci($conf->{"hostpci$i"});
2830         next if !$d;
2831
2832         my $pcie = $d->{pcie};
2833         if($pcie){
2834             die "q35 machine model is not enabled" if !$q35;
2835             $pciaddr = print_pcie_addr("hostpci$i");
2836         }else{
2837             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2838         }
2839
2840         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2841         my $xvga = '';
2842         if ($d->{'x-vga'}) {
2843             $xvga = ',x-vga=on';
2844             $kvm_off = 1;
2845             $vga = 'none';
2846             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2847                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2848             }
2849             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2850                 $xvga = "";
2851             }
2852         }
2853         my $pcidevices = $d->{pciid};
2854         my $multifunction = 1 if @$pcidevices > 1;
2855
2856         my $j=0;
2857         foreach my $pcidevice (@$pcidevices) {
2858
2859             my $id = "hostpci$i";
2860             $id .= ".$j" if $multifunction;
2861             my $addr = $pciaddr;
2862             $addr .= ".$j" if $multifunction;
2863             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2864
2865             if($j == 0){
2866                 $devicestr .= "$rombar$xvga";
2867                 $devicestr .= ",multifunction=on" if $multifunction;
2868             }
2869
2870             push @$devices, '-device', $devicestr;
2871             $j++;
2872         }
2873     }
2874
2875     # usb devices
2876     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2877         next if !$conf->{"usb$i"};
2878         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2879         next if !$d;
2880
2881         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2882         my $usbbus = '';
2883         if (defined($d->{usb3}) && $d->{usb3}) {
2884             $usbbus = ',bus=xhci.0';
2885         }
2886
2887         if (defined($d->{host})) {
2888             $d = parse_usb_device($d->{host});
2889             if (defined($d->{vendorid}) && defined($d->{productid})) {
2890                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2891             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2892                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2893             } elsif (defined($d->{spice}) && $d->{spice}) {
2894                 # usb redir support for spice, currently no usb3
2895                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2896                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2897             }
2898         }
2899     }
2900
2901     # serial devices
2902     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2903         if (my $path = $conf->{"serial$i"}) {
2904             if ($path eq 'socket') {
2905                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2906                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2907                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2908             } else {
2909                 die "no such serial device\n" if ! -c $path;
2910                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2911                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2912             }
2913         }
2914     }
2915
2916     # parallel devices
2917     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2918         if (my $path = $conf->{"parallel$i"}) {
2919             die "no such parallel device\n" if ! -c $path;
2920             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2921             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2922             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2923         }
2924     }
2925
2926     my $vmname = $conf->{name} || "vm$vmid";
2927
2928     push @$cmd, '-name', $vmname;
2929
2930     my $sockets = 1;
2931     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2932     $sockets = $conf->{sockets} if  $conf->{sockets};
2933
2934     my $cores = $conf->{cores} || 1;
2935
2936     my $maxcpus = $sockets * $cores;
2937
2938     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2939
2940     my $allowed_vcpus = $cpuinfo->{cpus};
2941
2942     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2943         if ($allowed_vcpus < $maxcpus);
2944
2945     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2946
2947     push @$cmd, '-nodefaults';
2948
2949     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2950
2951     my $bootindex_hash = {};
2952     my $i = 1;
2953     foreach my $o (split(//, $bootorder)) {
2954         $bootindex_hash->{$o} = $i*100;
2955         $i++;
2956     }
2957
2958     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2959
2960     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2961
2962     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2963
2964     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2965
2966     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2967         my $socket = vnc_socket($vmid);
2968         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2969     } else {
2970         push @$cmd, '-nographic';
2971     }
2972
2973     # time drift fix
2974     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2975
2976     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2977     my $useLocaltime = $conf->{localtime};
2978
2979     if ($ostype) {
2980         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2981
2982         if ($ostype =~ m/^w/) { # windows
2983             $useLocaltime = 1 if !defined($conf->{localtime});
2984
2985             # use time drift fix when acpi is enabled
2986             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2987                 $tdf = 1 if !defined($conf->{tdf});
2988             }
2989         }
2990
2991         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2992             $ostype eq 'wvista') {
2993             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2994             push @$cmd, '-no-hpet';
2995             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2996                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2997                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2998                 push @$cpuFlags , 'hv_time' if !$nokvm;
2999
3000                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3001                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3002                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3003                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3004                 }
3005
3006             } else {
3007                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3008             }
3009         }
3010
3011         if ($ostype eq 'win7' || $ostype eq 'win8') {
3012             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3013         }
3014     }
3015
3016     push @$rtcFlags, 'driftfix=slew' if $tdf;
3017
3018     if ($nokvm) {
3019         push @$machineFlags, 'accel=tcg';
3020     } else {
3021         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3022     }
3023
3024     if ($machine_type) {
3025         push @$machineFlags, "type=${machine_type}";
3026     }
3027
3028     if ($conf->{startdate}) {
3029         push @$rtcFlags, "base=$conf->{startdate}";
3030     } elsif ($useLocaltime) {
3031         push @$rtcFlags, 'base=localtime';
3032     }
3033
3034     my $cpu = $nokvm ? "qemu64" : "kvm64";
3035     if (my $cputype = $conf->{cpu}) {
3036         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3037             or die "Cannot parse cpu description: $cputype\n";
3038         $cpu = $cpuconf->{cputype};
3039         $kvm_off = 1 if $cpuconf->{hidden};
3040     }
3041
3042     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3043
3044     push @$cpuFlags , '-x2apic'
3045         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3046
3047     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3048
3049     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3050
3051     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3052
3053         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3054         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3055     }
3056
3057     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3058
3059     push @$cpuFlags, 'kvm=off' if $kvm_off;
3060
3061     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3062         die "internal error"; # should not happen
3063
3064     push @$cpuFlags, "vendor=${cpu_vendor}"
3065         if $cpu_vendor ne 'default';
3066
3067     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3068
3069     push @$cmd, '-cpu', $cpu;
3070
3071     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3072
3073     push @$cmd, '-S' if $conf->{freeze};
3074
3075     # set keyboard layout
3076     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3077     push @$cmd, '-k', $kb if $kb;
3078
3079     # enable sound
3080     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3081     #push @$cmd, '-soundhw', 'es1370';
3082     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3083
3084     if($conf->{agent}) {
3085         my $qgasocket = qmp_socket($vmid, 1);
3086         my $pciaddr = print_pci_addr("qga0", $bridges);
3087         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3088         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3089         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3090     }
3091
3092     my $spice_port;
3093
3094     if ($qxlnum) {
3095         if ($qxlnum > 1) {
3096             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3097                 for(my $i = 1; $i < $qxlnum; $i++){
3098                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3099                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3100                 }
3101             } else {
3102                 # assume other OS works like Linux
3103                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3104                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3105             }
3106         }
3107
3108         my $pciaddr = print_pci_addr("spice", $bridges);
3109
3110         my $nodename = PVE::INotify::nodename();
3111         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3112         $spice_port = PVE::Tools::next_spice_port($pfamily);
3113
3114         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3115
3116         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3117         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3118         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3119     }
3120
3121     # enable balloon by default, unless explicitly disabled
3122     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3123         $pciaddr = print_pci_addr("balloon0", $bridges);
3124         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3125     }
3126
3127     if ($conf->{watchdog}) {
3128         my $wdopts = parse_watchdog($conf->{watchdog});
3129         $pciaddr = print_pci_addr("watchdog", $bridges);
3130         my $watchdog = $wdopts->{model} || 'i6300esb';
3131         push @$devices, '-device', "$watchdog$pciaddr";
3132         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3133     }
3134
3135     my $vollist = [];
3136     my $scsicontroller = {};
3137     my $ahcicontroller = {};
3138     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3139
3140     # Add iscsi initiator name if available
3141     if (my $initiator = get_initiator_name()) {
3142         push @$devices, '-iscsi', "initiator-name=$initiator";
3143     }
3144
3145     foreach_drive($conf, sub {
3146         my ($ds, $drive) = @_;
3147
3148         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3149             push @$vollist, $drive->{file};
3150         }
3151
3152         $use_virtio = 1 if $ds =~ m/^virtio/;
3153
3154         if (drive_is_cdrom ($drive)) {
3155             if ($bootindex_hash->{d}) {
3156                 $drive->{bootindex} = $bootindex_hash->{d};
3157                 $bootindex_hash->{d} += 1;
3158             }
3159         } else {
3160             if ($bootindex_hash->{c}) {
3161                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3162                 $bootindex_hash->{c} += 1;
3163             }
3164         }
3165
3166         if($drive->{interface} eq 'virtio'){
3167            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3168         }
3169
3170         if ($drive->{interface} eq 'scsi') {
3171
3172             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3173
3174             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3175             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3176
3177             my $iothread = '';
3178             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3179                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3180                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3181             } elsif ($drive->{iothread}) {
3182                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3183             }
3184
3185             my $queues = '';
3186             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3187                 $queues = ",num_queues=$drive->{queues}";
3188             }
3189
3190             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3191             $scsicontroller->{$controller}=1;
3192         }
3193
3194         if ($drive->{interface} eq 'sata') {
3195            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3196            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3197            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3198            $ahcicontroller->{$controller}=1;
3199         }
3200
3201         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3202         push @$devices, '-drive',$drive_cmd;
3203         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3204     });
3205
3206     for (my $i = 0; $i < $MAX_NETS; $i++) {
3207          next if !$conf->{"net$i"};
3208          my $d = parse_net($conf->{"net$i"});
3209          next if !$d;
3210
3211          $use_virtio = 1 if $d->{model} eq 'virtio';
3212
3213          if ($bootindex_hash->{n}) {
3214             $d->{bootindex} = $bootindex_hash->{n};
3215             $bootindex_hash->{n} += 1;
3216          }
3217
3218          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3219          push @$devices, '-netdev', $netdevfull;
3220
3221          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3222          push @$devices, '-device', $netdevicefull;
3223     }
3224
3225     if (!$q35) {
3226         # add pci bridges
3227         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3228            $bridges->{1} = 1;
3229            $bridges->{2} = 1;
3230         }
3231
3232         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3233
3234         while (my ($k, $v) = each %$bridges) {
3235             $pciaddr = print_pci_addr("pci.$k");
3236             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3237         }
3238     }
3239
3240     # add custom args
3241     if ($conf->{args}) {
3242         my $aa = PVE::Tools::split_args($conf->{args});
3243         push @$cmd, @$aa;
3244     }
3245
3246     push @$cmd, @$devices;
3247     push @$cmd, '-rtc', join(',', @$rtcFlags)
3248         if scalar(@$rtcFlags);
3249     push @$cmd, '-machine', join(',', @$machineFlags)
3250         if scalar(@$machineFlags);
3251     push @$cmd, '-global', join(',', @$globalFlags)
3252         if scalar(@$globalFlags);
3253
3254     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3255 }
3256
3257 sub vnc_socket {
3258     my ($vmid) = @_;
3259     return "${var_run_tmpdir}/$vmid.vnc";
3260 }
3261
3262 sub spice_port {
3263     my ($vmid) = @_;
3264
3265     my $res = vm_mon_cmd($vmid, 'query-spice');
3266
3267     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3268 }
3269
3270 sub qmp_socket {
3271     my ($vmid, $qga) = @_;
3272     my $sockettype = $qga ? 'qga' : 'qmp';
3273     return "${var_run_tmpdir}/$vmid.$sockettype";
3274 }
3275
3276 sub pidfile_name {
3277     my ($vmid) = @_;
3278     return "${var_run_tmpdir}/$vmid.pid";
3279 }
3280
3281 sub vm_devices_list {
3282     my ($vmid) = @_;
3283
3284     my $res = vm_mon_cmd($vmid, 'query-pci');
3285     my $devices = {};
3286     foreach my $pcibus (@$res) {
3287         foreach my $device (@{$pcibus->{devices}}) {
3288             next if !$device->{'qdev_id'};
3289             if ($device->{'pci_bridge'}) {
3290                 $devices->{$device->{'qdev_id'}} = 1;
3291                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3292                     next if !$bridge_device->{'qdev_id'};
3293                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3294                     $devices->{$device->{'qdev_id'}}++;
3295                 }
3296             } else {
3297                 $devices->{$device->{'qdev_id'}} = 1;
3298             }
3299         }
3300     }
3301
3302     my $resblock = vm_mon_cmd($vmid, 'query-block');
3303     foreach my $block (@$resblock) {
3304         if($block->{device} =~ m/^drive-(\S+)/){
3305                 $devices->{$1} = 1;
3306         }
3307     }
3308
3309     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3310     foreach my $mice (@$resmice) {
3311         if ($mice->{name} eq 'QEMU HID Tablet') {
3312             $devices->{tablet} = 1;
3313             last;
3314         }
3315     }
3316
3317     return $devices;
3318 }
3319
3320 sub vm_deviceplug {
3321     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3322
3323     my $q35 = machine_type_is_q35($conf);
3324
3325     my $devices_list = vm_devices_list($vmid);
3326     return 1 if defined($devices_list->{$deviceid});
3327
3328     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3329
3330     if ($deviceid eq 'tablet') {
3331
3332         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3333
3334     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3335
3336         qemu_iothread_add($vmid, $deviceid, $device);
3337
3338         qemu_driveadd($storecfg, $vmid, $device);
3339         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3340
3341         qemu_deviceadd($vmid, $devicefull);
3342         eval { qemu_deviceaddverify($vmid, $deviceid); };
3343         if (my $err = $@) {
3344             eval { qemu_drivedel($vmid, $deviceid); };
3345             warn $@ if $@;
3346             die $err;
3347         }
3348
3349     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3350
3351
3352         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3353         my $pciaddr = print_pci_addr($deviceid);
3354         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3355
3356         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3357
3358         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3359             qemu_iothread_add($vmid, $deviceid, $device);
3360             $devicefull .= ",iothread=iothread-$deviceid";
3361         }
3362
3363         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3364             $devicefull .= ",num_queues=$device->{queues}";
3365         }
3366
3367         qemu_deviceadd($vmid, $devicefull);
3368         qemu_deviceaddverify($vmid, $deviceid);
3369
3370     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3371
3372         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3373         qemu_driveadd($storecfg, $vmid, $device);
3374
3375         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3376         eval { qemu_deviceadd($vmid, $devicefull); };
3377         if (my $err = $@) {
3378             eval { qemu_drivedel($vmid, $deviceid); };
3379             warn $@ if $@;
3380             die $err;
3381         }
3382
3383     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3384
3385         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3386
3387         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3388         my $use_old_bios_files = undef;
3389         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3390
3391         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3392         qemu_deviceadd($vmid, $netdevicefull);
3393         eval { qemu_deviceaddverify($vmid, $deviceid); };
3394         if (my $err = $@) {
3395             eval { qemu_netdevdel($vmid, $deviceid); };
3396             warn $@ if $@;
3397             die $err;
3398         }
3399
3400     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3401
3402         my $bridgeid = $2;
3403         my $pciaddr = print_pci_addr($deviceid);
3404         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3405
3406         qemu_deviceadd($vmid, $devicefull);
3407         qemu_deviceaddverify($vmid, $deviceid);
3408
3409     } else {
3410         die "can't hotplug device '$deviceid'\n";
3411     }
3412
3413     return 1;
3414 }
3415
3416 # fixme: this should raise exceptions on error!
3417 sub vm_deviceunplug {
3418     my ($vmid, $conf, $deviceid) = @_;
3419
3420     my $devices_list = vm_devices_list($vmid);
3421     return 1 if !defined($devices_list->{$deviceid});
3422
3423     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3424
3425     if ($deviceid eq 'tablet') {
3426
3427         qemu_devicedel($vmid, $deviceid);
3428
3429     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3430
3431         qemu_devicedel($vmid, $deviceid);
3432         qemu_devicedelverify($vmid, $deviceid);
3433         qemu_drivedel($vmid, $deviceid);
3434         qemu_iothread_del($conf, $vmid, $deviceid);
3435
3436     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3437
3438         qemu_devicedel($vmid, $deviceid);
3439         qemu_devicedelverify($vmid, $deviceid);
3440         qemu_iothread_del($conf, $vmid, $deviceid);
3441
3442     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3443
3444         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3445         my $device = parse_drive($deviceid, $conf->{$deviceid});
3446         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3447
3448         qemu_devicedel($vmid, $deviceid);
3449         qemu_drivedel($vmid, $deviceid);
3450         qemu_deletescsihw($conf, $vmid, $deviceid);
3451
3452     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3453
3454         qemu_devicedel($vmid, $deviceid);
3455         qemu_devicedelverify($vmid, $deviceid);
3456         qemu_netdevdel($vmid, $deviceid);
3457
3458     } else {
3459         die "can't unplug device '$deviceid'\n";
3460     }
3461
3462     return 1;
3463 }
3464
3465 sub qemu_deviceadd {
3466     my ($vmid, $devicefull) = @_;
3467
3468     $devicefull = "driver=".$devicefull;
3469     my %options =  split(/[=,]/, $devicefull);
3470
3471     vm_mon_cmd($vmid, "device_add" , %options);
3472 }
3473
3474 sub qemu_devicedel {
3475     my ($vmid, $deviceid) = @_;
3476
3477     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3478 }
3479
3480 sub qemu_iothread_add {
3481     my($vmid, $deviceid, $device) = @_;
3482
3483     if ($device->{iothread}) {
3484         my $iothreads = vm_iothreads_list($vmid);
3485         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3486     }
3487 }
3488
3489 sub qemu_iothread_del {
3490     my($conf, $vmid, $deviceid) = @_;
3491
3492     my $device = parse_drive($deviceid, $conf->{$deviceid});
3493     if ($device->{iothread}) {
3494         my $iothreads = vm_iothreads_list($vmid);
3495         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3496     }
3497 }
3498
3499 sub qemu_objectadd {
3500     my($vmid, $objectid, $qomtype) = @_;
3501
3502     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3503
3504     return 1;
3505 }
3506
3507 sub qemu_objectdel {
3508     my($vmid, $objectid) = @_;
3509
3510     vm_mon_cmd($vmid, "object-del", id => $objectid);
3511
3512     return 1;
3513 }
3514
3515 sub qemu_driveadd {
3516     my ($storecfg, $vmid, $device) = @_;
3517
3518     my $drive = print_drive_full($storecfg, $vmid, $device);
3519     $drive =~ s/\\/\\\\/g;
3520     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3521
3522     # If the command succeeds qemu prints: "OK"
3523     return 1 if $ret =~ m/OK/s;
3524
3525     die "adding drive failed: $ret\n";
3526 }
3527
3528 sub qemu_drivedel {
3529     my($vmid, $deviceid) = @_;
3530
3531     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3532     $ret =~ s/^\s+//;
3533
3534     return 1 if $ret eq "";
3535
3536     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3537     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3538
3539     die "deleting drive $deviceid failed : $ret\n";
3540 }
3541
3542 sub qemu_deviceaddverify {
3543     my ($vmid, $deviceid) = @_;
3544
3545     for (my $i = 0; $i <= 5; $i++) {
3546          my $devices_list = vm_devices_list($vmid);
3547          return 1 if defined($devices_list->{$deviceid});
3548          sleep 1;
3549     }
3550
3551     die "error on hotplug device '$deviceid'\n";
3552 }
3553
3554
3555 sub qemu_devicedelverify {
3556     my ($vmid, $deviceid) = @_;
3557
3558     # need to verify that the device is correctly removed as device_del
3559     # is async and empty return is not reliable
3560
3561     for (my $i = 0; $i <= 5; $i++) {
3562          my $devices_list = vm_devices_list($vmid);
3563          return 1 if !defined($devices_list->{$deviceid});
3564          sleep 1;
3565     }
3566
3567     die "error on hot-unplugging device '$deviceid'\n";
3568 }
3569
3570 sub qemu_findorcreatescsihw {
3571     my ($storecfg, $conf, $vmid, $device) = @_;
3572
3573     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3574
3575     my $scsihwid="$controller_prefix$controller";
3576     my $devices_list = vm_devices_list($vmid);
3577
3578     if(!defined($devices_list->{$scsihwid})) {
3579         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3580     }
3581
3582     return 1;
3583 }
3584
3585 sub qemu_deletescsihw {
3586     my ($conf, $vmid, $opt) = @_;
3587
3588     my $device = parse_drive($opt, $conf->{$opt});
3589
3590     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3591         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3592         return 1;
3593     }
3594
3595     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3596
3597     my $devices_list = vm_devices_list($vmid);
3598     foreach my $opt (keys %{$devices_list}) {
3599         if (PVE::QemuServer::is_valid_drivename($opt)) {
3600             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3601             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3602                 return 1;
3603             }
3604         }
3605     }
3606
3607     my $scsihwid="scsihw$controller";
3608
3609     vm_deviceunplug($vmid, $conf, $scsihwid);
3610
3611     return 1;
3612 }
3613
3614 sub qemu_add_pci_bridge {
3615     my ($storecfg, $conf, $vmid, $device) = @_;
3616
3617     my $bridges = {};
3618
3619     my $bridgeid;
3620
3621     print_pci_addr($device, $bridges);
3622
3623     while (my ($k, $v) = each %$bridges) {
3624         $bridgeid = $k;
3625     }
3626     return 1 if !defined($bridgeid) || $bridgeid < 1;
3627
3628     my $bridge = "pci.$bridgeid";
3629     my $devices_list = vm_devices_list($vmid);
3630
3631     if (!defined($devices_list->{$bridge})) {
3632         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3633     }
3634
3635     return 1;
3636 }
3637
3638 sub qemu_set_link_status {
3639     my ($vmid, $device, $up) = @_;
3640
3641     vm_mon_cmd($vmid, "set_link", name => $device,
3642                up => $up ? JSON::true : JSON::false);
3643 }
3644
3645 sub qemu_netdevadd {
3646     my ($vmid, $conf, $device, $deviceid) = @_;
3647
3648     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3649     my %options =  split(/[=,]/, $netdev);
3650
3651     vm_mon_cmd($vmid, "netdev_add",  %options);
3652     return 1;
3653 }
3654
3655 sub qemu_netdevdel {
3656     my ($vmid, $deviceid) = @_;
3657
3658     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3659 }
3660
3661 sub qemu_cpu_hotplug {
3662     my ($vmid, $conf, $vcpus) = @_;
3663
3664     my $sockets = 1;
3665     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3666     $sockets = $conf->{sockets} if  $conf->{sockets};
3667     my $cores = $conf->{cores} || 1;
3668     my $maxcpus = $sockets * $cores;
3669
3670     $vcpus = $maxcpus if !$vcpus;
3671
3672     die "you can't add more vcpus than maxcpus\n"
3673         if $vcpus > $maxcpus;
3674
3675     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3676     die "online cpu unplug is not yet possible\n"
3677         if $vcpus < $currentvcpus;
3678
3679     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3680     die "vcpus in running vm is different than configuration\n"
3681         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3682
3683     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3684         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3685     }
3686 }
3687
3688 sub qemu_block_set_io_throttle {
3689     my ($vmid, $deviceid,
3690         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3691         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3692
3693     return if !check_running($vmid) ;
3694
3695     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3696         bps => int($bps),
3697         bps_rd => int($bps_rd),
3698         bps_wr => int($bps_wr),
3699         iops => int($iops),
3700         iops_rd => int($iops_rd),
3701         iops_wr => int($iops_wr),
3702         bps_max => int($bps_max),
3703         bps_rd_max => int($bps_rd_max),
3704         bps_wr_max => int($bps_wr_max),
3705         iops_max => int($iops_max),
3706         iops_rd_max => int($iops_rd_max),
3707         iops_wr_max => int($iops_wr_max)
3708     );
3709
3710 }
3711
3712 # old code, only used to shutdown old VM after update
3713 sub __read_avail {
3714     my ($fh, $timeout) = @_;
3715
3716     my $sel = new IO::Select;
3717     $sel->add($fh);
3718
3719     my $res = '';
3720     my $buf;
3721
3722     my @ready;
3723     while (scalar (@ready = $sel->can_read($timeout))) {
3724         my $count;
3725         if ($count = $fh->sysread($buf, 8192)) {
3726             if ($buf =~ /^(.*)\(qemu\) $/s) {
3727                 $res .= $1;
3728                 last;
3729             } else {
3730                 $res .= $buf;
3731             }
3732         } else {
3733             if (!defined($count)) {
3734                 die "$!\n";
3735             }
3736             last;
3737         }
3738     }
3739
3740     die "monitor read timeout\n" if !scalar(@ready);
3741
3742     return $res;
3743 }
3744
3745 # old code, only used to shutdown old VM after update
3746 sub vm_monitor_command {
3747     my ($vmid, $cmdstr, $nocheck) = @_;
3748
3749     my $res;
3750
3751     eval {
3752         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3753
3754         my $sname = "${var_run_tmpdir}/$vmid.mon";
3755
3756         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3757             die "unable to connect to VM $vmid socket - $!\n";
3758
3759         my $timeout = 3;
3760
3761         # hack: migrate sometime blocks the monitor (when migrate_downtime
3762         # is set)
3763         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3764             $timeout = 60*60; # 1 hour
3765         }
3766
3767         # read banner;
3768         my $data = __read_avail($sock, $timeout);
3769
3770         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3771             die "got unexpected qemu monitor banner\n";
3772         }
3773
3774         my $sel = new IO::Select;
3775         $sel->add($sock);
3776
3777         if (!scalar(my @ready = $sel->can_write($timeout))) {
3778             die "monitor write error - timeout";
3779         }
3780
3781         my $fullcmd = "$cmdstr\r";
3782
3783         # syslog('info', "VM $vmid monitor command: $cmdstr");
3784
3785         my $b;
3786         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3787             die "monitor write error - $!";
3788         }
3789
3790         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3791
3792         $timeout = 20;
3793
3794         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3795             $timeout = 60*60; # 1 hour
3796         } elsif ($cmdstr =~ m/^(eject|change)/) {
3797             $timeout = 60; # note: cdrom mount command is slow
3798         }
3799         if ($res = __read_avail($sock, $timeout)) {
3800
3801             my @lines = split("\r?\n", $res);
3802
3803             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3804
3805             $res = join("\n", @lines);
3806             $res .= "\n";
3807         }
3808     };
3809
3810     my $err = $@;
3811
3812     if ($err) {
3813         syslog("err", "VM $vmid monitor command failed - $err");
3814         die $err;
3815     }
3816
3817     return $res;
3818 }
3819
3820 sub qemu_block_resize {
3821     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3822
3823     my $running = check_running($vmid);
3824
3825     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3826
3827     return if !$running;
3828
3829     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3830
3831 }
3832
3833 sub qemu_volume_snapshot {
3834     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3835
3836     my $running = check_running($vmid);
3837
3838     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3839         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3840     } else {
3841         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3842     }
3843 }
3844
3845 sub qemu_volume_snapshot_delete {
3846     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3847
3848     my $running = check_running($vmid);
3849
3850     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3851
3852     return if !$running;
3853
3854     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3855 }
3856
3857 sub set_migration_caps {
3858     my ($vmid) = @_;
3859
3860     my $cap_ref = [];
3861
3862     my $enabled_cap = {
3863         "auto-converge" => 1,
3864         "xbzrle" => 1,
3865         "x-rdma-pin-all" => 0,
3866         "zero-blocks" => 0,
3867         "compress" => 0
3868     };
3869
3870     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3871
3872     for my $supported_capability (@$supported_capabilities) {
3873         push @$cap_ref, {
3874             capability => $supported_capability->{capability},
3875             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3876         };
3877     }
3878
3879     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3880 }
3881
3882 my $fast_plug_option = {
3883     'lock' => 1,
3884     'name' => 1,
3885     'onboot' => 1,
3886     'shares' => 1,
3887     'startup' => 1,
3888     'description' => 1,
3889 };
3890
3891 # hotplug changes in [PENDING]
3892 # $selection hash can be used to only apply specified options, for
3893 # example: { cores => 1 } (only apply changed 'cores')
3894 # $errors ref is used to return error messages
3895 sub vmconfig_hotplug_pending {
3896     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3897
3898     my $defaults = load_defaults();
3899
3900     # commit values which do not have any impact on running VM first
3901     # Note: those option cannot raise errors, we we do not care about
3902     # $selection and always apply them.
3903
3904     my $add_error = sub {
3905         my ($opt, $msg) = @_;
3906         $errors->{$opt} = "hotplug problem - $msg";
3907     };
3908
3909     my $changes = 0;
3910     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3911         if ($fast_plug_option->{$opt}) {
3912             $conf->{$opt} = $conf->{pending}->{$opt};
3913             delete $conf->{pending}->{$opt};
3914             $changes = 1;
3915         }
3916     }
3917
3918     if ($changes) {
3919         PVE::QemuConfig->write_config($vmid, $conf);
3920         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3921     }
3922
3923     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3924
3925     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3926     while (my ($opt, $force) = each %$pending_delete_hash) {
3927         next if $selection && !$selection->{$opt};
3928         eval {
3929             if ($opt eq 'hotplug') {
3930                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3931             } elsif ($opt eq 'tablet') {
3932                 die "skip\n" if !$hotplug_features->{usb};
3933                 if ($defaults->{tablet}) {
3934                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3935                 } else {
3936                     vm_deviceunplug($vmid, $conf, $opt);
3937                 }
3938             } elsif ($opt eq 'vcpus') {
3939                 die "skip\n" if !$hotplug_features->{cpu};
3940                 qemu_cpu_hotplug($vmid, $conf, undef);
3941             } elsif ($opt eq 'balloon') {
3942                 # enable balloon device is not hotpluggable
3943                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3944             } elsif ($fast_plug_option->{$opt}) {
3945                 # do nothing
3946             } elsif ($opt =~ m/^net(\d+)$/) {
3947                 die "skip\n" if !$hotplug_features->{network};
3948                 vm_deviceunplug($vmid, $conf, $opt);
3949             } elsif (is_valid_drivename($opt)) {
3950                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3951                 vm_deviceunplug($vmid, $conf, $opt);
3952                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3953             } elsif ($opt =~ m/^memory$/) {
3954                 die "skip\n" if !$hotplug_features->{memory};
3955                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3956             } elsif ($opt eq 'cpuunits') {
3957                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3958             } elsif ($opt eq 'cpulimit') {
3959                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3960             } else {
3961                 die "skip\n";
3962             }
3963         };
3964         if (my $err = $@) {
3965             &$add_error($opt, $err) if $err ne "skip\n";
3966         } else {
3967             # save new config if hotplug was successful
3968             delete $conf->{$opt};
3969             vmconfig_undelete_pending_option($conf, $opt);
3970             PVE::QemuConfig->write_config($vmid, $conf);
3971             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3972         }
3973     }
3974
3975     foreach my $opt (keys %{$conf->{pending}}) {
3976         next if $selection && !$selection->{$opt};
3977         my $value = $conf->{pending}->{$opt};
3978         eval {
3979             if ($opt eq 'hotplug') {
3980                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3981             } elsif ($opt eq 'tablet') {
3982                 die "skip\n" if !$hotplug_features->{usb};
3983                 if ($value == 1) {
3984                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3985                 } elsif ($value == 0) {
3986                     vm_deviceunplug($vmid, $conf, $opt);
3987                 }
3988             } elsif ($opt eq 'vcpus') {
3989                 die "skip\n" if !$hotplug_features->{cpu};
3990                 qemu_cpu_hotplug($vmid, $conf, $value);
3991             } elsif ($opt eq 'balloon') {
3992                 # enable/disable balloning device is not hotpluggable
3993                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3994                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3995                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3996
3997                 # allow manual ballooning if shares is set to zero
3998                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3999                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4000                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4001                 }
4002             } elsif ($opt =~ m/^net(\d+)$/) {
4003                 # some changes can be done without hotplug
4004                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4005                                     $vmid, $opt, $value);
4006             } elsif (is_valid_drivename($opt)) {
4007                 # some changes can be done without hotplug
4008                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4009                                      $vmid, $opt, $value, 1);
4010             } elsif ($opt =~ m/^memory$/) { #dimms
4011                 die "skip\n" if !$hotplug_features->{memory};
4012                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4013             } elsif ($opt eq 'cpuunits') {
4014                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4015             } elsif ($opt eq 'cpulimit') {
4016                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4017                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4018             } else {
4019                 die "skip\n";  # skip non-hot-pluggable options
4020             }
4021         };
4022         if (my $err = $@) {
4023             &$add_error($opt, $err) if $err ne "skip\n";
4024         } else {
4025             # save new config if hotplug was successful
4026             $conf->{$opt} = $value;
4027             delete $conf->{pending}->{$opt};
4028             PVE::QemuConfig->write_config($vmid, $conf);
4029             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4030         }
4031     }
4032 }
4033
4034 sub try_deallocate_drive {
4035     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4036
4037     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4038         my $volid = $drive->{file};
4039         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4040             my $sid = PVE::Storage::parse_volume_id($volid);
4041             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4042
4043             # check if the disk is really unused
4044             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4045                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4046             PVE::Storage::vdisk_free($storecfg, $volid);
4047             return 1;
4048         } else {
4049             # If vm is not owner of this disk remove from config
4050             return 1;
4051         }
4052     }
4053
4054     return undef;
4055 }
4056
4057 sub vmconfig_delete_or_detach_drive {
4058     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4059
4060     my $drive = parse_drive($opt, $conf->{$opt});
4061
4062     my $rpcenv = PVE::RPCEnvironment::get();
4063     my $authuser = $rpcenv->get_user();
4064
4065     if ($force) {
4066         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4067         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4068     } else {
4069         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4070     }
4071 }
4072
4073 sub vmconfig_apply_pending {
4074     my ($vmid, $conf, $storecfg) = @_;
4075
4076     # cold plug
4077
4078     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4079     while (my ($opt, $force) = each %$pending_delete_hash) {
4080         die "internal error" if $opt =~ m/^unused/;
4081         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4082         if (!defined($conf->{$opt})) {
4083             vmconfig_undelete_pending_option($conf, $opt);
4084             PVE::QemuConfig->write_config($vmid, $conf);
4085         } elsif (is_valid_drivename($opt)) {
4086             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4087             vmconfig_undelete_pending_option($conf, $opt);
4088             delete $conf->{$opt};
4089             PVE::QemuConfig->write_config($vmid, $conf);
4090         } else {
4091             vmconfig_undelete_pending_option($conf, $opt);
4092             delete $conf->{$opt};
4093             PVE::QemuConfig->write_config($vmid, $conf);
4094         }
4095     }
4096
4097     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4098
4099     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4100         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4101
4102         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4103             # skip if nothing changed
4104         } elsif (is_valid_drivename($opt)) {
4105             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4106                 if defined($conf->{$opt});
4107             $conf->{$opt} = $conf->{pending}->{$opt};
4108         } else {
4109             $conf->{$opt} = $conf->{pending}->{$opt};
4110         }
4111
4112         delete $conf->{pending}->{$opt};
4113         PVE::QemuConfig->write_config($vmid, $conf);
4114     }
4115 }
4116
4117 my $safe_num_ne = sub {
4118     my ($a, $b) = @_;
4119
4120     return 0 if !defined($a) && !defined($b);
4121     return 1 if !defined($a);
4122     return 1 if !defined($b);
4123
4124     return $a != $b;
4125 };
4126
4127 my $safe_string_ne = sub {
4128     my ($a, $b) = @_;
4129
4130     return 0 if !defined($a) && !defined($b);
4131     return 1 if !defined($a);
4132     return 1 if !defined($b);
4133
4134     return $a ne $b;
4135 };
4136
4137 sub vmconfig_update_net {
4138     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4139
4140     my $newnet = parse_net($value);
4141
4142     if ($conf->{$opt}) {
4143         my $oldnet = parse_net($conf->{$opt});
4144
4145         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4146             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4147             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4148             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4149
4150             # for non online change, we try to hot-unplug
4151             die "skip\n" if !$hotplug;
4152             vm_deviceunplug($vmid, $conf, $opt);
4153         } else {
4154
4155             die "internal error" if $opt !~ m/net(\d+)/;
4156             my $iface = "tap${vmid}i$1";
4157
4158             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4159                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4160                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4161                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4162                 PVE::Network::tap_unplug($iface);
4163                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4164             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4165                 # Rate can be applied on its own but any change above needs to
4166                 # include the rate in tap_plug since OVS resets everything.
4167                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4168             }
4169
4170             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4171                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4172             }
4173
4174             return 1;
4175         }
4176     }
4177
4178     if ($hotplug) {
4179         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4180     } else {
4181         die "skip\n";
4182     }
4183 }
4184
4185 sub vmconfig_update_disk {
4186     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4187
4188     # fixme: do we need force?
4189
4190     my $drive = parse_drive($opt, $value);
4191
4192     if ($conf->{$opt}) {
4193
4194         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4195
4196             my $media = $drive->{media} || 'disk';
4197             my $oldmedia = $old_drive->{media} || 'disk';
4198             die "unable to change media type\n" if $media ne $oldmedia;
4199
4200             if (!drive_is_cdrom($old_drive)) {
4201
4202                 if ($drive->{file} ne $old_drive->{file}) {
4203
4204                     die "skip\n" if !$hotplug;
4205
4206                     # unplug and register as unused
4207                     vm_deviceunplug($vmid, $conf, $opt);
4208                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4209
4210                 } else {
4211                     # update existing disk
4212
4213                     # skip non hotpluggable value
4214                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4215                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4216                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4217                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4218                         die "skip\n";
4219                     }
4220
4221                     # apply throttle
4222                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4223                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4224                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4225                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4226                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4227                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4228                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4229                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4230                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4231                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4232                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4233                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4234
4235                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4236                                                    ($drive->{mbps} || 0)*1024*1024,
4237                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4238                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4239                                                    $drive->{iops} || 0,
4240                                                    $drive->{iops_rd} || 0,
4241                                                    $drive->{iops_wr} || 0,
4242                                                    ($drive->{mbps_max} || 0)*1024*1024,
4243                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4244                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4245                                                    $drive->{iops_max} || 0,
4246                                                    $drive->{iops_rd_max} || 0,
4247                                                    $drive->{iops_wr_max} || 0);
4248
4249                     }
4250
4251                     return 1;
4252                 }
4253
4254             } else { # cdrom
4255
4256                 if ($drive->{file} eq 'none') {
4257                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4258                 } else {
4259                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4260                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4261                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4262                 }
4263
4264                 return 1;
4265             }
4266         }
4267     }
4268
4269     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4270     # hotplug new disks
4271     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4272     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4273 }
4274
4275 sub vm_start {
4276     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4277         $forcemachine, $spice_ticket) = @_;
4278
4279     PVE::QemuConfig->lock_config($vmid, sub {
4280         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4281
4282         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4283
4284         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4285
4286         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4287
4288         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4289             vmconfig_apply_pending($vmid, $conf, $storecfg);
4290             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4291         }
4292
4293         my $defaults = load_defaults();
4294
4295         # set environment variable useful inside network script
4296         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4297
4298         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4299
4300         my $migrate_port = 0;
4301         my $migrate_uri;
4302         if ($statefile) {
4303             if ($statefile eq 'tcp') {
4304                 my $localip = "localhost";
4305                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4306                 my $nodename = PVE::INotify::nodename();
4307                 if ($datacenterconf->{migration_unsecure}) {
4308                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4309                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4310                 }
4311                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4312                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4313                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4314                 push @$cmd, '-incoming', $migrate_uri;
4315                 push @$cmd, '-S';
4316
4317             } elsif ($statefile eq 'unix') {
4318                 # should be default for secure migrations as a ssh TCP forward
4319                 # tunnel is not deterministic reliable ready and fails regurarly
4320                 # to set up in time, so use UNIX socket forwards
4321                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4322                 unlink $socket_addr;
4323
4324                 $migrate_uri = "unix:$socket_addr";
4325
4326                 push @$cmd, '-incoming', $migrate_uri;
4327                 push @$cmd, '-S';
4328
4329             } else {
4330                 push @$cmd, '-loadstate', $statefile;
4331             }
4332         } elsif ($paused) {
4333             push @$cmd, '-S';
4334         }
4335
4336         # host pci devices
4337         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4338           my $d = parse_hostpci($conf->{"hostpci$i"});
4339           next if !$d;
4340           my $pcidevices = $d->{pciid};
4341           foreach my $pcidevice (@$pcidevices) {
4342                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4343
4344                 my $info = pci_device_info("0000:$pciid");
4345                 die "IOMMU not present\n" if !check_iommu_support();
4346                 die "no pci device info for device '$pciid'\n" if !$info;
4347                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4348                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4349           }
4350         }
4351
4352         PVE::Storage::activate_volumes($storecfg, $vollist);
4353
4354         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4355             my $cmd = [];
4356             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4357             eval  { run_command($cmd); };
4358         }
4359
4360         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4361                                                   : $defaults->{cpuunits};
4362
4363         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4364
4365         my %properties = (
4366             Slice => 'qemu.slice',
4367             KillMode => 'none',
4368             CPUShares => $cpuunits
4369         );
4370
4371         if (my $cpulimit = $conf->{cpulimit}) {
4372             $properties{CPUQuota} = int($cpulimit * 100);
4373         }
4374         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4375
4376         if ($conf->{hugepages}) {
4377
4378             my $code = sub {
4379                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4380                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4381
4382                 PVE::QemuServer::Memory::hugepages_mount();
4383                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4384
4385                 eval  {
4386                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4387                     run_command($cmd, %run_params);
4388                 };
4389
4390                 if (my $err = $@) {
4391                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4392                     die $err;
4393                 }
4394
4395                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4396             };
4397             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4398
4399         } else {
4400             eval  {
4401                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4402                 run_command($cmd, %run_params);
4403             };
4404         }
4405
4406         if (my $err = $@) {
4407             # deactivate volumes if start fails
4408             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4409             die "start failed: $err";
4410         }
4411
4412         print "migration listens on $migrate_uri\n" if $migrate_uri;
4413
4414         if ($statefile && $statefile ne 'tcp')  {
4415             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4416             warn $@ if $@;
4417         }
4418
4419         if ($migratedfrom) {
4420
4421             eval {
4422                 set_migration_caps($vmid);
4423             };
4424             warn $@ if $@;
4425
4426             if ($spice_port) {
4427                 print "spice listens on port $spice_port\n";
4428                 if ($spice_ticket) {
4429                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4430                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4431                 }
4432             }
4433
4434         } else {
4435
4436             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4437                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4438                     if $conf->{balloon};
4439             }
4440
4441             foreach my $opt (keys %$conf) {
4442                 next if $opt !~  m/^net\d+$/;
4443                 my $nicconf = parse_net($conf->{$opt});
4444                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4445             }
4446         }
4447
4448         vm_mon_cmd_nocheck($vmid, 'qom-set',
4449                     path => "machine/peripheral/balloon0",
4450                     property => "guest-stats-polling-interval",
4451                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4452
4453     });
4454 }
4455
4456 sub vm_mon_cmd {
4457     my ($vmid, $execute, %params) = @_;
4458
4459     my $cmd = { execute => $execute, arguments => \%params };
4460     vm_qmp_command($vmid, $cmd);
4461 }
4462
4463 sub vm_mon_cmd_nocheck {
4464     my ($vmid, $execute, %params) = @_;
4465
4466     my $cmd = { execute => $execute, arguments => \%params };
4467     vm_qmp_command($vmid, $cmd, 1);
4468 }
4469
4470 sub vm_qmp_command {
4471     my ($vmid, $cmd, $nocheck) = @_;
4472
4473     my $res;
4474
4475     my $timeout;
4476     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4477         $timeout = $cmd->{arguments}->{timeout};
4478         delete $cmd->{arguments}->{timeout};
4479     }
4480
4481     eval {
4482         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4483         my $sname = qmp_socket($vmid);
4484         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4485             my $qmpclient = PVE::QMPClient->new();
4486
4487             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4488         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4489             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4490                 if scalar(%{$cmd->{arguments}});
4491             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4492         } else {
4493             die "unable to open monitor socket\n";
4494         }
4495     };
4496     if (my $err = $@) {
4497         syslog("err", "VM $vmid qmp command failed - $err");
4498         die $err;
4499     }
4500
4501     return $res;
4502 }
4503
4504 sub vm_human_monitor_command {
4505     my ($vmid, $cmdline) = @_;
4506
4507     my $res;
4508
4509     my $cmd = {
4510         execute => 'human-monitor-command',
4511         arguments => { 'command-line' => $cmdline},
4512     };
4513
4514     return vm_qmp_command($vmid, $cmd);
4515 }
4516
4517 sub vm_commandline {
4518     my ($storecfg, $vmid) = @_;
4519
4520     my $conf = PVE::QemuConfig->load_config($vmid);
4521
4522     my $defaults = load_defaults();
4523
4524     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4525
4526     return join(' ', @$cmd);
4527 }
4528
4529 sub vm_reset {
4530     my ($vmid, $skiplock) = @_;
4531
4532     PVE::QemuConfig->lock_config($vmid, sub {
4533
4534         my $conf = PVE::QemuConfig->load_config($vmid);
4535
4536         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4537
4538         vm_mon_cmd($vmid, "system_reset");
4539     });
4540 }
4541
4542 sub get_vm_volumes {
4543     my ($conf) = @_;
4544
4545     my $vollist = [];
4546     foreach_volid($conf, sub {
4547         my ($volid, $is_cdrom) = @_;
4548
4549         return if $volid =~ m|^/|;
4550
4551         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4552         return if !$sid;
4553
4554         push @$vollist, $volid;
4555     });
4556
4557     return $vollist;
4558 }
4559
4560 sub vm_stop_cleanup {
4561     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4562
4563     eval {
4564
4565         if (!$keepActive) {
4566             my $vollist = get_vm_volumes($conf);
4567             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4568         }
4569
4570         foreach my $ext (qw(mon qmp pid vnc qga)) {
4571             unlink "/var/run/qemu-server/${vmid}.$ext";
4572         }
4573
4574         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4575     };
4576     warn $@ if $@; # avoid errors - just warn
4577 }
4578
4579 # Note: use $nockeck to skip tests if VM configuration file exists.
4580 # We need that when migration VMs to other nodes (files already moved)
4581 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4582 sub vm_stop {
4583     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4584
4585     $force = 1 if !defined($force) && !$shutdown;
4586
4587     if ($migratedfrom){
4588         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4589         kill 15, $pid if $pid;
4590         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4591         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4592         return;
4593     }
4594
4595     PVE::QemuConfig->lock_config($vmid, sub {
4596
4597         my $pid = check_running($vmid, $nocheck);
4598         return if !$pid;
4599
4600         my $conf;
4601         if (!$nocheck) {
4602             $conf = PVE::QemuConfig->load_config($vmid);
4603             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4604             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4605                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4606                 $timeout = $opts->{down} if $opts->{down};
4607             }
4608         }
4609
4610         $timeout = 60 if !defined($timeout);
4611
4612         eval {
4613             if ($shutdown) {
4614                 if (defined($conf) && $conf->{agent}) {
4615                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4616                 } else {
4617                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4618                 }
4619             } else {
4620                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4621             }
4622         };
4623         my $err = $@;
4624
4625         if (!$err) {
4626             my $count = 0;
4627             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4628                 $count++;
4629                 sleep 1;
4630             }
4631
4632             if ($count >= $timeout) {
4633                 if ($force) {
4634                     warn "VM still running - terminating now with SIGTERM\n";
4635                     kill 15, $pid;
4636                 } else {
4637                     die "VM quit/powerdown failed - got timeout\n";
4638                 }
4639             } else {
4640                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4641                 return;
4642             }
4643         } else {
4644             if ($force) {
4645                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4646                 kill 15, $pid;
4647             } else {
4648                 die "VM quit/powerdown failed\n";
4649             }
4650         }
4651
4652         # wait again
4653         $timeout = 10;
4654
4655         my $count = 0;
4656         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4657             $count++;
4658             sleep 1;
4659         }
4660
4661         if ($count >= $timeout) {
4662             warn "VM still running - terminating now with SIGKILL\n";
4663             kill 9, $pid;
4664             sleep 1;
4665         }
4666
4667         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4668    });
4669 }
4670
4671 sub vm_suspend {
4672     my ($vmid, $skiplock) = @_;
4673
4674     PVE::QemuConfig->lock_config($vmid, sub {
4675
4676         my $conf = PVE::QemuConfig->load_config($vmid);
4677
4678         PVE::QemuConfig->check_lock($conf)
4679             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4680
4681         vm_mon_cmd($vmid, "stop");
4682     });
4683 }
4684
4685 sub vm_resume {
4686     my ($vmid, $skiplock, $nocheck) = @_;
4687
4688     PVE::QemuConfig->lock_config($vmid, sub {
4689
4690         if (!$nocheck) {
4691
4692             my $conf = PVE::QemuConfig->load_config($vmid);
4693
4694             PVE::QemuConfig->check_lock($conf)
4695                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4696
4697             vm_mon_cmd($vmid, "cont");
4698
4699         } else {
4700             vm_mon_cmd_nocheck($vmid, "cont");
4701         }
4702     });
4703 }
4704
4705 sub vm_sendkey {
4706     my ($vmid, $skiplock, $key) = @_;
4707
4708     PVE::QemuConfig->lock_config($vmid, sub {
4709
4710         my $conf = PVE::QemuConfig->load_config($vmid);
4711
4712         # there is no qmp command, so we use the human monitor command
4713         vm_human_monitor_command($vmid, "sendkey $key");
4714     });
4715 }
4716
4717 sub vm_destroy {
4718     my ($storecfg, $vmid, $skiplock) = @_;
4719
4720     PVE::QemuConfig->lock_config($vmid, sub {
4721
4722         my $conf = PVE::QemuConfig->load_config($vmid);
4723
4724         if (!check_running($vmid)) {
4725             destroy_vm($storecfg, $vmid, undef, $skiplock);
4726         } else {
4727             die "VM $vmid is running - destroy failed\n";
4728         }
4729     });
4730 }
4731
4732 # pci helpers
4733
4734 sub file_write {
4735     my ($filename, $buf) = @_;
4736
4737     my $fh = IO::File->new($filename, "w");
4738     return undef if !$fh;
4739
4740     my $res = print $fh $buf;
4741
4742     $fh->close();
4743
4744     return $res;
4745 }
4746
4747 sub pci_device_info {
4748     my ($name) = @_;
4749
4750     my $res;
4751
4752     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4753     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4754
4755     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4756     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4757
4758     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4759     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4760
4761     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4762     return undef if !defined($product) || $product !~ s/^0x//;
4763
4764     $res = {
4765         name => $name,
4766         vendor => $vendor,
4767         product => $product,
4768         domain => $domain,
4769         bus => $bus,
4770         slot => $slot,
4771         func => $func,
4772         irq => $irq,
4773         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4774     };
4775
4776     return $res;
4777 }
4778
4779 sub pci_dev_reset {
4780     my ($dev) = @_;
4781
4782     my $name = $dev->{name};
4783
4784     my $fn = "$pcisysfs/devices/$name/reset";
4785
4786     return file_write($fn, "1");
4787 }
4788
4789 sub pci_dev_bind_to_vfio {
4790     my ($dev) = @_;
4791
4792     my $name = $dev->{name};
4793
4794     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4795
4796     if (!-d $vfio_basedir) {
4797         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4798     }
4799     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4800
4801     my $testdir = "$vfio_basedir/$name";
4802     return 1 if -d $testdir;
4803
4804     my $data = "$dev->{vendor} $dev->{product}";
4805     return undef if !file_write("$vfio_basedir/new_id", $data);
4806
4807     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4808     if (!file_write($fn, $name)) {
4809         return undef if -f $fn;
4810     }
4811
4812     $fn = "$vfio_basedir/bind";
4813     if (! -d $testdir) {
4814         return undef if !file_write($fn, $name);
4815     }
4816
4817     return -d $testdir;
4818 }
4819
4820 sub pci_dev_group_bind_to_vfio {
4821     my ($pciid) = @_;
4822
4823     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4824
4825     if (!-d $vfio_basedir) {
4826         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4827     }
4828     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4829
4830     # get IOMMU group devices
4831     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4832       my @devs = grep /^0000:/, readdir($D);
4833     closedir($D);
4834
4835     foreach my $pciid (@devs) {
4836         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4837
4838         # pci bridges, switches or root ports are not supported
4839         # they have a pci_bus subdirectory so skip them
4840         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4841
4842         my $info = pci_device_info($1);
4843         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4844     }
4845
4846     return 1;
4847 }
4848
4849 sub print_pci_addr {
4850     my ($id, $bridges) = @_;
4851
4852     my $res = '';
4853     my $devices = {
4854         piix3 => { bus => 0, addr => 1 },
4855         #addr2 : first videocard
4856         balloon0 => { bus => 0, addr => 3 },
4857         watchdog => { bus => 0, addr => 4 },
4858         scsihw0 => { bus => 0, addr => 5 },
4859         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4860         scsihw1 => { bus => 0, addr => 6 },
4861         ahci0 => { bus => 0, addr => 7 },
4862         qga0 => { bus => 0, addr => 8 },
4863         spice => { bus => 0, addr => 9 },
4864         virtio0 => { bus => 0, addr => 10 },
4865         virtio1 => { bus => 0, addr => 11 },
4866         virtio2 => { bus => 0, addr => 12 },
4867         virtio3 => { bus => 0, addr => 13 },
4868         virtio4 => { bus => 0, addr => 14 },
4869         virtio5 => { bus => 0, addr => 15 },
4870         hostpci0 => { bus => 0, addr => 16 },
4871         hostpci1 => { bus => 0, addr => 17 },
4872         net0 => { bus => 0, addr => 18 },
4873         net1 => { bus => 0, addr => 19 },
4874         net2 => { bus => 0, addr => 20 },
4875         net3 => { bus => 0, addr => 21 },
4876         net4 => { bus => 0, addr => 22 },
4877         net5 => { bus => 0, addr => 23 },
4878         vga1 => { bus => 0, addr => 24 },
4879         vga2 => { bus => 0, addr => 25 },
4880         vga3 => { bus => 0, addr => 26 },
4881         hostpci2 => { bus => 0, addr => 27 },
4882         hostpci3 => { bus => 0, addr => 28 },
4883         #addr29 : usb-host (pve-usb.cfg)
4884         'pci.1' => { bus => 0, addr => 30 },
4885         'pci.2' => { bus => 0, addr => 31 },
4886         'net6' => { bus => 1, addr => 1 },
4887         'net7' => { bus => 1, addr => 2 },
4888         'net8' => { bus => 1, addr => 3 },
4889         'net9' => { bus => 1, addr => 4 },
4890         'net10' => { bus => 1, addr => 5 },
4891         'net11' => { bus => 1, addr => 6 },
4892         'net12' => { bus => 1, addr => 7 },
4893         'net13' => { bus => 1, addr => 8 },
4894         'net14' => { bus => 1, addr => 9 },
4895         'net15' => { bus => 1, addr => 10 },
4896         'net16' => { bus => 1, addr => 11 },
4897         'net17' => { bus => 1, addr => 12 },
4898         'net18' => { bus => 1, addr => 13 },
4899         'net19' => { bus => 1, addr => 14 },
4900         'net20' => { bus => 1, addr => 15 },
4901         'net21' => { bus => 1, addr => 16 },
4902         'net22' => { bus => 1, addr => 17 },
4903         'net23' => { bus => 1, addr => 18 },
4904         'net24' => { bus => 1, addr => 19 },
4905         'net25' => { bus => 1, addr => 20 },
4906         'net26' => { bus => 1, addr => 21 },
4907         'net27' => { bus => 1, addr => 22 },
4908         'net28' => { bus => 1, addr => 23 },
4909         'net29' => { bus => 1, addr => 24 },
4910         'net30' => { bus => 1, addr => 25 },
4911         'net31' => { bus => 1, addr => 26 },
4912         'xhci' => { bus => 1, addr => 27 },
4913         'virtio6' => { bus => 2, addr => 1 },
4914         'virtio7' => { bus => 2, addr => 2 },
4915         'virtio8' => { bus => 2, addr => 3 },
4916         'virtio9' => { bus => 2, addr => 4 },
4917         'virtio10' => { bus => 2, addr => 5 },
4918         'virtio11' => { bus => 2, addr => 6 },
4919         'virtio12' => { bus => 2, addr => 7 },
4920         'virtio13' => { bus => 2, addr => 8 },
4921         'virtio14' => { bus => 2, addr => 9 },
4922         'virtio15' => { bus => 2, addr => 10 },
4923         'virtioscsi0' => { bus => 3, addr => 1 },
4924         'virtioscsi1' => { bus => 3, addr => 2 },
4925         'virtioscsi2' => { bus => 3, addr => 3 },
4926         'virtioscsi3' => { bus => 3, addr => 4 },
4927         'virtioscsi4' => { bus => 3, addr => 5 },
4928         'virtioscsi5' => { bus => 3, addr => 6 },
4929         'virtioscsi6' => { bus => 3, addr => 7 },
4930         'virtioscsi7' => { bus => 3, addr => 8 },
4931         'virtioscsi8' => { bus => 3, addr => 9 },
4932         'virtioscsi9' => { bus => 3, addr => 10 },
4933         'virtioscsi10' => { bus => 3, addr => 11 },
4934         'virtioscsi11' => { bus => 3, addr => 12 },
4935         'virtioscsi12' => { bus => 3, addr => 13 },
4936         'virtioscsi13' => { bus => 3, addr => 14 },
4937         'virtioscsi14' => { bus => 3, addr => 15 },
4938         'virtioscsi15' => { bus => 3, addr => 16 },
4939         'virtioscsi16' => { bus => 3, addr => 17 },
4940         'virtioscsi17' => { bus => 3, addr => 18 },
4941         'virtioscsi18' => { bus => 3, addr => 19 },
4942         'virtioscsi19' => { bus => 3, addr => 20 },
4943         'virtioscsi20' => { bus => 3, addr => 21 },
4944         'virtioscsi21' => { bus => 3, addr => 22 },
4945         'virtioscsi22' => { bus => 3, addr => 23 },
4946         'virtioscsi23' => { bus => 3, addr => 24 },
4947         'virtioscsi24' => { bus => 3, addr => 25 },
4948         'virtioscsi25' => { bus => 3, addr => 26 },
4949         'virtioscsi26' => { bus => 3, addr => 27 },
4950         'virtioscsi27' => { bus => 3, addr => 28 },
4951         'virtioscsi28' => { bus => 3, addr => 29 },
4952         'virtioscsi29' => { bus => 3, addr => 30 },
4953         'virtioscsi30' => { bus => 3, addr => 31 },
4954
4955     };
4956
4957     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4958            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4959            my $bus = $devices->{$id}->{bus};
4960            $res = ",bus=pci.$bus,addr=$addr";
4961            $bridges->{$bus} = 1 if $bridges;
4962     }
4963     return $res;
4964
4965 }
4966
4967 sub print_pcie_addr {
4968     my ($id) = @_;
4969
4970     my $res = '';
4971     my $devices = {
4972         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4973         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4974         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4975         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4976     };
4977
4978     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4979            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4980            my $bus = $devices->{$id}->{bus};
4981            $res = ",bus=$bus,addr=$addr";
4982     }
4983     return $res;
4984
4985 }
4986
4987 # vzdump restore implementaion
4988
4989 sub tar_archive_read_firstfile {
4990     my $archive = shift;
4991
4992     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4993
4994     # try to detect archive type first
4995     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4996         die "unable to open file '$archive'\n";
4997     my $firstfile = <$fh>;
4998     kill 15, $pid;
4999     close $fh;
5000
5001     die "ERROR: archive contaions no data\n" if !$firstfile;
5002     chomp $firstfile;
5003
5004     return $firstfile;
5005 }
5006
5007 sub tar_restore_cleanup {
5008     my ($storecfg, $statfile) = @_;
5009
5010     print STDERR "starting cleanup\n";
5011
5012     if (my $fd = IO::File->new($statfile, "r")) {
5013         while (defined(my $line = <$fd>)) {
5014             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5015                 my $volid = $2;
5016                 eval {
5017                     if ($volid =~ m|^/|) {
5018                         unlink $volid || die 'unlink failed\n';
5019                     } else {
5020                         PVE::Storage::vdisk_free($storecfg, $volid);
5021                     }
5022                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5023                 };
5024                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5025             } else {
5026                 print STDERR "unable to parse line in statfile - $line";
5027             }
5028         }
5029         $fd->close();
5030     }
5031 }
5032
5033 sub restore_archive {
5034     my ($archive, $vmid, $user, $opts) = @_;
5035
5036     my $format = $opts->{format};
5037     my $comp;
5038
5039     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5040         $format = 'tar' if !$format;
5041         $comp = 'gzip';
5042     } elsif ($archive =~ m/\.tar$/) {
5043         $format = 'tar' if !$format;
5044     } elsif ($archive =~ m/.tar.lzo$/) {
5045         $format = 'tar' if !$format;
5046         $comp = 'lzop';
5047     } elsif ($archive =~ m/\.vma$/) {
5048         $format = 'vma' if !$format;
5049     } elsif ($archive =~ m/\.vma\.gz$/) {
5050         $format = 'vma' if !$format;
5051         $comp = 'gzip';
5052     } elsif ($archive =~ m/\.vma\.lzo$/) {
5053         $format = 'vma' if !$format;
5054         $comp = 'lzop';
5055     } else {
5056         $format = 'vma' if !$format; # default
5057     }
5058
5059     # try to detect archive format
5060     if ($format eq 'tar') {
5061         return restore_tar_archive($archive, $vmid, $user, $opts);
5062     } else {
5063         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5064     }
5065 }
5066
5067 sub restore_update_config_line {
5068     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5069
5070     return if $line =~ m/^\#qmdump\#/;
5071     return if $line =~ m/^\#vzdump\#/;
5072     return if $line =~ m/^lock:/;
5073     return if $line =~ m/^unused\d+:/;
5074     return if $line =~ m/^parent:/;
5075     return if $line =~ m/^template:/; # restored VM is never a template
5076
5077     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5078         # try to convert old 1.X settings
5079         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5080         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5081             my ($model, $macaddr) = split(/\=/, $devconfig);
5082             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5083             my $net = {
5084                 model => $model,
5085                 bridge => "vmbr$ind",
5086                 macaddr => $macaddr,
5087             };
5088             my $netstr = print_net($net);
5089
5090             print $outfd "net$cookie->{netcount}: $netstr\n";
5091             $cookie->{netcount}++;
5092         }
5093     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5094         my ($id, $netstr) = ($1, $2);
5095         my $net = parse_net($netstr);
5096         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5097         $netstr = print_net($net);
5098         print $outfd "$id: $netstr\n";
5099     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5100         my $virtdev = $1;
5101         my $value = $3;
5102         my $di = parse_drive($virtdev, $value);
5103         if (defined($di->{backup}) && !$di->{backup}) {
5104             print $outfd "#$line";
5105         } elsif ($map->{$virtdev}) {
5106             delete $di->{format}; # format can change on restore
5107             $di->{file} = $map->{$virtdev};
5108             $value = print_drive($vmid, $di);
5109             print $outfd "$virtdev: $value\n";
5110         } else {
5111             print $outfd $line;
5112         }
5113     } else {
5114         print $outfd $line;
5115     }
5116 }
5117
5118 sub scan_volids {
5119     my ($cfg, $vmid) = @_;
5120
5121     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5122
5123     my $volid_hash = {};
5124     foreach my $storeid (keys %$info) {
5125         foreach my $item (@{$info->{$storeid}}) {
5126             next if !($item->{volid} && $item->{size});
5127             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5128             $volid_hash->{$item->{volid}} = $item;
5129         }
5130     }
5131
5132     return $volid_hash;
5133 }
5134
5135 sub is_volume_in_use {
5136     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5137
5138     my $path = PVE::Storage::path($storecfg, $volid);
5139
5140     my $scan_config = sub {
5141         my ($cref, $snapname) = @_;
5142
5143         foreach my $key (keys %$cref) {
5144             my $value = $cref->{$key};
5145             if (is_valid_drivename($key)) {
5146                 next if $skip_drive && $key eq $skip_drive;
5147                 my $drive = parse_drive($key, $value);
5148                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5149                 return 1 if $volid eq $drive->{file};
5150                 if ($drive->{file} =~ m!^/!) {
5151                     return 1 if $drive->{file} eq $path;
5152                 } else {
5153                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5154                     next if !$storeid;
5155                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5156                     next if !$scfg;
5157                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5158                 }
5159             }
5160         }
5161
5162         return 0;
5163     };
5164
5165     return 1 if &$scan_config($conf);
5166
5167     undef $skip_drive;
5168
5169     foreach my $snapname (keys %{$conf->{snapshots}}) {
5170         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5171     }
5172
5173     return 0;
5174 }
5175
5176 sub update_disksize {
5177     my ($vmid, $conf, $volid_hash) = @_;
5178
5179     my $changes;
5180
5181     my $used = {};
5182
5183     # Note: it is allowed to define multiple storages with same path (alias), so
5184     # we need to check both 'volid' and real 'path' (two different volid can point
5185     # to the same path).
5186
5187     my $usedpath = {};
5188
5189     # update size info
5190     foreach my $opt (keys %$conf) {
5191         if (is_valid_drivename($opt)) {
5192             my $drive = parse_drive($opt, $conf->{$opt});
5193             my $volid = $drive->{file};
5194             next if !$volid;
5195
5196             $used->{$volid} = 1;
5197             if ($volid_hash->{$volid} &&
5198                 (my $path = $volid_hash->{$volid}->{path})) {
5199                 $usedpath->{$path} = 1;
5200             }
5201
5202             next if drive_is_cdrom($drive);
5203             next if !$volid_hash->{$volid};
5204
5205             $drive->{size} = $volid_hash->{$volid}->{size};
5206             my $new = print_drive($vmid, $drive);
5207             if ($new ne $conf->{$opt}) {
5208                 $changes = 1;
5209                 $conf->{$opt} = $new;
5210             }
5211         }
5212     }
5213
5214     # remove 'unusedX' entry if volume is used
5215     foreach my $opt (keys %$conf) {
5216         next if $opt !~ m/^unused\d+$/;
5217         my $volid = $conf->{$opt};
5218         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5219         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5220             $changes = 1;
5221             delete $conf->{$opt};
5222         }
5223     }
5224
5225     foreach my $volid (sort keys %$volid_hash) {
5226         next if $volid =~ m/vm-$vmid-state-/;
5227         next if $used->{$volid};
5228         my $path = $volid_hash->{$volid}->{path};
5229         next if !$path; # just to be sure
5230         next if $usedpath->{$path};
5231         $changes = 1;
5232         PVE::QemuConfig->add_unused_volume($conf, $volid);
5233         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5234     }
5235
5236     return $changes;
5237 }
5238
5239 sub rescan {
5240     my ($vmid, $nolock) = @_;
5241
5242     my $cfg = PVE::Storage::config();
5243
5244     my $volid_hash = scan_volids($cfg, $vmid);
5245
5246     my $updatefn =  sub {
5247         my ($vmid) = @_;
5248
5249         my $conf = PVE::QemuConfig->load_config($vmid);
5250
5251         PVE::QemuConfig->check_lock($conf);
5252
5253         my $vm_volids = {};
5254         foreach my $volid (keys %$volid_hash) {
5255             my $info = $volid_hash->{$volid};
5256             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5257         }
5258
5259         my $changes = update_disksize($vmid, $conf, $vm_volids);
5260
5261         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5262     };
5263
5264     if (defined($vmid)) {
5265         if ($nolock) {
5266             &$updatefn($vmid);
5267         } else {
5268             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5269         }
5270     } else {
5271         my $vmlist = config_list();
5272         foreach my $vmid (keys %$vmlist) {
5273             if ($nolock) {
5274                 &$updatefn($vmid);
5275             } else {
5276                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5277             }
5278         }
5279     }
5280 }
5281
5282 sub restore_vma_archive {
5283     my ($archive, $vmid, $user, $opts, $comp) = @_;
5284
5285     my $input = $archive eq '-' ? "<&STDIN" : undef;
5286     my $readfrom = $archive;
5287
5288     my $uncomp = '';
5289     if ($comp) {
5290         $readfrom = '-';
5291         my $qarchive = PVE::Tools::shellquote($archive);
5292         if ($comp eq 'gzip') {
5293             $uncomp = "zcat $qarchive|";
5294         } elsif ($comp eq 'lzop') {
5295             $uncomp = "lzop -d -c $qarchive|";
5296         } else {
5297             die "unknown compression method '$comp'\n";
5298         }
5299
5300     }
5301
5302     my $tmpdir = "/var/tmp/vzdumptmp$$";
5303     rmtree $tmpdir;
5304
5305     # disable interrupts (always do cleanups)
5306     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5307         warn "got interrupt - ignored\n";
5308     };
5309
5310     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5311     POSIX::mkfifo($mapfifo, 0600);
5312     my $fifofh;
5313
5314     my $openfifo = sub {
5315         open($fifofh, '>', $mapfifo) || die $!;
5316     };
5317
5318     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5319
5320     my $oldtimeout;
5321     my $timeout = 5;
5322
5323     my $devinfo = {};
5324
5325     my $rpcenv = PVE::RPCEnvironment::get();
5326
5327     my $conffile = PVE::QemuConfig->config_file($vmid);
5328     my $tmpfn = "$conffile.$$.tmp";
5329
5330     # Note: $oldconf is undef if VM does not exists
5331     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5332     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5333
5334     my $print_devmap = sub {
5335         my $virtdev_hash = {};
5336
5337         my $cfgfn = "$tmpdir/qemu-server.conf";
5338
5339         # we can read the config - that is already extracted
5340         my $fh = IO::File->new($cfgfn, "r") ||
5341             "unable to read qemu-server.conf - $!\n";
5342
5343         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5344         if (-f $fwcfgfn) {
5345             my $pve_firewall_dir = '/etc/pve/firewall';
5346             mkdir $pve_firewall_dir; # make sure the dir exists
5347             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5348         }
5349
5350         while (defined(my $line = <$fh>)) {
5351             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5352                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5353                 die "archive does not contain data for drive '$virtdev'\n"
5354                     if !$devinfo->{$devname};
5355                 if (defined($opts->{storage})) {
5356                     $storeid = $opts->{storage} || 'local';
5357                 } elsif (!$storeid) {
5358                     $storeid = 'local';
5359                 }
5360                 $format = 'raw' if !$format;
5361                 $devinfo->{$devname}->{devname} = $devname;
5362                 $devinfo->{$devname}->{virtdev} = $virtdev;
5363                 $devinfo->{$devname}->{format} = $format;
5364                 $devinfo->{$devname}->{storeid} = $storeid;
5365
5366                 # check permission on storage
5367                 my $pool = $opts->{pool}; # todo: do we need that?
5368                 if ($user ne 'root@pam') {
5369                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5370                 }
5371
5372                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5373             }
5374         }
5375
5376         foreach my $devname (keys %$devinfo) {
5377             die "found no device mapping information for device '$devname'\n"
5378                 if !$devinfo->{$devname}->{virtdev};
5379         }
5380
5381         my $cfg = PVE::Storage::config();
5382
5383         # create empty/temp config
5384         if ($oldconf) {
5385             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5386             foreach_drive($oldconf, sub {
5387                 my ($ds, $drive) = @_;
5388
5389                 return if drive_is_cdrom($drive);
5390
5391                 my $volid = $drive->{file};
5392
5393                 return if !$volid || $volid =~ m|^/|;
5394
5395                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5396                 return if !$path || !$owner || ($owner != $vmid);
5397
5398                 # Note: only delete disk we want to restore
5399                 # other volumes will become unused
5400                 if ($virtdev_hash->{$ds}) {
5401                     PVE::Storage::vdisk_free($cfg, $volid);
5402                 }
5403             });
5404
5405             # delete vmstate files
5406             # since after the restore we have no snapshots anymore
5407             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5408                 my $snap = $oldconf->{snapshots}->{$snapname};
5409                 if ($snap->{vmstate}) {
5410                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5411                     if (my $err = $@) {
5412                         warn $err;
5413                     }
5414                 }
5415             }
5416         }
5417
5418         my $map = {};
5419         foreach my $virtdev (sort keys %$virtdev_hash) {
5420             my $d = $virtdev_hash->{$virtdev};
5421             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5422             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5423
5424             # test if requested format is supported
5425             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5426             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5427             $d->{format} = $defFormat if !$supported;
5428
5429             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5430                                                   $d->{format}, undef, $alloc_size);
5431             print STDERR "new volume ID is '$volid'\n";
5432             $d->{volid} = $volid;
5433             my $path = PVE::Storage::path($cfg, $volid);
5434
5435             PVE::Storage::activate_volumes($cfg,[$volid]);
5436
5437             my $write_zeros = 1;
5438             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5439                 $write_zeros = 0;
5440             }
5441
5442             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5443
5444             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5445             $map->{$virtdev} = $volid;
5446         }
5447
5448         $fh->seek(0, 0) || die "seek failed - $!\n";
5449
5450         my $outfd = new IO::File ($tmpfn, "w") ||
5451             die "unable to write config for VM $vmid\n";
5452
5453         my $cookie = { netcount => 0 };
5454         while (defined(my $line = <$fh>)) {
5455             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5456         }
5457
5458         $fh->close();
5459         $outfd->close();
5460     };
5461
5462     eval {
5463         # enable interrupts
5464         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5465             die "interrupted by signal\n";
5466         };
5467         local $SIG{ALRM} = sub { die "got timeout\n"; };
5468
5469         $oldtimeout = alarm($timeout);
5470
5471         my $parser = sub {
5472             my $line = shift;
5473
5474             print "$line\n";
5475
5476             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5477                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5478                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5479             } elsif ($line =~ m/^CTIME: /) {
5480                 # we correctly received the vma config, so we can disable
5481                 # the timeout now for disk allocation (set to 10 minutes, so
5482                 # that we always timeout if something goes wrong)
5483                 alarm(600);
5484                 &$print_devmap();
5485                 print $fifofh "done\n";
5486                 my $tmp = $oldtimeout || 0;
5487                 $oldtimeout = undef;
5488                 alarm($tmp);
5489                 close($fifofh);
5490             }
5491         };
5492
5493         print "restore vma archive: $cmd\n";
5494         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5495     };
5496     my $err = $@;
5497
5498     alarm($oldtimeout) if $oldtimeout;
5499
5500     my $vollist = [];
5501     foreach my $devname (keys %$devinfo) {
5502         my $volid = $devinfo->{$devname}->{volid};
5503         push @$vollist, $volid if $volid;
5504     }
5505
5506     my $cfg = PVE::Storage::config();
5507     PVE::Storage::deactivate_volumes($cfg, $vollist);
5508
5509     unlink $mapfifo;
5510
5511     if ($err) {
5512         rmtree $tmpdir;
5513         unlink $tmpfn;
5514
5515         foreach my $devname (keys %$devinfo) {
5516             my $volid = $devinfo->{$devname}->{volid};
5517             next if !$volid;
5518             eval {
5519                 if ($volid =~ m|^/|) {
5520                     unlink $volid || die 'unlink failed\n';
5521                 } else {
5522                     PVE::Storage::vdisk_free($cfg, $volid);
5523                 }
5524                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5525             };
5526             print STDERR "unable to cleanup '$volid' - $@" if $@;
5527         }
5528         die $err;
5529     }
5530
5531     rmtree $tmpdir;
5532
5533     rename($tmpfn, $conffile) ||
5534         die "unable to commit configuration file '$conffile'\n";
5535
5536     PVE::Cluster::cfs_update(); # make sure we read new file
5537
5538     eval { rescan($vmid, 1); };
5539     warn $@ if $@;
5540 }
5541
5542 sub restore_tar_archive {
5543     my ($archive, $vmid, $user, $opts) = @_;
5544
5545     if ($archive ne '-') {
5546         my $firstfile = tar_archive_read_firstfile($archive);
5547         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5548             if $firstfile ne 'qemu-server.conf';
5549     }
5550
5551     my $storecfg = PVE::Storage::config();
5552
5553     # destroy existing data - keep empty config
5554     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5555     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5556
5557     my $tocmd = "/usr/lib/qemu-server/qmextract";
5558
5559     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5560     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5561     $tocmd .= ' --prealloc' if $opts->{prealloc};
5562     $tocmd .= ' --info' if $opts->{info};
5563
5564     # tar option "xf" does not autodetect compression when read from STDIN,
5565     # so we pipe to zcat
5566     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5567         PVE::Tools::shellquote("--to-command=$tocmd");
5568
5569     my $tmpdir = "/var/tmp/vzdumptmp$$";
5570     mkpath $tmpdir;
5571
5572     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5573     local $ENV{VZDUMP_VMID} = $vmid;
5574     local $ENV{VZDUMP_USER} = $user;
5575
5576     my $conffile = PVE::QemuConfig->config_file($vmid);
5577     my $tmpfn = "$conffile.$$.tmp";
5578
5579     # disable interrupts (always do cleanups)
5580     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5581         print STDERR "got interrupt - ignored\n";
5582     };
5583
5584     eval {
5585         # enable interrupts
5586         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5587             die "interrupted by signal\n";
5588         };
5589
5590         if ($archive eq '-') {
5591             print "extracting archive from STDIN\n";
5592             run_command($cmd, input => "<&STDIN");
5593         } else {
5594             print "extracting archive '$archive'\n";
5595             run_command($cmd);
5596         }
5597
5598         return if $opts->{info};
5599
5600         # read new mapping
5601         my $map = {};
5602         my $statfile = "$tmpdir/qmrestore.stat";
5603         if (my $fd = IO::File->new($statfile, "r")) {
5604             while (defined (my $line = <$fd>)) {
5605                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5606                     $map->{$1} = $2 if $1;
5607                 } else {
5608                     print STDERR "unable to parse line in statfile - $line\n";
5609                 }
5610             }
5611             $fd->close();
5612         }
5613
5614         my $confsrc = "$tmpdir/qemu-server.conf";
5615
5616         my $srcfd = new IO::File($confsrc, "r") ||
5617             die "unable to open file '$confsrc'\n";
5618
5619         my $outfd = new IO::File ($tmpfn, "w") ||
5620             die "unable to write config for VM $vmid\n";
5621
5622         my $cookie = { netcount => 0 };
5623         while (defined (my $line = <$srcfd>)) {
5624             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5625         }
5626
5627         $srcfd->close();
5628         $outfd->close();
5629     };
5630     my $err = $@;
5631
5632     if ($err) {
5633
5634         unlink $tmpfn;
5635
5636         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5637
5638         die $err;
5639     }
5640
5641     rmtree $tmpdir;
5642
5643     rename $tmpfn, $conffile ||
5644         die "unable to commit configuration file '$conffile'\n";
5645
5646     PVE::Cluster::cfs_update(); # make sure we read new file
5647
5648     eval { rescan($vmid, 1); };
5649     warn $@ if $@;
5650 };
5651
5652 sub foreach_writable_storage {
5653     my ($conf, $func) = @_;
5654
5655     my $sidhash = {};
5656
5657     foreach my $ds (keys %$conf) {
5658         next if !is_valid_drivename($ds);
5659
5660         my $drive = parse_drive($ds, $conf->{$ds});
5661         next if !$drive;
5662         next if drive_is_cdrom($drive);
5663
5664         my $volid = $drive->{file};
5665
5666         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5667         $sidhash->{$sid} = $sid if $sid;
5668     }
5669
5670     foreach my $sid (sort keys %$sidhash) {
5671         &$func($sid);
5672     }
5673 }
5674
5675 sub do_snapshots_with_qemu {
5676     my ($storecfg, $volid) = @_;
5677
5678     my $storage_name = PVE::Storage::parse_volume_id($volid);
5679
5680     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5681         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5682         return 1;
5683     }
5684
5685     if ($volid =~ m/\.(qcow2|qed)$/){
5686         return 1;
5687     }
5688
5689     return undef;
5690 }
5691
5692 sub qga_check_running {
5693     my ($vmid) = @_;
5694
5695     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5696     if ($@) {
5697         warn "Qemu Guest Agent are not running - $@";
5698         return 0;
5699     }
5700     return 1;
5701 }
5702
5703 sub template_create {
5704     my ($vmid, $conf, $disk) = @_;
5705
5706     my $storecfg = PVE::Storage::config();
5707
5708     foreach_drive($conf, sub {
5709         my ($ds, $drive) = @_;
5710
5711         return if drive_is_cdrom($drive);
5712         return if $disk && $ds ne $disk;
5713
5714         my $volid = $drive->{file};
5715         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5716
5717         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5718         $drive->{file} = $voliddst;
5719         $conf->{$ds} = print_drive($vmid, $drive);
5720         PVE::QemuConfig->write_config($vmid, $conf);
5721     });
5722 }
5723
5724 sub qemu_img_convert {
5725     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5726
5727     my $storecfg = PVE::Storage::config();
5728     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5729     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5730
5731     if ($src_storeid && $dst_storeid) {
5732
5733         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5734
5735         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5736         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5737
5738         my $src_format = qemu_img_format($src_scfg, $src_volname);
5739         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5740
5741         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5742         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5743
5744         my $cmd = [];
5745         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5746         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5747         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5748         if ($is_zero_initialized) {
5749             push @$cmd, "zeroinit:$dst_path";
5750         } else {
5751             push @$cmd, $dst_path;
5752         }
5753
5754         my $parser = sub {
5755             my $line = shift;
5756             if($line =~ m/\((\S+)\/100\%\)/){
5757                 my $percent = $1;
5758                 my $transferred = int($size * $percent / 100);
5759                 my $remaining = $size - $transferred;
5760
5761                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5762             }
5763
5764         };
5765
5766         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5767         my $err = $@;
5768         die "copy failed: $err" if $err;
5769     }
5770 }
5771
5772 sub qemu_img_format {
5773     my ($scfg, $volname) = @_;
5774
5775     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5776         return $1;
5777     } else {
5778         return "raw";
5779     }
5780 }
5781
5782 sub qemu_drive_mirror {
5783     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5784
5785     my $storecfg = PVE::Storage::config();
5786     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5787
5788     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5789
5790     my $format = qemu_img_format($dst_scfg, $dst_volname);
5791
5792     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5793
5794     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5795
5796     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5797     $opts->{format} = $format if $format;
5798
5799     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5800
5801     my $finish_job = sub {
5802         while (1) {
5803             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5804             my $stat = @$stats[0];
5805             last if !$stat;
5806             sleep 1;
5807         }
5808     };
5809
5810     eval {
5811     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5812         while (1) {
5813             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5814             my $stat = @$stats[0];
5815             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5816             die "error job is not mirroring" if $stat->{type} ne "mirror";
5817
5818             my $busy = $stat->{busy};
5819             my $ready = $stat->{ready};
5820
5821             if (my $total = $stat->{len}) {
5822                 my $transferred = $stat->{offset} || 0;
5823                 my $remaining = $total - $transferred;
5824                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5825
5826                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5827             }
5828
5829
5830             if ($stat->{ready} eq 'true') {
5831
5832                 last if $vmiddst != $vmid;
5833
5834                 # try to switch the disk if source and destination are on the same guest
5835                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5836                 if (!$@) {
5837                     &$finish_job();
5838                     last;
5839                 }
5840                 die $@ if $@ !~ m/cannot be completed/;
5841             }
5842             sleep 1;
5843         }
5844
5845
5846     };
5847     my $err = $@;
5848
5849     my $cancel_job = sub {
5850         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5851         &$finish_job();
5852     };
5853
5854     if ($err) {
5855         eval { &$cancel_job(); };
5856         die "mirroring error: $err";
5857     }
5858
5859     if ($vmiddst != $vmid) {
5860         # if we clone a disk for a new target vm, we don't switch the disk
5861         &$cancel_job(); # so we call block-job-cancel
5862     }
5863 }
5864
5865 sub clone_disk {
5866     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5867         $newvmid, $storage, $format, $full, $newvollist) = @_;
5868
5869     my $newvolid;
5870
5871     if (!$full) {
5872         print "create linked clone of drive $drivename ($drive->{file})\n";
5873         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5874         push @$newvollist, $newvolid;
5875     } else {
5876         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5877         $storeid = $storage if $storage;
5878
5879         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5880         if (!$format) {
5881             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5882             $format = qemu_img_format($scfg, $volname);
5883         }
5884
5885         # test if requested format is supported - else use default
5886         my $supported = grep { $_ eq $format } @$validFormats;
5887         $format = $defFormat if !$supported;
5888
5889         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5890
5891         print "create full clone of drive $drivename ($drive->{file})\n";
5892         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5893         push @$newvollist, $newvolid;
5894
5895         PVE::Storage::activate_volumes($storecfg, $newvollist);
5896
5897         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5898         if (!$running || $snapname) {
5899             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5900         } else {
5901             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5902         }
5903     }
5904
5905     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5906
5907     my $disk = $drive;
5908     $disk->{format} = undef;
5909     $disk->{file} = $newvolid;
5910     $disk->{size} = $size;
5911
5912     return $disk;
5913 }
5914
5915 # this only works if VM is running
5916 sub get_current_qemu_machine {
5917     my ($vmid) = @_;
5918
5919     my $cmd = { execute => 'query-machines', arguments => {} };
5920     my $res = vm_qmp_command($vmid, $cmd);
5921
5922     my ($current, $default);
5923     foreach my $e (@$res) {
5924         $default = $e->{name} if $e->{'is-default'};
5925         $current = $e->{name} if $e->{'is-current'};
5926     }
5927
5928     # fallback to the default machine if current is not supported by qemu
5929     return $current || $default || 'pc';
5930 }
5931
5932 sub qemu_machine_feature_enabled {
5933     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5934
5935     my $current_major;
5936     my $current_minor;
5937
5938     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5939
5940         $current_major = $3;
5941         $current_minor = $4;
5942
5943     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5944
5945         $current_major = $1;
5946         $current_minor = $2;
5947     }
5948
5949     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5950
5951
5952 }
5953
5954 sub qemu_machine_pxe {
5955     my ($vmid, $conf, $machine) = @_;
5956
5957     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5958
5959     foreach my $opt (keys %$conf) {
5960         next if $opt !~ m/^net(\d+)$/;
5961         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5962         next if !$net;
5963         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5964         return $machine.".pxe" if $romfile =~ m/pxe/;
5965         last;
5966     }
5967
5968     return $machine;
5969 }
5970
5971 sub qemu_use_old_bios_files {
5972     my ($machine_type) = @_;
5973
5974     return if !$machine_type;
5975
5976     my $use_old_bios_files = undef;
5977
5978     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5979         $machine_type = $1;
5980         $use_old_bios_files = 1;
5981     } else {
5982         my $kvmver = kvm_user_version();
5983         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5984         # load new efi bios files on migration. So this hack is required to allow
5985         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5986         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5987         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5988     }
5989
5990     return ($use_old_bios_files, $machine_type);
5991 }
5992
5993 sub lspci {
5994
5995     my $devices = {};
5996
5997     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5998             my (undef, $id, $function) = @_;
5999             my $res = { id => $id, function => $function};
6000             push @{$devices->{$id}}, $res;
6001     });
6002
6003     return $devices;
6004 }
6005
6006 sub vm_iothreads_list {
6007     my ($vmid) = @_;
6008
6009     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6010
6011     my $iothreads = {};
6012     foreach my $iothread (@$res) {
6013         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6014     }
6015
6016     return $iothreads;
6017 }
6018
6019 sub scsihw_infos {
6020     my ($conf, $drive) = @_;
6021
6022     my $maxdev = 0;
6023
6024     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6025         $maxdev = 7;
6026     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6027         $maxdev = 1;
6028     } else {
6029         $maxdev = 256;
6030     }
6031
6032     my $controller = int($drive->{index} / $maxdev);
6033     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6034
6035     return ($maxdev, $controller, $controller_prefix);
6036 }
6037
6038 # bash completion helper
6039
6040 sub complete_backup_archives {
6041     my ($cmdname, $pname, $cvalue) = @_;
6042
6043     my $cfg = PVE::Storage::config();
6044
6045     my $storeid;
6046
6047     if ($cvalue =~ m/^([^:]+):/) {
6048         $storeid = $1;
6049     }
6050
6051     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6052
6053     my $res = [];
6054     foreach my $id (keys %$data) {
6055         foreach my $item (@{$data->{$id}}) {
6056             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6057             push @$res, $item->{volid} if defined($item->{volid});
6058         }
6059     }
6060
6061     return $res;
6062 }
6063
6064 my $complete_vmid_full = sub {
6065     my ($running) = @_;
6066
6067     my $idlist = vmstatus();
6068
6069     my $res = [];
6070
6071     foreach my $id (keys %$idlist) {
6072         my $d = $idlist->{$id};
6073         if (defined($running)) {
6074             next if $d->{template};
6075             next if $running && $d->{status} ne 'running';
6076             next if !$running && $d->{status} eq 'running';
6077         }
6078         push @$res, $id;
6079
6080     }
6081     return $res;
6082 };
6083
6084 sub complete_vmid {
6085     return &$complete_vmid_full();
6086 }
6087
6088 sub complete_vmid_stopped {
6089     return &$complete_vmid_full(0);
6090 }
6091
6092 sub complete_vmid_running {
6093     return &$complete_vmid_full(1);
6094 }
6095
6096 sub complete_storage {
6097
6098     my $cfg = PVE::Storage::config();
6099     my $ids = $cfg->{ids};
6100
6101     my $res = [];
6102     foreach my $sid (keys %$ids) {
6103         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6104         next if !$ids->{$sid}->{content}->{images};
6105         push @$res, $sid;
6106     }
6107
6108     return $res;
6109 }
6110
6111 1;