PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::Memory;
  34 use Time::HiRes qw(gettimeofday);
  35 use File::Copy qw(copy);
  36 use URI::Escape;
  37
  38 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  39
  40 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  41
  42 # Note about locking: we use flock on the config file protect
  43 # against concurent actions.
  44 # Aditionaly, we have a 'lock' setting in the config file. This
  45 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  46 # allowed when such lock is set. But you can ignore this kind of
  47 # lock with the --skiplock flag.
  48
  49 cfs_register_file('/qemu-server/',
  50                   \&parse_vm_config,
  51                   \&write_vm_config);
  52
  53 PVE::JSONSchema::register_standard_option('skiplock', {
  54     description => "Ignore locks - only root is allowed to use this option.",
  55     type => 'boolean',
  56     optional => 1,
  57 });
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 #no warnings 'redefine';
  73
  74 sub cgroups_write {
  75    my ($controller, $vmid, $option, $value) = @_;
  76
  77    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  78    PVE::ProcFSTools::write_proc_entry($path, $value);
  79
  80 }
  81
  82 my $nodename = PVE::INotify::nodename();
  83
  84 mkdir "/etc/pve/nodes/$nodename";
  85 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  86 mkdir $confdir;
  87
  88 my $var_run_tmpdir = "/var/run/qemu-server";
  89 mkdir $var_run_tmpdir;
  90
  91 my $lock_dir = "/var/lock/qemu-server";
  92 mkdir $lock_dir;
  93
  94 my $pcisysfs = "/sys/bus/pci";
  95
  96 my $cpu_vendor_list = {
  97     # Intel CPUs
  98     486 => 'GenuineIntel',
  99     pentium => 'GenuineIntel',
 100     pentium2  => 'GenuineIntel',
 101     pentium3  => 'GenuineIntel',
 102     coreduo => 'GenuineIntel',
 103     core2duo => 'GenuineIntel',
 104     Conroe  => 'GenuineIntel',
 105     Penryn  => 'GenuineIntel',
 106     Nehalem  => 'GenuineIntel',
 107     Westmere => 'GenuineIntel',
 108     SandyBridge => 'GenuineIntel',
 109     IvyBridge => 'GenuineIntel',
 110     Haswell => 'GenuineIntel',
 111     'Haswell-noTSX' => 'GenuineIntel',
 112     Broadwell => 'GenuineIntel',
 113     'Broadwell-noTSX' => 'GenuineIntel',
 114
 115     # AMD CPUs
 116     athlon => 'AuthenticAMD',
 117     phenom  => 'AuthenticAMD',
 118     Opteron_G1  => 'AuthenticAMD',
 119     Opteron_G2  => 'AuthenticAMD',
 120     Opteron_G3  => 'AuthenticAMD',
 121     Opteron_G4  => 'AuthenticAMD',
 122     Opteron_G5  => 'AuthenticAMD',
 123
 124     # generic types, use vendor from host node
 125     host => 'default',
 126     kvm32 => 'default',
 127     kvm64 => 'default',
 128     qemu32 => 'default',
 129     qemu64 => 'default',
 130 };
 131
 132 my $cpu_fmt = {
 133     cputype => {
 134         description => "Emulated CPU type.",
 135         type => 'string',
 136         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 137         format_description => 'cputype',
 138         default => 'kvm64',
 139         default_key => 1,
 140     },
 141     hidden => {
 142         description => "Do not identify as a KVM virtual machine.",
 143         type => 'boolean',
 144         optional => 1,
 145         default => 0
 146     },
 147 };
 148
 149 my $watchdog_fmt = {
 150     model => {
 151         default_key => 1,
 152         type => 'string',
 153         enum => [qw(i6300esb ib700)],
 154         description => "Watchdog type to emulate.",
 155         default => 'i6300esb',
 156         optional => 1,
 157     },
 158     action => {
 159         type => 'string',
 160         enum => [qw(reset shutdown poweroff pause debug none)],
 161         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 162         optional => 1,
 163     },
 164 };
 165 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 166
 167 my $confdesc = {
 168     onboot => {
 169         optional => 1,
 170         type => 'boolean',
 171         description => "Specifies whether a VM will be started during system bootup.",
 172         default => 0,
 173     },
 174     autostart => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Automatic restart after crash (currently ignored).",
 178         default => 0,
 179     },
 180     hotplug => {
 181         optional => 1,
 182         type => 'string', format => 'pve-hotplug-features',
 183         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 184         default => 'network,disk,usb',
 185     },
 186     reboot => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 190         default => 1,
 191     },
 192     lock => {
 193         optional => 1,
 194         type => 'string',
 195         description => "Lock/unlock the VM.",
 196         enum => [qw(migrate backup snapshot rollback)],
 197     },
 198     cpulimit => {
 199         optional => 1,
 200         type => 'number',
 201         description => "Limit of CPU usage.",
 202         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 203         minimum => 0,
 204         maximum => 128,
 205         default => 0,
 206     },
 207     cpuunits => {
 208         optional => 1,
 209         type => 'integer',
 210         description => "CPU weight for a VM.",
 211         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 212         minimum => 0,
 213         maximum => 500000,
 214         default => 1000,
 215     },
 216     memory => {
 217         optional => 1,
 218         type => 'integer',
 219         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 220         minimum => 16,
 221         default => 512,
 222     },
 223     balloon => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 227         minimum => 0,
 228     },
 229     shares => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 233         minimum => 0,
 234         maximum => 50000,
 235         default => 1000,
 236     },
 237     keyboard => {
 238         optional => 1,
 239         type => 'string',
 240         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 241         enum => PVE::Tools::kvmkeymaplist(),
 242         default => 'en-us',
 243     },
 244     name => {
 245         optional => 1,
 246         type => 'string', format => 'dns-name',
 247         description => "Set a name for the VM. Only used on the configuration web interface.",
 248     },
 249     scsihw => {
 250         optional => 1,
 251         type => 'string',
 252         description => "SCSI controller model",
 253         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 254         default => 'lsi',
 255     },
 256     description => {
 257         optional => 1,
 258         type => 'string',
 259         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 260     },
 261     ostype => {
 262         optional => 1,
 263         type => 'string',
 264         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 265         description => "Specify guest operating system.",
 266         verbose_description => <<EODESC,
 267 Specify guest operating system. This is used to enable special
 268 optimization/features for specific operating systems:
 269
 270 [horizontal]
 271 other;; unspecified OS
 272 wxp;; Microsoft Windows XP
 273 w2k;; Microsoft Windows 2000
 274 w2k3;; Microsoft Windows 2003
 275 w2k8;; Microsoft Windows 2008
 276 wvista;; Microsoft Windows Vista
 277 win7;; Microsoft Windows 7
 278 win8;; Microsoft Windows 8/2012
 279 l24;; Linux 2.4 Kernel
 280 l26;; Linux 2.6/3.X Kernel
 281 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 282 EODESC
 283     },
 284     boot => {
 285         optional => 1,
 286         type => 'string',
 287         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 288         pattern => '[acdn]{1,4}',
 289         default => 'cdn',
 290     },
 291     bootdisk => {
 292         optional => 1,
 293         type => 'string', format => 'pve-qm-bootdisk',
 294         description => "Enable booting from specified disk.",
 295         pattern => '(ide|sata|scsi|virtio)\d+',
 296     },
 297     smp => {
 298         optional => 1,
 299         type => 'integer',
 300         description => "The number of CPUs. Please use option -sockets instead.",
 301         minimum => 1,
 302         default => 1,
 303     },
 304     sockets => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPU sockets.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     cores => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of cores per socket.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     numa => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Enable/disable NUMA.",
 322         default => 0,
 323     },
 324     vcpus => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "Number of hotplugged vcpus.",
 328         minimum => 1,
 329         default => 0,
 330     },
 331     acpi => {
 332         optional => 1,
 333         type => 'boolean',
 334         description => "Enable/disable ACPI.",
 335         default => 1,
 336     },
 337     agent => {
 338         optional => 1,
 339         type => 'boolean',
 340         description => "Enable/disable Qemu GuestAgent.",
 341         default => 0,
 342     },
 343     kvm => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable KVM hardware virtualization.",
 347         default => 1,
 348     },
 349     tdf => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable time drift fix.",
 353         default => 0,
 354     },
 355     localtime => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 359     },
 360     freeze => {
 361         optional => 1,
 362         type => 'boolean',
 363         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 364     },
 365     vga => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Select the VGA type.",
 369         verbose_description => "Select the VGA type. If you want to use high resolution" .
 370             " modes (>= 1280x1024x16) then you should use the options " .
 371             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 372             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 373             "display sever. For win* OS you can select how many independent " .
 374             "displays you want, Linux guests can add displays them self. " .
 375             "You can also run without any graphic card, using a serial device" .
 376             " as terminal.",
 377         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 378     },
 379     watchdog => {
 380         optional => 1,
 381         type => 'string', format => 'pve-qm-watchdog',
 382         description => "Create a virtual hardware watchdog device.",
 383         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 384             " (by a guest action), the watchdog must be periodically polled " .
 385             "by an agent inside the guest or else the watchdog will reset " .
 386             "the guest (or execute the respective action specified)",
 387     },
 388     startdate => {
 389         optional => 1,
 390         type => 'string',
 391         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 392         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 393         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 394         default => 'now',
 395     },
 396     startup =>  get_standard_option('pve-startup-order'),
 397     template => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Enable/disable Template.",
 401         default => 0,
 402     },
 403     args => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Arbitrary arguments passed to kvm.",
 407         verbose_description => <<EODESCR,
 408 Arbitrary arguments passed to kvm, for example:
 409
 410 args: -no-reboot -no-hpet
 411
 412 NOTE: this option is for experts only.
 413 EODESCR
 414     },
 415     tablet => {
 416         optional => 1,
 417         type => 'boolean',
 418         default => 1,
 419         description => "Enable/disable the USB tablet device.",
 420         verbose_description => "Enable/disable the USB tablet device. This device is " .
 421             "usually needed to allow absolute mouse positioning with VNC. " .
 422             "Else the mouse runs out of sync with normal VNC clients. " .
 423             "If you're running lots of console-only guests on one host, " .
 424             "you may consider disabling this to save some context switches. " .
 425             "This is turned off by default if you use spice (-vga=qxl).",
 426     },
 427     migrate_speed => {
 428         optional => 1,
 429         type => 'integer',
 430         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 431         minimum => 0,
 432         default => 0,
 433     },
 434     migrate_downtime => {
 435         optional => 1,
 436         type => 'number',
 437         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 438         minimum => 0,
 439         default => 0.1,
 440     },
 441     cdrom => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-drive',
 444         typetext => 'volume',
 445         description => "This is an alias for option -ide2",
 446     },
 447     cpu => {
 448         optional => 1,
 449         description => "Emulated CPU type.",
 450         type => 'string',
 451         format => $cpu_fmt,
 452     },
 453     parent => get_standard_option('pve-snapshot-name', {
 454         optional => 1,
 455         description => "Parent snapshot name. This is used internally, and should not be modified.",
 456     }),
 457     snaptime => {
 458         optional => 1,
 459         description => "Timestamp for snapshots.",
 460         type => 'integer',
 461         minimum => 0,
 462     },
 463     vmstate => {
 464         optional => 1,
 465         type => 'string', format => 'pve-volume-id',
 466         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 467     },
 468     machine => {
 469         description => "Specific the Qemu machine type.",
 470         type => 'string',
 471         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 472         maxLength => 40,
 473         optional => 1,
 474     },
 475     smbios1 => {
 476         description => "Specify SMBIOS type 1 fields.",
 477         type => 'string', format => 'pve-qm-smbios1',
 478         maxLength => 256,
 479         optional => 1,
 480     },
 481     protection => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 485         default => 0,
 486     },
 487     bios => {
 488         optional => 1,
 489         type => 'string',
 490         enum => [ qw(seabios ovmf) ],
 491         description => "Select BIOS implementation.",
 492         default => 'seabios',
 493     },
 494 };
 495
 496 # what about other qemu settings ?
 497 #cpu => 'string',
 498 #machine => 'string',
 499 #fda => 'file',
 500 #fdb => 'file',
 501 #mtdblock => 'file',
 502 #sd => 'file',
 503 #pflash => 'file',
 504 #snapshot => 'bool',
 505 #bootp => 'file',
 506 ##tftp => 'dir',
 507 ##smb => 'dir',
 508 #kernel => 'file',
 509 #append => 'string',
 510 #initrd => 'file',
 511 ##soundhw => 'string',
 512
 513 while (my ($k, $v) = each %$confdesc) {
 514     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 515 }
 516
 517 my $MAX_IDE_DISKS = 4;
 518 my $MAX_SCSI_DISKS = 14;
 519 my $MAX_VIRTIO_DISKS = 16;
 520 my $MAX_SATA_DISKS = 6;
 521 my $MAX_USB_DEVICES = 5;
 522 my $MAX_NETS = 32;
 523 my $MAX_UNUSED_DISKS = 8;
 524 my $MAX_HOSTPCI_DEVICES = 4;
 525 my $MAX_SERIAL_PORTS = 4;
 526 my $MAX_PARALLEL_PORTS = 3;
 527 my $MAX_NUMA = 8;
 528
 529 my $numa_fmt = {
 530     cpus => {
 531         type => "string",
 532         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 533         description => "CPUs accessing this NUMA node.",
 534         format_description => "id[-id];...",
 535     },
 536     memory => {
 537         type => "number",
 538         description => "Amount of memory this NUMA node provides.",
 539         optional => 1,
 540     },
 541     hostnodes => {
 542         type => "string",
 543         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 544         description => "Host NUMA nodes to use.",
 545         format_description => "id[-id];...",
 546         optional => 1,
 547     },
 548     policy => {
 549         type => 'string',
 550         enum => [qw(preferred bind interleave)],
 551         description => "NUMA allocation policy.",
 552         optional => 1,
 553     },
 554 };
 555 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 556 my $numadesc = {
 557     optional => 1,
 558     type => 'string', format => $numa_fmt,
 559     description => "NUMA topology.",
 560 };
 561 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 562
 563 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 564     $confdesc->{"numa$i"} = $numadesc;
 565 }
 566
 567 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 568                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 569                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 570 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 571
 572 my $net_fmt_bridge_descr = <<__EOD__;
 573 Bridge to attach the network device to. The Proxmox VE standard bridge
 574 is called 'vmbr0'.
 575
 576 If you do not specify a bridge, we create a kvm user (NATed) network
 577 device, which provides DHCP and DNS services. The following addresses
 578 are used:
 579
 580  10.0.2.2   Gateway
 581  10.0.2.3   DNS Server
 582  10.0.2.4   SMB Server
 583
 584 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 585 __EOD__
 586
 587 my $net_fmt = {
 588     macaddr => {
 589         type => 'string',
 590         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 591         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 592         format_description => "XX:XX:XX:XX:XX:XX",
 593         optional => 1,
 594     },
 595     model => {
 596         type => 'string',
 597         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 598         format_description => 'model',
 599         enum => $nic_model_list,
 600         default_key => 1,
 601     },
 602     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 603     bridge => {
 604         type => 'string',
 605         description => $net_fmt_bridge_descr,
 606         format_description => 'bridge',
 607         optional => 1,
 608     },
 609     queues => {
 610         type => 'integer',
 611         minimum => 0, maximum => 16,
 612         description => 'Number of packet queues to be used on the device.',
 613         optional => 1,
 614     },
 615     rate => {
 616         type => 'number',
 617         minimum => 0,
 618         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 619         optional => 1,
 620     },
 621     tag => {
 622         type => 'integer',
 623         minimum => 1, maximum => 4094,
 624         description => 'VLAN tag to apply to packets on this interface.',
 625         optional => 1,
 626     },
 627     trunks => {
 628         type => 'string',
 629         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 630         description => 'VLAN trunks to pass through this interface.',
 631         format_description => 'vlanid[;vlanid...]',
 632         optional => 1,
 633     },
 634     firewall => {
 635         type => 'boolean',
 636         description => 'Whether this interface should be protected by the firewall.',
 637         optional => 1,
 638     },
 639     link_down => {
 640         type => 'boolean',
 641         description => 'Whether this interface should be disconnected (like pulling the plug).',
 642         optional => 1,
 643     },
 644 };
 645
 646 my $netdesc = {
 647     optional => 1,
 648     type => 'string', format => $net_fmt,
 649     description => "Specify network devices.",
 650 };
 651
 652 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 653
 654 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 655     $confdesc->{"net$i"} = $netdesc;
 656 }
 657
 658 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 659 sub verify_volume_id_or_qm_path {
 660     my ($volid, $noerr) = @_;
 661
 662     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 663         return $volid;
 664     }
 665
 666     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 667     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 668     if ($@) {
 669         return undef if $noerr;
 670         die $@;
 671     }
 672     return $volid;
 673 }
 674
 675 my $drivename_hash;
 676
 677 my %drivedesc_base = (
 678     volume => { alias => 'file' },
 679     file => {
 680         type => 'string',
 681         format => 'pve-volume-id-or-qm-path',
 682         default_key => 1,
 683         format_description => 'volume',
 684         description => "The drive's backing volume.",
 685     },
 686     media => {
 687         type => 'string',
 688         enum => [qw(cdrom disk)],
 689         description => "The drive's media type.",
 690         default => 'disk',
 691         optional => 1
 692     },
 693     cyls => {
 694         type => 'integer',
 695         description => "Force the drive's physical geometry to have a specific cylinder count.",
 696         optional => 1
 697     },
 698     heads => {
 699         type => 'integer',
 700         description => "Force the drive's physical geometry to have a specific head count.",
 701         optional => 1
 702     },
 703     secs => {
 704         type => 'integer',
 705         description => "Force the drive's physical geometry to have a specific sector count.",
 706         optional => 1
 707     },
 708     trans => {
 709         type => 'string',
 710         enum => [qw(none lba auto)],
 711         description => "Force disk geometry bios translation mode.",
 712         optional => 1,
 713     },
 714     snapshot => {
 715         type => 'boolean',
 716         description => "Whether the drive should be included when making snapshots.",
 717         optional => 1,
 718     },
 719     cache => {
 720         type => 'string',
 721         enum => [qw(none writethrough writeback unsafe directsync)],
 722         description => "The drive's cache mode",
 723         optional => 1,
 724     },
 725     format => {
 726         type => 'string',
 727         format_description => 'image format',
 728         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 729         description => "The drive's backing file's data format.",
 730         optional => 1,
 731     },
 732     size => {
 733         type => 'string',
 734         format => 'disk-size',
 735         format_description => 'DiskSize',
 736         description => "Disk size. This is purely informational and has no effect.",
 737         optional => 1,
 738     },
 739     backup => {
 740         type => 'boolean',
 741         description => "Whether the drive should be included when making backups.",
 742         optional => 1,
 743     },
 744     werror => {
 745         type => 'string',
 746         enum => [qw(enospc ignore report stop)],
 747         description => 'Write error action.',
 748         optional => 1,
 749     },
 750     aio => {
 751         type => 'string',
 752         enum => [qw(native threads)],
 753         description => 'AIO type to use.',
 754         optional => 1,
 755     },
 756     discard => {
 757         type => 'string',
 758         enum => [qw(ignore on)],
 759         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 760         optional => 1,
 761     },
 762     detect_zeroes => {
 763         type => 'boolean',
 764         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 765         optional => 1,
 766     },
 767     serial => {
 768         type => 'string',
 769         format => 'urlencoded',
 770         format_description => 'serial',
 771         maxLength => 20*3, # *3 since it's %xx url enoded
 772         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 773         optional => 1,
 774     }
 775 );
 776
 777 my %rerror_fmt = (
 778     rerror => {
 779         type => 'string',
 780         enum => [qw(ignore report stop)],
 781         description => 'Read error action.',
 782         optional => 1,
 783     },
 784 );
 785
 786 my %iothread_fmt = ( iothread => {
 787         type => 'boolean',
 788         description => "Whether to use iothreads for this drive",
 789         optional => 1,
 790 });
 791
 792 my %model_fmt = (
 793     model => {
 794         type => 'string',
 795         format => 'urlencoded',
 796         format_description => 'model',
 797         maxLength => 40*3, # *3 since it's %xx url enoded
 798         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 799         optional => 1,
 800     },
 801 );
 802
 803 my %queues_fmt = (
 804     queues => {
 805         type => 'integer',
 806         description => "Number of queues.",
 807         minimum => 2,
 808         optional => 1
 809     }
 810 );
 811
 812 my $add_throttle_desc = sub {
 813     my ($key, $type, $what, $unit, $longunit) = @_;
 814     $drivedesc_base{$key} = {
 815         type => $type,
 816         format_description => $unit,
 817         description => "Maximum $what speed in $longunit per second.",
 818         optional => 1,
 819     };
 820 };
 821 # throughput: (leaky bucket)
 822 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 823 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 824 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 825 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 826 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 827 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 828 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 829 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 830 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 831
 832 # pools: (pool of IO before throttling starts taking effect)
 833 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 836 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 837 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 838 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 839
 840 my $ide_fmt = {
 841     %drivedesc_base,
 842     %rerror_fmt,
 843     %model_fmt,
 844 };
 845
 846 my $idedesc = {
 847     optional => 1,
 848     type => 'string', format => $ide_fmt,
 849     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 850 };
 851 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 852
 853 my $scsi_fmt = {
 854     %drivedesc_base,
 855     %iothread_fmt,
 856     %queues_fmt,
 857 };
 858 my $scsidesc = {
 859     optional => 1,
 860     type => 'string', format => $scsi_fmt,
 861     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 862 };
 863 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 864
 865 my $sata_fmt = {
 866     %drivedesc_base,
 867     %rerror_fmt,
 868 };
 869 my $satadesc = {
 870     optional => 1,
 871     type => 'string', format => $sata_fmt,
 872     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 873 };
 874 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 875
 876 my $virtio_fmt = {
 877     %drivedesc_base,
 878     %iothread_fmt,
 879     %rerror_fmt,
 880 };
 881 my $virtiodesc = {
 882     optional => 1,
 883     type => 'string', format => $virtio_fmt,
 884     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 887
 888 my $alldrive_fmt = {
 889     %drivedesc_base,
 890     %rerror_fmt,
 891     %iothread_fmt,
 892     %model_fmt,
 893     %queues_fmt,
 894 };
 895
 896 my $usb_fmt = {
 897     host => {
 898         default_key => 1,
 899         type => 'string', format => 'pve-qm-usb-device',
 900         format_description => 'HOSTUSBDEVICE|spice',
 901         description => <<EODESCR,
 902 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 903
 904  'bus-port(.port)*' (decimal numbers) or
 905  'vendor_id:product_id' (hexadeciaml numbers) or
 906  'spice'
 907
 908 You can use the 'lsusb -t' command to list existing usb devices.
 909
 910 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 911
 912 The value 'spice' can be used to add a usb redirection devices for spice.
 913 EODESCR
 914     },
 915     usb3 => {
 916         optional => 1,
 917         type => 'boolean',
 918         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 919         default => 0,
 920     },
 921 };
 922
 923 my $usbdesc = {
 924     optional => 1,
 925     type => 'string', format => $usb_fmt,
 926     description => "Configure an USB device (n is 0 to 4).",
 927 };
 928 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 929
 930 # NOTE: the match-groups of this regex are used in parse_hostpci
 931 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 932 my $hostpci_fmt = {
 933     host => {
 934         default_key => 1,
 935         type => 'string',
 936         pattern => qr/$PCIRE(;$PCIRE)*/,
 937         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 938         description => <<EODESCR,
 939 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 940 of PCI virtual functions of the host. HOSTPCIID syntax is:
 941
 942 'bus:dev.func' (hexadecimal numbers)
 943
 944 You can us the 'lspci' command to list existing PCI devices.
 945 EODESCR
 946     },
 947     rombar => {
 948         type => 'boolean',
 949         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 950         optional => 1,
 951         default => 1,
 952     },
 953     pcie => {
 954         type => 'boolean',
 955         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 956         optional => 1,
 957         default => 0,
 958     },
 959     'x-vga' => {
 960         type => 'boolean',
 961         description =>  "Enable vfio-vga device support.",
 962         optional => 1,
 963         default => 0,
 964     },
 965 };
 966 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 967
 968 my $hostpcidesc = {
 969         optional => 1,
 970         type => 'string', format => 'pve-qm-hostpci',
 971         description => "Map host PCI devices into guest.",
 972         verbose_description =>  <<EODESCR,
 973 Map host PCI devices into guest.
 974
 975 NOTE: This option allows direct access to host hardware. So it is no longer
 976 possible to migrate such machines - use with special care.
 977
 978 CAUTION: Experimental! User reported problems with this option.
 979 EODESCR
 980 };
 981 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 982
 983 my $serialdesc = {
 984         optional => 1,
 985         type => 'string',
 986         pattern => '(/dev/.+|socket)',
 987         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 988         verbose_description =>  <<EODESCR,
 989 Create a serial device inside the VM (n is 0 to 3), and pass through a
 990 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 991 host side (use 'qm terminal' to open a terminal connection).
 992
 993 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 994
 995 CAUTION: Experimental! User reported problems with this option.
 996 EODESCR
 997 };
 998
 999 my $paralleldesc= {
1000         optional => 1,
1001         type => 'string',
1002         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1003         description =>  "Map host parallel devices (n is 0 to 2).",
1004         verbose_description =>  <<EODESCR,
1005 Map host parallel devices (n is 0 to 2).
1006
1007 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1008
1009 CAUTION: Experimental! User reported problems with this option.
1010 EODESCR
1011 };
1012
1013 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1014     $confdesc->{"parallel$i"} = $paralleldesc;
1015 }
1016
1017 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1018     $confdesc->{"serial$i"} = $serialdesc;
1019 }
1020
1021 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1022     $confdesc->{"hostpci$i"} = $hostpcidesc;
1023 }
1024
1025 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1026     $drivename_hash->{"ide$i"} = 1;
1027     $confdesc->{"ide$i"} = $idedesc;
1028 }
1029
1030 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1031     $drivename_hash->{"sata$i"} = 1;
1032     $confdesc->{"sata$i"} = $satadesc;
1033 }
1034
1035 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1036     $drivename_hash->{"scsi$i"} = 1;
1037     $confdesc->{"scsi$i"} = $scsidesc ;
1038 }
1039
1040 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1041     $drivename_hash->{"virtio$i"} = 1;
1042     $confdesc->{"virtio$i"} = $virtiodesc;
1043 }
1044
1045 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1046     $confdesc->{"usb$i"} = $usbdesc;
1047 }
1048
1049 my $unuseddesc = {
1050     optional => 1,
1051     type => 'string', format => 'pve-volume-id',
1052     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1053 };
1054
1055 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1056     $confdesc->{"unused$i"} = $unuseddesc;
1057 }
1058
1059 my $kvm_api_version = 0;
1060
1061 sub kvm_version {
1062
1063     return $kvm_api_version if $kvm_api_version;
1064
1065     my $fh = IO::File->new("</dev/kvm") ||
1066         return 0;
1067
1068     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1069         $kvm_api_version = $v;
1070     }
1071
1072     $fh->close();
1073
1074     return  $kvm_api_version;
1075 }
1076
1077 my $kvm_user_version;
1078
1079 sub kvm_user_version {
1080
1081     return $kvm_user_version if $kvm_user_version;
1082
1083     $kvm_user_version = 'unknown';
1084
1085     my $code = sub {
1086         my $line = shift;
1087         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1088             $kvm_user_version = $2;
1089         }
1090     };
1091
1092     eval { run_command("kvm -version", outfunc => $code); };
1093     warn $@ if $@;
1094
1095     return $kvm_user_version;
1096
1097 }
1098
1099 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1100
1101 sub valid_drive_names {
1102     # order is important - used to autoselect boot disk
1103     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1104             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1105             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1106             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1107 }
1108
1109 sub is_valid_drivename {
1110     my $dev = shift;
1111
1112     return defined($drivename_hash->{$dev});
1113 }
1114
1115 sub option_exists {
1116     my $key = shift;
1117     return defined($confdesc->{$key});
1118 }
1119
1120 sub nic_models {
1121     return $nic_model_list;
1122 }
1123
1124 sub os_list_description {
1125
1126     return {
1127         other => 'Other',
1128         wxp => 'Windows XP',
1129         w2k => 'Windows 2000',
1130         w2k3 =>, 'Windows 2003',
1131         w2k8 => 'Windows 2008',
1132         wvista => 'Windows Vista',
1133         win7 => 'Windows 7',
1134         win8 => 'Windows 8/2012',
1135         l24 => 'Linux 2.4',
1136         l26 => 'Linux 2.6',
1137     };
1138 }
1139
1140 my $cdrom_path;
1141
1142 sub get_cdrom_path {
1143
1144     return  $cdrom_path if $cdrom_path;
1145
1146     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1147     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1148     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1149 }
1150
1151 sub get_iso_path {
1152     my ($storecfg, $vmid, $cdrom) = @_;
1153
1154     if ($cdrom eq 'cdrom') {
1155         return get_cdrom_path();
1156     } elsif ($cdrom eq 'none') {
1157         return '';
1158     } elsif ($cdrom =~ m|^/|) {
1159         return $cdrom;
1160     } else {
1161         return PVE::Storage::path($storecfg, $cdrom);
1162     }
1163 }
1164
1165 # try to convert old style file names to volume IDs
1166 sub filename_to_volume_id {
1167     my ($vmid, $file, $media) = @_;
1168
1169     if (!($file eq 'none' || $file eq 'cdrom' ||
1170           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1171
1172         return undef if $file =~ m|/|;
1173
1174         if ($media && $media eq 'cdrom') {
1175             $file = "local:iso/$file";
1176         } else {
1177             $file = "local:$vmid/$file";
1178         }
1179     }
1180
1181     return $file;
1182 }
1183
1184 sub verify_media_type {
1185     my ($opt, $vtype, $media) = @_;
1186
1187     return if !$media;
1188
1189     my $etype;
1190     if ($media eq 'disk') {
1191         $etype = 'images';
1192     } elsif ($media eq 'cdrom') {
1193         $etype = 'iso';
1194     } else {
1195         die "internal error";
1196     }
1197
1198     return if ($vtype eq $etype);
1199
1200     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1201 }
1202
1203 sub cleanup_drive_path {
1204     my ($opt, $storecfg, $drive) = @_;
1205
1206     # try to convert filesystem paths to volume IDs
1207
1208     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1209         ($drive->{file} !~ m|^/dev/.+|) &&
1210         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1211         ($drive->{file} !~ m/^\d+$/)) {
1212         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1213         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1214         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1215         verify_media_type($opt, $vtype, $drive->{media});
1216         $drive->{file} = $volid;
1217     }
1218
1219     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1220 }
1221
1222 sub parse_hotplug_features {
1223     my ($data) = @_;
1224
1225     my $res = {};
1226
1227     return $res if $data eq '0';
1228
1229     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1230
1231     foreach my $feature (PVE::Tools::split_list($data)) {
1232         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1233             $res->{$1} = 1;
1234         } else {
1235             die "invalid hotplug feature '$feature'\n";
1236         }
1237     }
1238     return $res;
1239 }
1240
1241 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1242 sub pve_verify_hotplug_features {
1243     my ($value, $noerr) = @_;
1244
1245     return $value if parse_hotplug_features($value);
1246
1247     return undef if $noerr;
1248
1249     die "unable to parse hotplug option\n";
1250 }
1251
1252 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1253 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1254 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1255 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1256 #        [,iothread=on][,serial=serial][,model=model]
1257
1258 sub parse_drive {
1259     my ($key, $data) = @_;
1260
1261     my ($interface, $index);
1262
1263     if ($key =~ m/^([^\d]+)(\d+)$/) {
1264         $interface = $1;
1265         $index = $2;
1266     } else {
1267         return undef;
1268     }
1269
1270     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1271                                      : $confdesc->{$key}->{format};
1272     if (!$desc) {
1273         warn "invalid drive key: $key\n";
1274         return undef;
1275     }
1276     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1277     return undef if !$res;
1278     $res->{interface} = $interface;
1279     $res->{index} = $index;
1280
1281     my $error = 0;
1282     foreach my $opt (qw(bps bps_rd bps_wr)) {
1283         if (my $bps = defined(delete $res->{$opt})) {
1284             if (defined($res->{"m$opt"})) {
1285                 warn "both $opt and m$opt specified\n";
1286                 ++$error;
1287                 next;
1288             }
1289             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1290         }
1291     }
1292     return undef if $error;
1293
1294     return undef if $res->{mbps_rd} && $res->{mbps};
1295     return undef if $res->{mbps_wr} && $res->{mbps};
1296     return undef if $res->{iops_rd} && $res->{iops};
1297     return undef if $res->{iops_wr} && $res->{iops};
1298
1299     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1300         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1301         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1302         return undef if $res->{interface} eq 'virtio';
1303     }
1304
1305     if (my $size = $res->{size}) {
1306         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1307     }
1308
1309     return $res;
1310 }
1311
1312 sub print_drive {
1313     my ($vmid, $drive) = @_;
1314     my $data = { %$drive };
1315     delete $data->{$_} for qw(index interface);
1316     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1317 }
1318
1319 sub scsi_inquiry {
1320     my($fh, $noerr) = @_;
1321
1322     my $SG_IO = 0x2285;
1323     my $SG_GET_VERSION_NUM = 0x2282;
1324
1325     my $versionbuf = "\x00" x 8;
1326     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1327     if (!$ret) {
1328         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1329         return undef;
1330     }
1331     my $version = unpack("I", $versionbuf);
1332     if ($version < 30000) {
1333         die "scsi generic interface too old\n"  if !$noerr;
1334         return undef;
1335     }
1336
1337     my $buf = "\x00" x 36;
1338     my $sensebuf = "\x00" x 8;
1339     my $cmd = pack("C x3 C x1", 0x12, 36);
1340
1341     # see /usr/include/scsi/sg.h
1342     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1343
1344     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1345                       length($sensebuf), 0, length($buf), $buf,
1346                       $cmd, $sensebuf, 6000);
1347
1348     $ret = ioctl($fh, $SG_IO, $packet);
1349     if (!$ret) {
1350         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1351         return undef;
1352     }
1353
1354     my @res = unpack($sg_io_hdr_t, $packet);
1355     if ($res[17] || $res[18]) {
1356         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1357         return undef;
1358     }
1359
1360     my $res = {};
1361     (my $byte0, my $byte1, $res->{vendor},
1362      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1363
1364     $res->{removable} = $byte1 & 128 ? 1 : 0;
1365     $res->{type} = $byte0 & 31;
1366
1367     return $res;
1368 }
1369
1370 sub path_is_scsi {
1371     my ($path) = @_;
1372
1373     my $fh = IO::File->new("+<$path") || return undef;
1374     my $res = scsi_inquiry($fh, 1);
1375     close($fh);
1376
1377     return $res;
1378 }
1379
1380 sub machine_type_is_q35 {
1381     my ($conf) = @_;
1382
1383     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1384 }
1385
1386 sub print_tabletdevice_full {
1387     my ($conf) = @_;
1388
1389     my $q35 = machine_type_is_q35($conf);
1390
1391     # we use uhci for old VMs because tablet driver was buggy in older qemu
1392     my $usbbus = $q35 ? "ehci" : "uhci";
1393
1394     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1395 }
1396
1397 sub print_drivedevice_full {
1398     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1399
1400     my $device = '';
1401     my $maxdev = 0;
1402
1403     if ($drive->{interface} eq 'virtio') {
1404         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1405         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1406         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1407     } elsif ($drive->{interface} eq 'scsi') {
1408
1409         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1410         my $unit = $drive->{index} % $maxdev;
1411         my $devicetype = 'hd';
1412         my $path = '';
1413         if (drive_is_cdrom($drive)) {
1414             $devicetype = 'cd';
1415         } else {
1416             if ($drive->{file} =~ m|^/|) {
1417                 $path = $drive->{file};
1418                 if (my $info = path_is_scsi($path)) {
1419                     if ($info->{type} == 0) {
1420                         $devicetype = 'block';
1421                     } elsif ($info->{type} == 1) { # tape
1422                         $devicetype = 'generic';
1423                     }
1424                 }
1425             } else {
1426                  $path = PVE::Storage::path($storecfg, $drive->{file});
1427             }
1428
1429             if($path =~ m/^iscsi\:\/\//){
1430                 $devicetype = 'generic';
1431             }
1432         }
1433
1434         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1435             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1436         } else {
1437             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1438         }
1439
1440     } elsif ($drive->{interface} eq 'ide'){
1441         $maxdev = 2;
1442         my $controller = int($drive->{index} / $maxdev);
1443         my $unit = $drive->{index} % $maxdev;
1444         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1445
1446         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1447         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1448             $model = URI::Escape::uri_unescape($model);
1449             $device .= ",model=$model";
1450         }
1451     } elsif ($drive->{interface} eq 'sata'){
1452         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1453         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1454         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1455     } elsif ($drive->{interface} eq 'usb') {
1456         die "implement me";
1457         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1458     } else {
1459         die "unsupported interface type";
1460     }
1461
1462     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1463
1464     return $device;
1465 }
1466
1467 sub get_initiator_name {
1468     my $initiator;
1469
1470     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1471     while (defined(my $line = <$fh>)) {
1472         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1473         $initiator = $1;
1474         last;
1475     }
1476     $fh->close();
1477
1478     return $initiator;
1479 }
1480
1481 sub print_drive_full {
1482     my ($storecfg, $vmid, $drive) = @_;
1483
1484     my $path;
1485     my $volid = $drive->{file};
1486     my $format;
1487
1488     if (drive_is_cdrom($drive)) {
1489         $path = get_iso_path($storecfg, $vmid, $volid);
1490     } else {
1491         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1492         if ($storeid) {
1493             $path = PVE::Storage::path($storecfg, $volid);
1494             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1495             $format = qemu_img_format($scfg, $volname);
1496         } else {
1497             $path = $volid;
1498             $format = "raw";
1499         }
1500    }
1501
1502     my $opts = '';
1503     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1504     foreach my $o (@qemu_drive_options) {
1505         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1506     }
1507     if (my $serial = $drive->{serial}) {
1508         $serial = URI::Escape::uri_unescape($serial);
1509         $opts .= ",serial=$serial";
1510     }
1511
1512     $opts .= ",format=$format" if $format && !$drive->{format};
1513
1514     foreach my $o (qw(bps bps_rd bps_wr)) {
1515         my $v = $drive->{"m$o"};
1516         $opts .= ",$o=" . int($v*1024*1024) if $v;
1517     }
1518
1519     my $cache_direct = 0;
1520
1521     if (my $cache = $drive->{cache}) {
1522         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1523     } elsif (!drive_is_cdrom($drive)) {
1524         $opts .= ",cache=none";
1525         $cache_direct = 1;
1526     }
1527
1528     # aio native works only with O_DIRECT
1529     if (!$drive->{aio}) {
1530         if($cache_direct) {
1531             $opts .= ",aio=native";
1532         } else {
1533             $opts .= ",aio=threads";
1534         }
1535     }
1536
1537     if (!drive_is_cdrom($drive)) {
1538         my $detectzeroes;
1539         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1540             $detectzeroes = 'off';
1541         } elsif ($drive->{discard}) {
1542             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1543         } else {
1544             # This used to be our default with discard not being specified:
1545             $detectzeroes = 'on';
1546         }
1547         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1548     }
1549
1550     my $pathinfo = $path ? "file=$path," : '';
1551
1552     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1553 }
1554
1555 sub print_netdevice_full {
1556     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1557
1558     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1559
1560     my $device = $net->{model};
1561     if ($net->{model} eq 'virtio') {
1562          $device = 'virtio-net-pci';
1563      };
1564
1565     my $pciaddr = print_pci_addr("$netid", $bridges);
1566     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1567     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1568         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1569         my $vectors = $net->{queues} * 2 + 2;
1570         $tmpstr .= ",vectors=$vectors,mq=on";
1571     }
1572     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1573
1574     if ($use_old_bios_files) {
1575         my $romfile;
1576         if ($device eq 'virtio-net-pci') {
1577             $romfile = 'pxe-virtio.rom';
1578         } elsif ($device eq 'e1000') {
1579             $romfile = 'pxe-e1000.rom';
1580         } elsif ($device eq 'ne2k') {
1581             $romfile = 'pxe-ne2k_pci.rom';
1582         } elsif ($device eq 'pcnet') {
1583             $romfile = 'pxe-pcnet.rom';
1584         } elsif ($device eq 'rtl8139') {
1585             $romfile = 'pxe-rtl8139.rom';
1586         }
1587         $tmpstr .= ",romfile=$romfile" if $romfile;
1588     }
1589
1590     return $tmpstr;
1591 }
1592
1593 sub print_netdev_full {
1594     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1595
1596     my $i = '';
1597     if ($netid =~ m/^net(\d+)$/) {
1598         $i = int($1);
1599     }
1600
1601     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1602
1603     my $ifname = "tap${vmid}i$i";
1604
1605     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1606     die "interface name '$ifname' is too long (max 15 character)\n"
1607         if length($ifname) >= 16;
1608
1609     my $vhostparam = '';
1610     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1611
1612     my $vmname = $conf->{name} || "vm$vmid";
1613
1614     my $netdev = "";
1615     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1616
1617     if ($net->{bridge}) {
1618         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1619     } else {
1620         $netdev = "type=user,id=$netid,hostname=$vmname";
1621     }
1622
1623     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1624
1625     return $netdev;
1626 }
1627
1628 sub drive_is_cdrom {
1629     my ($drive) = @_;
1630
1631     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1632
1633 }
1634
1635 sub parse_number_sets {
1636     my ($set) = @_;
1637     my $res = [];
1638     foreach my $part (split(/;/, $set)) {
1639         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1640             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1641             push @$res, [ $1, $2 ];
1642         } else {
1643             die "invalid range: $part\n";
1644         }
1645     }
1646     return $res;
1647 }
1648
1649 sub parse_numa {
1650     my ($data) = @_;
1651
1652     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1653     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1654     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1655     return $res;
1656 }
1657
1658 sub parse_hostpci {
1659     my ($value) = @_;
1660
1661     return undef if !$value;
1662
1663     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1664
1665     my @idlist = split(/;/, $res->{host});
1666     delete $res->{host};
1667     foreach my $id (@idlist) {
1668         if ($id =~ /^$PCIRE$/) {
1669             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1670         } else {
1671             # should have been caught by parse_property_string already
1672             die "failed to parse PCI id: $id\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1679 sub parse_net {
1680     my ($data) = @_;
1681
1682     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1683     if ($@) {
1684         warn $@;
1685         return undef;
1686     }
1687     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1688     return $res;
1689 }
1690
1691 sub print_net {
1692     my $net = shift;
1693
1694     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1695 }
1696
1697 sub add_random_macs {
1698     my ($settings) = @_;
1699
1700     foreach my $opt (keys %$settings) {
1701         next if $opt !~ m/^net(\d+)$/;
1702         my $net = parse_net($settings->{$opt});
1703         next if !$net;
1704         $settings->{$opt} = print_net($net);
1705     }
1706 }
1707
1708 sub vm_is_volid_owner {
1709     my ($storecfg, $vmid, $volid) = @_;
1710
1711     if ($volid !~  m|^/|) {
1712         my ($path, $owner);
1713         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1714         if ($owner && ($owner == $vmid)) {
1715             return 1;
1716         }
1717     }
1718
1719     return undef;
1720 }
1721
1722 sub split_flagged_list {
1723     my $text = shift || '';
1724     $text =~ s/[,;]/ /g;
1725     $text =~ s/^\s+//;
1726     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1727 }
1728
1729 sub join_flagged_list {
1730     my ($how, $lst) = @_;
1731     join $how, map { $lst->{$_} . $_ } keys %$lst;
1732 }
1733
1734 sub vmconfig_delete_pending_option {
1735     my ($conf, $key, $force) = @_;
1736
1737     delete $conf->{pending}->{$key};
1738     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1739     $pending_delete_hash->{$key} = $force ? '!' : '';
1740     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1741 }
1742
1743 sub vmconfig_undelete_pending_option {
1744     my ($conf, $key) = @_;
1745
1746     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1747     delete $pending_delete_hash->{$key};
1748
1749     if (%$pending_delete_hash) {
1750         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1751     } else {
1752         delete $conf->{pending}->{delete};
1753     }
1754 }
1755
1756 sub vmconfig_register_unused_drive {
1757     my ($storecfg, $vmid, $conf, $drive) = @_;
1758
1759     if (!drive_is_cdrom($drive)) {
1760         my $volid = $drive->{file};
1761         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1762             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1763         }
1764     }
1765 }
1766
1767 sub vmconfig_cleanup_pending {
1768     my ($conf) = @_;
1769
1770     # remove pending changes when nothing changed
1771     my $changes;
1772     foreach my $opt (keys %{$conf->{pending}}) {
1773         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1774             $changes = 1;
1775             delete $conf->{pending}->{$opt};
1776         }
1777     }
1778
1779     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1780     my $pending_delete_hash = {};
1781     while (my ($opt, $force) = each %$current_delete_hash) {
1782         if (defined($conf->{$opt})) {
1783             $pending_delete_hash->{$opt} = $force;
1784         } else {
1785             $changes = 1;
1786         }
1787     }
1788
1789     if (%$pending_delete_hash) {
1790         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1791     } else {
1792         delete $conf->{pending}->{delete};
1793     }
1794
1795     return $changes;
1796 }
1797
1798 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1799 my $smbios1_fmt = {
1800     uuid => {
1801         type => 'string',
1802         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1803         format_description => 'UUID',
1804         description => "Set SMBIOS1 UUID.",
1805         optional => 1,
1806     },
1807     version => {
1808         type => 'string',
1809         pattern => '\S+',
1810         format_description => 'string',
1811         description => "Set SMBIOS1 version.",
1812         optional => 1,
1813     },
1814     serial => {
1815         type => 'string',
1816         pattern => '\S+',
1817         format_description => 'string',
1818         description => "Set SMBIOS1 serial number.",
1819         optional => 1,
1820     },
1821     manufacturer => {
1822         type => 'string',
1823         pattern => '\S+',
1824         format_description => 'string',
1825         description => "Set SMBIOS1 manufacturer.",
1826         optional => 1,
1827     },
1828     product => {
1829         type => 'string',
1830         pattern => '\S+',
1831         format_description => 'string',
1832         description => "Set SMBIOS1 product ID.",
1833         optional => 1,
1834     },
1835     sku => {
1836         type => 'string',
1837         pattern => '\S+',
1838         format_description => 'string',
1839         description => "Set SMBIOS1 SKU string.",
1840         optional => 1,
1841     },
1842     family => {
1843         type => 'string',
1844         pattern => '\S+',
1845         format_description => 'string',
1846         description => "Set SMBIOS1 family string.",
1847         optional => 1,
1848     },
1849 };
1850
1851 sub parse_smbios1 {
1852     my ($data) = @_;
1853
1854     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1855     warn $@ if $@;
1856     return $res;
1857 }
1858
1859 sub print_smbios1 {
1860     my ($smbios1) = @_;
1861     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1862 }
1863
1864 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1865
1866 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1867 sub verify_bootdisk {
1868     my ($value, $noerr) = @_;
1869
1870     return $value if is_valid_drivename($value);
1871
1872     return undef if $noerr;
1873
1874     die "invalid boot disk '$value'\n";
1875 }
1876
1877 sub parse_watchdog {
1878     my ($value) = @_;
1879
1880     return undef if !$value;
1881
1882     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1883     warn $@ if $@;
1884     return $res;
1885 }
1886
1887 sub parse_usb_device {
1888     my ($value) = @_;
1889
1890     return undef if !$value;
1891
1892     my $res = {};
1893     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1894         $res->{vendorid} = $2;
1895         $res->{productid} = $4;
1896     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1897         $res->{hostbus} = $1;
1898         $res->{hostport} = $2;
1899     } elsif ($value =~ m/^spice$/i) {
1900         $res->{spice} = 1;
1901     } else {
1902         return undef;
1903     }
1904
1905     return $res;
1906 }
1907
1908 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1909 sub verify_usb_device {
1910     my ($value, $noerr) = @_;
1911
1912     return $value if parse_usb_device($value);
1913
1914     return undef if $noerr;
1915
1916     die "unable to parse usb device\n";
1917 }
1918
1919 # add JSON properties for create and set function
1920 sub json_config_properties {
1921     my $prop = shift;
1922
1923     foreach my $opt (keys %$confdesc) {
1924         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1925         $prop->{$opt} = $confdesc->{$opt};
1926     }
1927
1928     return $prop;
1929 }
1930
1931 sub check_type {
1932     my ($key, $value) = @_;
1933
1934     die "unknown setting '$key'\n" if !$confdesc->{$key};
1935
1936     my $type = $confdesc->{$key}->{type};
1937
1938     if (!defined($value)) {
1939         die "got undefined value\n";
1940     }
1941
1942     if ($value =~ m/[\n\r]/) {
1943         die "property contains a line feed\n";
1944     }
1945
1946     if ($type eq 'boolean') {
1947         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1948         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1949         die "type check ('boolean') failed - got '$value'\n";
1950     } elsif ($type eq 'integer') {
1951         return int($1) if $value =~ m/^(\d+)$/;
1952         die "type check ('integer') failed - got '$value'\n";
1953     } elsif ($type eq 'number') {
1954         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1955         die "type check ('number') failed - got '$value'\n";
1956     } elsif ($type eq 'string') {
1957         if (my $fmt = $confdesc->{$key}->{format}) {
1958             if ($fmt eq 'pve-qm-drive') {
1959                 # special case - we need to pass $key to parse_drive()
1960                 my $drive = parse_drive($key, $value);
1961                 return $value if $drive;
1962                 die "unable to parse drive options\n";
1963             }
1964             PVE::JSONSchema::check_format($fmt, $value);
1965             return $value;
1966         }
1967         $value =~ s/^\"(.*)\"$/$1/;
1968         return $value;
1969     } else {
1970         die "internal error"
1971     }
1972 }
1973
1974 sub check_iommu_support{
1975     #fixme : need to check IOMMU support
1976     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1977
1978     my $iommu=1;
1979     return $iommu;
1980
1981 }
1982
1983 sub touch_config {
1984     my ($vmid) = @_;
1985
1986     my $conf = PVE::QemuConfig->config_file($vmid);
1987     utime undef, undef, $conf;
1988 }
1989
1990 sub destroy_vm {
1991     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1992
1993     my $conffile = PVE::QemuConfig->config_file($vmid);
1994
1995     my $conf = PVE::QemuConfig->load_config($vmid);
1996
1997     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1998
1999     # only remove disks owned by this VM
2000     foreach_drive($conf, sub {
2001         my ($ds, $drive) = @_;
2002
2003         return if drive_is_cdrom($drive);
2004
2005         my $volid = $drive->{file};
2006
2007         return if !$volid || $volid =~ m|^/|;
2008
2009         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2010         return if !$path || !$owner || ($owner != $vmid);
2011
2012         PVE::Storage::vdisk_free($storecfg, $volid);
2013     });
2014
2015     if ($keep_empty_config) {
2016         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2017     } else {
2018         unlink $conffile;
2019     }
2020
2021     # also remove unused disk
2022     eval {
2023         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2024
2025         eval {
2026             PVE::Storage::foreach_volid($dl, sub {
2027                 my ($volid, $sid, $volname, $d) = @_;
2028                 PVE::Storage::vdisk_free($storecfg, $volid);
2029             });
2030         };
2031         warn $@ if $@;
2032
2033     };
2034     warn $@ if $@;
2035 }
2036
2037 sub parse_vm_config {
2038     my ($filename, $raw) = @_;
2039
2040     return undef if !defined($raw);
2041
2042     my $res = {
2043         digest => Digest::SHA::sha1_hex($raw),
2044         snapshots => {},
2045         pending => {},
2046     };
2047
2048     $filename =~ m|/qemu-server/(\d+)\.conf$|
2049         || die "got strange filename '$filename'";
2050
2051     my $vmid = $1;
2052
2053     my $conf = $res;
2054     my $descr;
2055     my $section = '';
2056
2057     my @lines = split(/\n/, $raw);
2058     foreach my $line (@lines) {
2059         next if $line =~ m/^\s*$/;
2060
2061         if ($line =~ m/^\[PENDING\]\s*$/i) {
2062             $section = 'pending';
2063             if (defined($descr)) {
2064                 $descr =~ s/\s+$//;
2065                 $conf->{description} = $descr;
2066             }
2067             $descr = undef;
2068             $conf = $res->{$section} = {};
2069             next;
2070
2071         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2072             $section = $1;
2073             if (defined($descr)) {
2074                 $descr =~ s/\s+$//;
2075                 $conf->{description} = $descr;
2076             }
2077             $descr = undef;
2078             $conf = $res->{snapshots}->{$section} = {};
2079             next;
2080         }
2081
2082         if ($line =~ m/^\#(.*)\s*$/) {
2083             $descr = '' if !defined($descr);
2084             $descr .= PVE::Tools::decode_text($1) . "\n";
2085             next;
2086         }
2087
2088         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2089             $descr = '' if !defined($descr);
2090             $descr .= PVE::Tools::decode_text($2);
2091         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2092             $conf->{snapstate} = $1;
2093         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2094             my $key = $1;
2095             my $value = $2;
2096             $conf->{$key} = $value;
2097         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2098             my $value = $1;
2099             if ($section eq 'pending') {
2100                 $conf->{delete} = $value; # we parse this later
2101             } else {
2102                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2103             }
2104         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2105             my $key = $1;
2106             my $value = $2;
2107             eval { $value = check_type($key, $value); };
2108             if ($@) {
2109                 warn "vm $vmid - unable to parse value of '$key' - $@";
2110             } else {
2111                 my $fmt = $confdesc->{$key}->{format};
2112                 if ($fmt && $fmt eq 'pve-qm-drive') {
2113                     my $v = parse_drive($key, $value);
2114                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2115                         $v->{file} = $volid;
2116                         $value = print_drive($vmid, $v);
2117                     } else {
2118                         warn "vm $vmid - unable to parse value of '$key'\n";
2119                         next;
2120                     }
2121                 }
2122
2123                 if ($key eq 'cdrom') {
2124                     $conf->{ide2} = $value;
2125                 } else {
2126                     $conf->{$key} = $value;
2127                 }
2128             }
2129         }
2130     }
2131
2132     if (defined($descr)) {
2133         $descr =~ s/\s+$//;
2134         $conf->{description} = $descr;
2135     }
2136     delete $res->{snapstate}; # just to be sure
2137
2138     return $res;
2139 }
2140
2141 sub write_vm_config {
2142     my ($filename, $conf) = @_;
2143
2144     delete $conf->{snapstate}; # just to be sure
2145
2146     if ($conf->{cdrom}) {
2147         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2148         $conf->{ide2} = $conf->{cdrom};
2149         delete $conf->{cdrom};
2150     }
2151
2152     # we do not use 'smp' any longer
2153     if ($conf->{sockets}) {
2154         delete $conf->{smp};
2155     } elsif ($conf->{smp}) {
2156         $conf->{sockets} = $conf->{smp};
2157         delete $conf->{cores};
2158         delete $conf->{smp};
2159     }
2160
2161     my $used_volids = {};
2162
2163     my $cleanup_config = sub {
2164         my ($cref, $pending, $snapname) = @_;
2165
2166         foreach my $key (keys %$cref) {
2167             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2168                 $key eq 'snapstate' || $key eq 'pending';
2169             my $value = $cref->{$key};
2170             if ($key eq 'delete') {
2171                 die "propertry 'delete' is only allowed in [PENDING]\n"
2172                     if !$pending;
2173                 # fixme: check syntax?
2174                 next;
2175             }
2176             eval { $value = check_type($key, $value); };
2177             die "unable to parse value of '$key' - $@" if $@;
2178
2179             $cref->{$key} = $value;
2180
2181             if (!$snapname && is_valid_drivename($key)) {
2182                 my $drive = parse_drive($key, $value);
2183                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2184             }
2185         }
2186     };
2187
2188     &$cleanup_config($conf);
2189
2190     &$cleanup_config($conf->{pending}, 1);
2191
2192     foreach my $snapname (keys %{$conf->{snapshots}}) {
2193         die "internal error" if $snapname eq 'pending';
2194         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2195     }
2196
2197     # remove 'unusedX' settings if we re-add a volume
2198     foreach my $key (keys %$conf) {
2199         my $value = $conf->{$key};
2200         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2201             delete $conf->{$key};
2202         }
2203     }
2204
2205     my $generate_raw_config = sub {
2206         my ($conf, $pending) = @_;
2207
2208         my $raw = '';
2209
2210         # add description as comment to top of file
2211         if (defined(my $descr = $conf->{description})) {
2212             if ($descr) {
2213                 foreach my $cl (split(/\n/, $descr)) {
2214                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2215                 }
2216             } else {
2217                 $raw .= "#\n" if $pending;
2218             }
2219         }
2220
2221         foreach my $key (sort keys %$conf) {
2222             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2223             $raw .= "$key: $conf->{$key}\n";
2224         }
2225         return $raw;
2226     };
2227
2228     my $raw = &$generate_raw_config($conf);
2229
2230     if (scalar(keys %{$conf->{pending}})){
2231         $raw .= "\n[PENDING]\n";
2232         $raw .= &$generate_raw_config($conf->{pending}, 1);
2233     }
2234
2235     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2236         $raw .= "\n[$snapname]\n";
2237         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2238     }
2239
2240     return $raw;
2241 }
2242
2243 sub load_defaults {
2244
2245     my $res = {};
2246
2247     # we use static defaults from our JSON schema configuration
2248     foreach my $key (keys %$confdesc) {
2249         if (defined(my $default = $confdesc->{$key}->{default})) {
2250             $res->{$key} = $default;
2251         }
2252     }
2253
2254     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2255     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2256
2257     return $res;
2258 }
2259
2260 sub config_list {
2261     my $vmlist = PVE::Cluster::get_vmlist();
2262     my $res = {};
2263     return $res if !$vmlist || !$vmlist->{ids};
2264     my $ids = $vmlist->{ids};
2265
2266     foreach my $vmid (keys %$ids) {
2267         my $d = $ids->{$vmid};
2268         next if !$d->{node} || $d->{node} ne $nodename;
2269         next if !$d->{type} || $d->{type} ne 'qemu';
2270         $res->{$vmid}->{exists} = 1;
2271     }
2272     return $res;
2273 }
2274
2275 # test if VM uses local resources (to prevent migration)
2276 sub check_local_resources {
2277     my ($conf, $noerr) = @_;
2278
2279     my $loc_res = 0;
2280
2281     $loc_res = 1 if $conf->{hostusb}; # old syntax
2282     $loc_res = 1 if $conf->{hostpci}; # old syntax
2283
2284     foreach my $k (keys %$conf) {
2285         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2286         # sockets are safe: they will recreated be on the target side post-migrate
2287         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2288         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2289     }
2290
2291     die "VM uses local resources\n" if $loc_res && !$noerr;
2292
2293     return $loc_res;
2294 }
2295
2296 # check if used storages are available on all nodes (use by migrate)
2297 sub check_storage_availability {
2298     my ($storecfg, $conf, $node) = @_;
2299
2300     foreach_drive($conf, sub {
2301         my ($ds, $drive) = @_;
2302
2303         my $volid = $drive->{file};
2304         return if !$volid;
2305
2306         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2307         return if !$sid;
2308
2309         # check if storage is available on both nodes
2310         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2311         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2312    });
2313 }
2314
2315 # list nodes where all VM images are available (used by has_feature API)
2316 sub shared_nodes {
2317     my ($conf, $storecfg) = @_;
2318
2319     my $nodelist = PVE::Cluster::get_nodelist();
2320     my $nodehash = { map { $_ => 1 } @$nodelist };
2321     my $nodename = PVE::INotify::nodename();
2322
2323     foreach_drive($conf, sub {
2324         my ($ds, $drive) = @_;
2325
2326         my $volid = $drive->{file};
2327         return if !$volid;
2328
2329         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2330         if ($storeid) {
2331             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2332             if ($scfg->{disable}) {
2333                 $nodehash = {};
2334             } elsif (my $avail = $scfg->{nodes}) {
2335                 foreach my $node (keys %$nodehash) {
2336                     delete $nodehash->{$node} if !$avail->{$node};
2337                 }
2338             } elsif (!$scfg->{shared}) {
2339                 foreach my $node (keys %$nodehash) {
2340                     delete $nodehash->{$node} if $node ne $nodename
2341                 }
2342             }
2343         }
2344     });
2345
2346     return $nodehash
2347 }
2348
2349 sub check_cmdline {
2350     my ($pidfile, $pid) = @_;
2351
2352     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2353     if (defined($fh)) {
2354         my $line = <$fh>;
2355         $fh->close;
2356         return undef if !$line;
2357         my @param = split(/\0/, $line);
2358
2359         my $cmd = $param[0];
2360         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2361
2362         for (my $i = 0; $i < scalar (@param); $i++) {
2363             my $p = $param[$i];
2364             next if !$p;
2365             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2366                 my $p = $param[$i+1];
2367                 return 1 if $p && ($p eq $pidfile);
2368                 return undef;
2369             }
2370         }
2371     }
2372     return undef;
2373 }
2374
2375 sub check_running {
2376     my ($vmid, $nocheck, $node) = @_;
2377
2378     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2379
2380     die "unable to find configuration file for VM $vmid - no such machine\n"
2381         if !$nocheck && ! -f $filename;
2382
2383     my $pidfile = pidfile_name($vmid);
2384
2385     if (my $fd = IO::File->new("<$pidfile")) {
2386         my $st = stat($fd);
2387         my $line = <$fd>;
2388         close($fd);
2389
2390         my $mtime = $st->mtime;
2391         if ($mtime > time()) {
2392             warn "file '$filename' modified in future\n";
2393         }
2394
2395         if ($line =~ m/^(\d+)$/) {
2396             my $pid = $1;
2397             if (check_cmdline($pidfile, $pid)) {
2398                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2399                     return $pid;
2400                 }
2401             }
2402         }
2403     }
2404
2405     return undef;
2406 }
2407
2408 sub vzlist {
2409
2410     my $vzlist = config_list();
2411
2412     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2413
2414     while (defined(my $de = $fd->read)) {
2415         next if $de !~ m/^(\d+)\.pid$/;
2416         my $vmid = $1;
2417         next if !defined($vzlist->{$vmid});
2418         if (my $pid = check_running($vmid)) {
2419             $vzlist->{$vmid}->{pid} = $pid;
2420         }
2421     }
2422
2423     return $vzlist;
2424 }
2425
2426 sub disksize {
2427     my ($storecfg, $conf) = @_;
2428
2429     my $bootdisk = $conf->{bootdisk};
2430     return undef if !$bootdisk;
2431     return undef if !is_valid_drivename($bootdisk);
2432
2433     return undef if !$conf->{$bootdisk};
2434
2435     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2436     return undef if !defined($drive);
2437
2438     return undef if drive_is_cdrom($drive);
2439
2440     my $volid = $drive->{file};
2441     return undef if !$volid;
2442
2443     return $drive->{size};
2444 }
2445
2446 my $last_proc_pid_stat;
2447
2448 # get VM status information
2449 # This must be fast and should not block ($full == false)
2450 # We only query KVM using QMP if $full == true (this can be slow)
2451 sub vmstatus {
2452     my ($opt_vmid, $full) = @_;
2453
2454     my $res = {};
2455
2456     my $storecfg = PVE::Storage::config();
2457
2458     my $list = vzlist();
2459     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2460
2461     my $cpucount = $cpuinfo->{cpus} || 1;
2462
2463     foreach my $vmid (keys %$list) {
2464         next if $opt_vmid && ($vmid ne $opt_vmid);
2465
2466         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2467         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2468
2469         my $d = {};
2470         $d->{pid} = $list->{$vmid}->{pid};
2471
2472         # fixme: better status?
2473         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2474
2475         my $size = disksize($storecfg, $conf);
2476         if (defined($size)) {
2477             $d->{disk} = 0; # no info available
2478             $d->{maxdisk} = $size;
2479         } else {
2480             $d->{disk} = 0;
2481             $d->{maxdisk} = 0;
2482         }
2483
2484         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2485         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2486         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2487
2488         $d->{name} = $conf->{name} || "VM $vmid";
2489         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2490
2491         if ($conf->{balloon}) {
2492             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2493             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2494         }
2495
2496         $d->{uptime} = 0;
2497         $d->{cpu} = 0;
2498         $d->{mem} = 0;
2499
2500         $d->{netout} = 0;
2501         $d->{netin} = 0;
2502
2503         $d->{diskread} = 0;
2504         $d->{diskwrite} = 0;
2505
2506         $d->{template} = PVE::QemuConfig->is_template($conf);
2507
2508         $res->{$vmid} = $d;
2509     }
2510
2511     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2512     foreach my $dev (keys %$netdev) {
2513         next if $dev !~ m/^tap([1-9]\d*)i/;
2514         my $vmid = $1;
2515         my $d = $res->{$vmid};
2516         next if !$d;
2517
2518         $d->{netout} += $netdev->{$dev}->{receive};
2519         $d->{netin} += $netdev->{$dev}->{transmit};
2520
2521         if ($full) {
2522             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2523             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2524         }
2525
2526     }
2527
2528     my $ctime = gettimeofday;
2529
2530     foreach my $vmid (keys %$list) {
2531
2532         my $d = $res->{$vmid};
2533         my $pid = $d->{pid};
2534         next if !$pid;
2535
2536         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2537         next if !$pstat; # not running
2538
2539         my $used = $pstat->{utime} + $pstat->{stime};
2540
2541         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2542
2543         if ($pstat->{vsize}) {
2544             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2545         }
2546
2547         my $old = $last_proc_pid_stat->{$pid};
2548         if (!$old) {
2549             $last_proc_pid_stat->{$pid} = {
2550                 time => $ctime,
2551                 used => $used,
2552                 cpu => 0,
2553             };
2554             next;
2555         }
2556
2557         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2558
2559         if ($dtime > 1000) {
2560             my $dutime = $used -  $old->{used};
2561
2562             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2563             $last_proc_pid_stat->{$pid} = {
2564                 time => $ctime,
2565                 used => $used,
2566                 cpu => $d->{cpu},
2567             };
2568         } else {
2569             $d->{cpu} = $old->{cpu};
2570         }
2571     }
2572
2573     return $res if !$full;
2574
2575     my $qmpclient = PVE::QMPClient->new();
2576
2577     my $ballooncb = sub {
2578         my ($vmid, $resp) = @_;
2579
2580         my $info = $resp->{'return'};
2581         return if !$info->{max_mem};
2582
2583         my $d = $res->{$vmid};
2584
2585         # use memory assigned to VM
2586         $d->{maxmem} = $info->{max_mem};
2587         $d->{balloon} = $info->{actual};
2588
2589         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2590             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2591             $d->{freemem} = $info->{free_mem};
2592         }
2593
2594         $d->{ballooninfo} = $info;
2595     };
2596
2597     my $blockstatscb = sub {
2598         my ($vmid, $resp) = @_;
2599         my $data = $resp->{'return'} || [];
2600         my $totalrdbytes = 0;
2601         my $totalwrbytes = 0;
2602
2603         for my $blockstat (@$data) {
2604             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2605             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2606
2607             $blockstat->{device} =~ s/drive-//;
2608             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2609         }
2610         $res->{$vmid}->{diskread} = $totalrdbytes;
2611         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2612     };
2613
2614     my $statuscb = sub {
2615         my ($vmid, $resp) = @_;
2616
2617         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2618         # this fails if ballon driver is not loaded, so this must be
2619         # the last commnand (following command are aborted if this fails).
2620         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2621
2622         my $status = 'unknown';
2623         if (!defined($status = $resp->{'return'}->{status})) {
2624             warn "unable to get VM status\n";
2625             return;
2626         }
2627
2628         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2629     };
2630
2631     foreach my $vmid (keys %$list) {
2632         next if $opt_vmid && ($vmid ne $opt_vmid);
2633         next if !$res->{$vmid}->{pid}; # not running
2634         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2635     }
2636
2637     $qmpclient->queue_execute(undef, 1);
2638
2639     foreach my $vmid (keys %$list) {
2640         next if $opt_vmid && ($vmid ne $opt_vmid);
2641         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2642     }
2643
2644     return $res;
2645 }
2646
2647 sub foreach_drive {
2648     my ($conf, $func) = @_;
2649
2650     foreach my $ds (valid_drive_names()) {
2651         next if !defined($conf->{$ds});
2652
2653         my $drive = parse_drive($ds, $conf->{$ds});
2654         next if !$drive;
2655
2656         &$func($ds, $drive);
2657     }
2658 }
2659
2660 sub foreach_volid {
2661     my ($conf, $func) = @_;
2662
2663     my $volhash = {};
2664
2665     my $test_volid = sub {
2666         my ($volid, $is_cdrom) = @_;
2667
2668         return if !$volid;
2669
2670         $volhash->{$volid} = $is_cdrom || 0;
2671     };
2672
2673     foreach_drive($conf, sub {
2674         my ($ds, $drive) = @_;
2675         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2676     });
2677
2678     foreach my $snapname (keys %{$conf->{snapshots}}) {
2679         my $snap = $conf->{snapshots}->{$snapname};
2680         &$test_volid($snap->{vmstate}, 0);
2681         foreach_drive($snap, sub {
2682             my ($ds, $drive) = @_;
2683             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2684         });
2685     }
2686
2687     foreach my $volid (keys %$volhash) {
2688         &$func($volid, $volhash->{$volid});
2689     }
2690 }
2691
2692 sub vga_conf_has_spice {
2693     my ($vga) = @_;
2694
2695     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2696
2697     return $1 || 1;
2698 }
2699
2700 sub config_to_command {
2701     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2702
2703     my $cmd = [];
2704     my $globalFlags = [];
2705     my $machineFlags = [];
2706     my $rtcFlags = [];
2707     my $cpuFlags = [];
2708     my $devices = [];
2709     my $pciaddr = '';
2710     my $bridges = {};
2711     my $kvmver = kvm_user_version();
2712     my $vernum = 0; # unknown
2713     my $ostype = $conf->{ostype};
2714     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2715         $vernum = $1*1000000+$2*1000;
2716     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2717         $vernum = $1*1000000+$2*1000+$3;
2718     }
2719
2720     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2721
2722     my $have_ovz = -f '/proc/vz/vestat';
2723
2724     my $q35 = machine_type_is_q35($conf);
2725     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2726     my $machine_type = $forcemachine || $conf->{machine};
2727     my $use_old_bios_files = undef;
2728     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2729
2730     my $cpuunits = defined($conf->{cpuunits}) ?
2731             $conf->{cpuunits} : $defaults->{cpuunits};
2732
2733     push @$cmd, '/usr/bin/kvm';
2734
2735     push @$cmd, '-id', $vmid;
2736
2737     my $use_virtio = 0;
2738
2739     my $qmpsocket = qmp_socket($vmid);
2740     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2741     push @$cmd, '-mon', "chardev=qmp,mode=control";
2742
2743
2744     push @$cmd, '-pidfile' , pidfile_name($vmid);
2745
2746     push @$cmd, '-daemonize';
2747
2748     if ($conf->{smbios1}) {
2749         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2750     }
2751
2752     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2753         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2754         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2755         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2756         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2757         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2758         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2759     }
2760
2761     if ($q35) {
2762         # the q35 chipset support native usb2, so we enable usb controller
2763         # by default for this machine type
2764         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2765     } else {
2766         $pciaddr = print_pci_addr("piix3", $bridges);
2767         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2768
2769         my $use_usb2 = 0;
2770         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2771             next if !$conf->{"usb$i"};
2772             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2773             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2774             $use_usb2 = 1;
2775         }
2776         # include usb device config
2777         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2778     }
2779
2780     # add usb3 controller if needed
2781
2782     my $use_usb3 = 0;
2783     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2784         next if !$conf->{"usb$i"};
2785         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2786         next if !$d || !$d->{usb3};
2787         $use_usb3 = 1;
2788     }
2789
2790     $pciaddr = print_pci_addr("xhci", $bridges);
2791     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2792
2793     my $vga = $conf->{vga};
2794
2795     my $qxlnum = vga_conf_has_spice($vga);
2796     $vga = 'qxl' if $qxlnum;
2797
2798     if (!$vga) {
2799         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2800                                 $conf->{ostype} eq 'win7' ||
2801                                 $conf->{ostype} eq 'w2k8')) {
2802             $vga = 'std';
2803         } else {
2804             $vga = 'cirrus';
2805         }
2806     }
2807
2808     # enable absolute mouse coordinates (needed by vnc)
2809     my $tablet;
2810     if (defined($conf->{tablet})) {
2811         $tablet = $conf->{tablet};
2812     } else {
2813         $tablet = $defaults->{tablet};
2814         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2815         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2816     }
2817
2818     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2819
2820     my $kvm_off = 0;
2821     # host pci devices
2822     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2823         my $d = parse_hostpci($conf->{"hostpci$i"});
2824         next if !$d;
2825
2826         my $pcie = $d->{pcie};
2827         if($pcie){
2828             die "q35 machine model is not enabled" if !$q35;
2829             $pciaddr = print_pcie_addr("hostpci$i");
2830         }else{
2831             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2832         }
2833
2834         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2835         my $xvga = '';
2836         if ($d->{'x-vga'}) {
2837             $xvga = ',x-vga=on';
2838             $kvm_off = 1;
2839             $vga = 'none';
2840             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2841                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2842             }
2843             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2844                 $xvga = "";
2845             }
2846         }
2847         my $pcidevices = $d->{pciid};
2848         my $multifunction = 1 if @$pcidevices > 1;
2849
2850         my $j=0;
2851         foreach my $pcidevice (@$pcidevices) {
2852
2853             my $id = "hostpci$i";
2854             $id .= ".$j" if $multifunction;
2855             my $addr = $pciaddr;
2856             $addr .= ".$j" if $multifunction;
2857             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2858
2859             if($j == 0){
2860                 $devicestr .= "$rombar$xvga";
2861                 $devicestr .= ",multifunction=on" if $multifunction;
2862             }
2863
2864             push @$devices, '-device', $devicestr;
2865             $j++;
2866         }
2867     }
2868
2869     # usb devices
2870     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2871         next if !$conf->{"usb$i"};
2872         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2873         next if !$d;
2874
2875         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2876         my $usbbus = '';
2877         if (defined($d->{usb3}) && $d->{usb3}) {
2878             $usbbus = ',bus=xhci.0';
2879         }
2880
2881         if (defined($d->{host})) {
2882             $d = parse_usb_device($d->{host});
2883             if (defined($d->{vendorid}) && defined($d->{productid})) {
2884                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2885             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2886                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2887             } elsif (defined($d->{spice}) && $d->{spice}) {
2888                 # usb redir support for spice, currently no usb3
2889                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2890                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2891             }
2892         }
2893     }
2894
2895     # serial devices
2896     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2897         if (my $path = $conf->{"serial$i"}) {
2898             if ($path eq 'socket') {
2899                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2900                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2901                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2902             } else {
2903                 die "no such serial device\n" if ! -c $path;
2904                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2905                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2906             }
2907         }
2908     }
2909
2910     # parallel devices
2911     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2912         if (my $path = $conf->{"parallel$i"}) {
2913             die "no such parallel device\n" if ! -c $path;
2914             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2915             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2916             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2917         }
2918     }
2919
2920     my $vmname = $conf->{name} || "vm$vmid";
2921
2922     push @$cmd, '-name', $vmname;
2923
2924     my $sockets = 1;
2925     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2926     $sockets = $conf->{sockets} if  $conf->{sockets};
2927
2928     my $cores = $conf->{cores} || 1;
2929
2930     my $maxcpus = $sockets * $cores;
2931
2932     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2933
2934     my $allowed_vcpus = $cpuinfo->{cpus};
2935
2936     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2937         if ($allowed_vcpus < $maxcpus);
2938
2939     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2940
2941     push @$cmd, '-nodefaults';
2942
2943     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2944
2945     my $bootindex_hash = {};
2946     my $i = 1;
2947     foreach my $o (split(//, $bootorder)) {
2948         $bootindex_hash->{$o} = $i*100;
2949         $i++;
2950     }
2951
2952     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2953
2954     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2955
2956     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2957
2958     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2959
2960     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2961         my $socket = vnc_socket($vmid);
2962         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2963     } else {
2964         push @$cmd, '-nographic';
2965     }
2966
2967     # time drift fix
2968     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2969
2970     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2971     my $useLocaltime = $conf->{localtime};
2972
2973     if ($ostype) {
2974         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2975
2976         if ($ostype =~ m/^w/) { # windows
2977             $useLocaltime = 1 if !defined($conf->{localtime});
2978
2979             # use time drift fix when acpi is enabled
2980             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2981                 $tdf = 1 if !defined($conf->{tdf});
2982             }
2983         }
2984
2985         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2986             $ostype eq 'wvista') {
2987             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2988             push @$cmd, '-no-hpet';
2989             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2990                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2991                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2992                 push @$cpuFlags , 'hv_time' if !$nokvm;
2993
2994                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
2995                     push @$cpuFlags , 'hv_reset' if !$nokvm;
2996                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
2997                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
2998                 }
2999
3000             } else {
3001                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3002             }
3003         }
3004
3005         if ($ostype eq 'win7' || $ostype eq 'win8') {
3006             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3007         }
3008     }
3009
3010     push @$rtcFlags, 'driftfix=slew' if $tdf;
3011
3012     if ($nokvm) {
3013         push @$machineFlags, 'accel=tcg';
3014     } else {
3015         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3016     }
3017
3018     if ($machine_type) {
3019         push @$machineFlags, "type=${machine_type}";
3020     }
3021
3022     if ($conf->{startdate}) {
3023         push @$rtcFlags, "base=$conf->{startdate}";
3024     } elsif ($useLocaltime) {
3025         push @$rtcFlags, 'base=localtime';
3026     }
3027
3028     my $cpu = $nokvm ? "qemu64" : "kvm64";
3029     if (my $cputype = $conf->{cpu}) {
3030         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3031             or die "Cannot parse cpu description: $cputype\n";
3032         $cpu = $cpuconf->{cputype};
3033         $kvm_off = 1 if $cpuconf->{hidden};
3034     }
3035
3036     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3037
3038     push @$cpuFlags , '-x2apic'
3039         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3040
3041     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3042
3043     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3044
3045     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3046
3047         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3048         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3049     }
3050
3051     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3052
3053     push @$cpuFlags, 'kvm=off' if $kvm_off;
3054
3055     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3056         die "internal error"; # should not happen
3057
3058     push @$cpuFlags, "vendor=${cpu_vendor}"
3059         if $cpu_vendor ne 'default';
3060
3061     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3062
3063     push @$cmd, '-cpu', $cpu;
3064
3065     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3066
3067     push @$cmd, '-S' if $conf->{freeze};
3068
3069     # set keyboard layout
3070     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3071     push @$cmd, '-k', $kb if $kb;
3072
3073     # enable sound
3074     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3075     #push @$cmd, '-soundhw', 'es1370';
3076     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3077
3078     if($conf->{agent}) {
3079         my $qgasocket = qmp_socket($vmid, 1);
3080         my $pciaddr = print_pci_addr("qga0", $bridges);
3081         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3082         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3083         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3084     }
3085
3086     my $spice_port;
3087
3088     if ($qxlnum) {
3089         if ($qxlnum > 1) {
3090             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3091                 for(my $i = 1; $i < $qxlnum; $i++){
3092                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3093                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3094                 }
3095             } else {
3096                 # assume other OS works like Linux
3097                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3098                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3099             }
3100         }
3101
3102         my $pciaddr = print_pci_addr("spice", $bridges);
3103
3104         my $nodename = PVE::INotify::nodename();
3105         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3106         $spice_port = PVE::Tools::next_spice_port($pfamily);
3107
3108         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3109
3110         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3111         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3112         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3113     }
3114
3115     # enable balloon by default, unless explicitly disabled
3116     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3117         $pciaddr = print_pci_addr("balloon0", $bridges);
3118         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3119     }
3120
3121     if ($conf->{watchdog}) {
3122         my $wdopts = parse_watchdog($conf->{watchdog});
3123         $pciaddr = print_pci_addr("watchdog", $bridges);
3124         my $watchdog = $wdopts->{model} || 'i6300esb';
3125         push @$devices, '-device', "$watchdog$pciaddr";
3126         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3127     }
3128
3129     my $vollist = [];
3130     my $scsicontroller = {};
3131     my $ahcicontroller = {};
3132     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3133
3134     # Add iscsi initiator name if available
3135     if (my $initiator = get_initiator_name()) {
3136         push @$devices, '-iscsi', "initiator-name=$initiator";
3137     }
3138
3139     foreach_drive($conf, sub {
3140         my ($ds, $drive) = @_;
3141
3142         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3143             push @$vollist, $drive->{file};
3144         }
3145
3146         $use_virtio = 1 if $ds =~ m/^virtio/;
3147
3148         if (drive_is_cdrom ($drive)) {
3149             if ($bootindex_hash->{d}) {
3150                 $drive->{bootindex} = $bootindex_hash->{d};
3151                 $bootindex_hash->{d} += 1;
3152             }
3153         } else {
3154             if ($bootindex_hash->{c}) {
3155                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3156                 $bootindex_hash->{c} += 1;
3157             }
3158         }
3159
3160         if($drive->{interface} eq 'virtio'){
3161            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3162         }
3163
3164         if ($drive->{interface} eq 'scsi') {
3165
3166             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3167
3168             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3169             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3170
3171             my $iothread = '';
3172             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3173                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3174                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3175             } elsif ($drive->{iothread}) {
3176                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3177             }
3178
3179             my $queues = '';
3180             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3181                 $queues = ",num_queues=$drive->{queues}";
3182             }
3183
3184             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3185             $scsicontroller->{$controller}=1;
3186         }
3187
3188         if ($drive->{interface} eq 'sata') {
3189            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3190            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3191            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3192            $ahcicontroller->{$controller}=1;
3193         }
3194
3195         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3196         push @$devices, '-drive',$drive_cmd;
3197         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3198     });
3199
3200     for (my $i = 0; $i < $MAX_NETS; $i++) {
3201          next if !$conf->{"net$i"};
3202          my $d = parse_net($conf->{"net$i"});
3203          next if !$d;
3204
3205          $use_virtio = 1 if $d->{model} eq 'virtio';
3206
3207          if ($bootindex_hash->{n}) {
3208             $d->{bootindex} = $bootindex_hash->{n};
3209             $bootindex_hash->{n} += 1;
3210          }
3211
3212          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3213          push @$devices, '-netdev', $netdevfull;
3214
3215          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3216          push @$devices, '-device', $netdevicefull;
3217     }
3218
3219     if (!$q35) {
3220         # add pci bridges
3221         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3222            $bridges->{1} = 1;
3223            $bridges->{2} = 1;
3224         }
3225
3226         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3227
3228         while (my ($k, $v) = each %$bridges) {
3229             $pciaddr = print_pci_addr("pci.$k");
3230             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3231         }
3232     }
3233
3234     # add custom args
3235     if ($conf->{args}) {
3236         my $aa = PVE::Tools::split_args($conf->{args});
3237         push @$cmd, @$aa;
3238     }
3239
3240     push @$cmd, @$devices;
3241     push @$cmd, '-rtc', join(',', @$rtcFlags)
3242         if scalar(@$rtcFlags);
3243     push @$cmd, '-machine', join(',', @$machineFlags)
3244         if scalar(@$machineFlags);
3245     push @$cmd, '-global', join(',', @$globalFlags)
3246         if scalar(@$globalFlags);
3247
3248     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3249 }
3250
3251 sub vnc_socket {
3252     my ($vmid) = @_;
3253     return "${var_run_tmpdir}/$vmid.vnc";
3254 }
3255
3256 sub spice_port {
3257     my ($vmid) = @_;
3258
3259     my $res = vm_mon_cmd($vmid, 'query-spice');
3260
3261     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3262 }
3263
3264 sub qmp_socket {
3265     my ($vmid, $qga) = @_;
3266     my $sockettype = $qga ? 'qga' : 'qmp';
3267     return "${var_run_tmpdir}/$vmid.$sockettype";
3268 }
3269
3270 sub pidfile_name {
3271     my ($vmid) = @_;
3272     return "${var_run_tmpdir}/$vmid.pid";
3273 }
3274
3275 sub vm_devices_list {
3276     my ($vmid) = @_;
3277
3278     my $res = vm_mon_cmd($vmid, 'query-pci');
3279     my $devices = {};
3280     foreach my $pcibus (@$res) {
3281         foreach my $device (@{$pcibus->{devices}}) {
3282             next if !$device->{'qdev_id'};
3283             if ($device->{'pci_bridge'}) {
3284                 $devices->{$device->{'qdev_id'}} = 1;
3285                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3286                     next if !$bridge_device->{'qdev_id'};
3287                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3288                     $devices->{$device->{'qdev_id'}}++;
3289                 }
3290             } else {
3291                 $devices->{$device->{'qdev_id'}} = 1;
3292             }
3293         }
3294     }
3295
3296     my $resblock = vm_mon_cmd($vmid, 'query-block');
3297     foreach my $block (@$resblock) {
3298         if($block->{device} =~ m/^drive-(\S+)/){
3299                 $devices->{$1} = 1;
3300         }
3301     }
3302
3303     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3304     foreach my $mice (@$resmice) {
3305         if ($mice->{name} eq 'QEMU HID Tablet') {
3306             $devices->{tablet} = 1;
3307             last;
3308         }
3309     }
3310
3311     return $devices;
3312 }
3313
3314 sub vm_deviceplug {
3315     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3316
3317     my $q35 = machine_type_is_q35($conf);
3318
3319     my $devices_list = vm_devices_list($vmid);
3320     return 1 if defined($devices_list->{$deviceid});
3321
3322     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3323
3324     if ($deviceid eq 'tablet') {
3325
3326         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3327
3328     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3329
3330         qemu_iothread_add($vmid, $deviceid, $device);
3331
3332         qemu_driveadd($storecfg, $vmid, $device);
3333         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3334
3335         qemu_deviceadd($vmid, $devicefull);
3336         eval { qemu_deviceaddverify($vmid, $deviceid); };
3337         if (my $err = $@) {
3338             eval { qemu_drivedel($vmid, $deviceid); };
3339             warn $@ if $@;
3340             die $err;
3341         }
3342
3343     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3344
3345
3346         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3347         my $pciaddr = print_pci_addr($deviceid);
3348         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3349
3350         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3351
3352         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3353             qemu_iothread_add($vmid, $deviceid, $device);
3354             $devicefull .= ",iothread=iothread-$deviceid";
3355         }
3356
3357         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3358             $devicefull .= ",num_queues=$device->{queues}";
3359         }
3360
3361         qemu_deviceadd($vmid, $devicefull);
3362         qemu_deviceaddverify($vmid, $deviceid);
3363
3364     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3365
3366         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3367         qemu_driveadd($storecfg, $vmid, $device);
3368
3369         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3370         eval { qemu_deviceadd($vmid, $devicefull); };
3371         if (my $err = $@) {
3372             eval { qemu_drivedel($vmid, $deviceid); };
3373             warn $@ if $@;
3374             die $err;
3375         }
3376
3377     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3378
3379         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3380
3381         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3382         my $use_old_bios_files = undef;
3383         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3384
3385         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3386         qemu_deviceadd($vmid, $netdevicefull);
3387         eval { qemu_deviceaddverify($vmid, $deviceid); };
3388         if (my $err = $@) {
3389             eval { qemu_netdevdel($vmid, $deviceid); };
3390             warn $@ if $@;
3391             die $err;
3392         }
3393
3394     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3395
3396         my $bridgeid = $2;
3397         my $pciaddr = print_pci_addr($deviceid);
3398         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3399
3400         qemu_deviceadd($vmid, $devicefull);
3401         qemu_deviceaddverify($vmid, $deviceid);
3402
3403     } else {
3404         die "can't hotplug device '$deviceid'\n";
3405     }
3406
3407     return 1;
3408 }
3409
3410 # fixme: this should raise exceptions on error!
3411 sub vm_deviceunplug {
3412     my ($vmid, $conf, $deviceid) = @_;
3413
3414     my $devices_list = vm_devices_list($vmid);
3415     return 1 if !defined($devices_list->{$deviceid});
3416
3417     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3418
3419     if ($deviceid eq 'tablet') {
3420
3421         qemu_devicedel($vmid, $deviceid);
3422
3423     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3424
3425         qemu_devicedel($vmid, $deviceid);
3426         qemu_devicedelverify($vmid, $deviceid);
3427         qemu_drivedel($vmid, $deviceid);
3428         qemu_iothread_del($conf, $vmid, $deviceid);
3429
3430     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3431
3432         qemu_devicedel($vmid, $deviceid);
3433         qemu_devicedelverify($vmid, $deviceid);
3434         qemu_iothread_del($conf, $vmid, $deviceid);
3435
3436     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3437
3438         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3439         my $device = parse_drive($deviceid, $conf->{$deviceid});
3440         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3441
3442         qemu_devicedel($vmid, $deviceid);
3443         qemu_drivedel($vmid, $deviceid);
3444         qemu_deletescsihw($conf, $vmid, $deviceid);
3445
3446     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3447
3448         qemu_devicedel($vmid, $deviceid);
3449         qemu_devicedelverify($vmid, $deviceid);
3450         qemu_netdevdel($vmid, $deviceid);
3451
3452     } else {
3453         die "can't unplug device '$deviceid'\n";
3454     }
3455
3456     return 1;
3457 }
3458
3459 sub qemu_deviceadd {
3460     my ($vmid, $devicefull) = @_;
3461
3462     $devicefull = "driver=".$devicefull;
3463     my %options =  split(/[=,]/, $devicefull);
3464
3465     vm_mon_cmd($vmid, "device_add" , %options);
3466 }
3467
3468 sub qemu_devicedel {
3469     my ($vmid, $deviceid) = @_;
3470
3471     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3472 }
3473
3474 sub qemu_iothread_add {
3475     my($vmid, $deviceid, $device) = @_;
3476
3477     if ($device->{iothread}) {
3478         my $iothreads = vm_iothreads_list($vmid);
3479         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3480     }
3481 }
3482
3483 sub qemu_iothread_del {
3484     my($conf, $vmid, $deviceid) = @_;
3485
3486     my $device = parse_drive($deviceid, $conf->{$deviceid});
3487     if ($device->{iothread}) {
3488         my $iothreads = vm_iothreads_list($vmid);
3489         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3490     }
3491 }
3492
3493 sub qemu_objectadd {
3494     my($vmid, $objectid, $qomtype) = @_;
3495
3496     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3497
3498     return 1;
3499 }
3500
3501 sub qemu_objectdel {
3502     my($vmid, $objectid) = @_;
3503
3504     vm_mon_cmd($vmid, "object-del", id => $objectid);
3505
3506     return 1;
3507 }
3508
3509 sub qemu_driveadd {
3510     my ($storecfg, $vmid, $device) = @_;
3511
3512     my $drive = print_drive_full($storecfg, $vmid, $device);
3513     $drive =~ s/\\/\\\\/g;
3514     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3515
3516     # If the command succeeds qemu prints: "OK"
3517     return 1 if $ret =~ m/OK/s;
3518
3519     die "adding drive failed: $ret\n";
3520 }
3521
3522 sub qemu_drivedel {
3523     my($vmid, $deviceid) = @_;
3524
3525     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3526     $ret =~ s/^\s+//;
3527
3528     return 1 if $ret eq "";
3529
3530     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3531     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3532
3533     die "deleting drive $deviceid failed : $ret\n";
3534 }
3535
3536 sub qemu_deviceaddverify {
3537     my ($vmid, $deviceid) = @_;
3538
3539     for (my $i = 0; $i <= 5; $i++) {
3540          my $devices_list = vm_devices_list($vmid);
3541          return 1 if defined($devices_list->{$deviceid});
3542          sleep 1;
3543     }
3544
3545     die "error on hotplug device '$deviceid'\n";
3546 }
3547
3548
3549 sub qemu_devicedelverify {
3550     my ($vmid, $deviceid) = @_;
3551
3552     # need to verify that the device is correctly removed as device_del
3553     # is async and empty return is not reliable
3554
3555     for (my $i = 0; $i <= 5; $i++) {
3556          my $devices_list = vm_devices_list($vmid);
3557          return 1 if !defined($devices_list->{$deviceid});
3558          sleep 1;
3559     }
3560
3561     die "error on hot-unplugging device '$deviceid'\n";
3562 }
3563
3564 sub qemu_findorcreatescsihw {
3565     my ($storecfg, $conf, $vmid, $device) = @_;
3566
3567     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3568
3569     my $scsihwid="$controller_prefix$controller";
3570     my $devices_list = vm_devices_list($vmid);
3571
3572     if(!defined($devices_list->{$scsihwid})) {
3573         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3574     }
3575
3576     return 1;
3577 }
3578
3579 sub qemu_deletescsihw {
3580     my ($conf, $vmid, $opt) = @_;
3581
3582     my $device = parse_drive($opt, $conf->{$opt});
3583
3584     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3585         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3586         return 1;
3587     }
3588
3589     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3590
3591     my $devices_list = vm_devices_list($vmid);
3592     foreach my $opt (keys %{$devices_list}) {
3593         if (PVE::QemuServer::is_valid_drivename($opt)) {
3594             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3595             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3596                 return 1;
3597             }
3598         }
3599     }
3600
3601     my $scsihwid="scsihw$controller";
3602
3603     vm_deviceunplug($vmid, $conf, $scsihwid);
3604
3605     return 1;
3606 }
3607
3608 sub qemu_add_pci_bridge {
3609     my ($storecfg, $conf, $vmid, $device) = @_;
3610
3611     my $bridges = {};
3612
3613     my $bridgeid;
3614
3615     print_pci_addr($device, $bridges);
3616
3617     while (my ($k, $v) = each %$bridges) {
3618         $bridgeid = $k;
3619     }
3620     return 1 if !defined($bridgeid) || $bridgeid < 1;
3621
3622     my $bridge = "pci.$bridgeid";
3623     my $devices_list = vm_devices_list($vmid);
3624
3625     if (!defined($devices_list->{$bridge})) {
3626         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3627     }
3628
3629     return 1;
3630 }
3631
3632 sub qemu_set_link_status {
3633     my ($vmid, $device, $up) = @_;
3634
3635     vm_mon_cmd($vmid, "set_link", name => $device,
3636                up => $up ? JSON::true : JSON::false);
3637 }
3638
3639 sub qemu_netdevadd {
3640     my ($vmid, $conf, $device, $deviceid) = @_;
3641
3642     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3643     my %options =  split(/[=,]/, $netdev);
3644
3645     vm_mon_cmd($vmid, "netdev_add",  %options);
3646     return 1;
3647 }
3648
3649 sub qemu_netdevdel {
3650     my ($vmid, $deviceid) = @_;
3651
3652     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3653 }
3654
3655 sub qemu_cpu_hotplug {
3656     my ($vmid, $conf, $vcpus) = @_;
3657
3658     my $sockets = 1;
3659     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3660     $sockets = $conf->{sockets} if  $conf->{sockets};
3661     my $cores = $conf->{cores} || 1;
3662     my $maxcpus = $sockets * $cores;
3663
3664     $vcpus = $maxcpus if !$vcpus;
3665
3666     die "you can't add more vcpus than maxcpus\n"
3667         if $vcpus > $maxcpus;
3668
3669     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3670     die "online cpu unplug is not yet possible\n"
3671         if $vcpus < $currentvcpus;
3672
3673     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3674     die "vcpus in running vm is different than configuration\n"
3675         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3676
3677     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3678         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3679     }
3680 }
3681
3682 sub qemu_block_set_io_throttle {
3683     my ($vmid, $deviceid,
3684         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3685         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3686
3687     return if !check_running($vmid) ;
3688
3689     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3690         bps => int($bps),
3691         bps_rd => int($bps_rd),
3692         bps_wr => int($bps_wr),
3693         iops => int($iops),
3694         iops_rd => int($iops_rd),
3695         iops_wr => int($iops_wr),
3696         bps_max => int($bps_max),
3697         bps_rd_max => int($bps_rd_max),
3698         bps_wr_max => int($bps_wr_max),
3699         iops_max => int($iops_max),
3700         iops_rd_max => int($iops_rd_max),
3701         iops_wr_max => int($iops_wr_max)
3702     );
3703
3704 }
3705
3706 # old code, only used to shutdown old VM after update
3707 sub __read_avail {
3708     my ($fh, $timeout) = @_;
3709
3710     my $sel = new IO::Select;
3711     $sel->add($fh);
3712
3713     my $res = '';
3714     my $buf;
3715
3716     my @ready;
3717     while (scalar (@ready = $sel->can_read($timeout))) {
3718         my $count;
3719         if ($count = $fh->sysread($buf, 8192)) {
3720             if ($buf =~ /^(.*)\(qemu\) $/s) {
3721                 $res .= $1;
3722                 last;
3723             } else {
3724                 $res .= $buf;
3725             }
3726         } else {
3727             if (!defined($count)) {
3728                 die "$!\n";
3729             }
3730             last;
3731         }
3732     }
3733
3734     die "monitor read timeout\n" if !scalar(@ready);
3735
3736     return $res;
3737 }
3738
3739 # old code, only used to shutdown old VM after update
3740 sub vm_monitor_command {
3741     my ($vmid, $cmdstr, $nocheck) = @_;
3742
3743     my $res;
3744
3745     eval {
3746         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3747
3748         my $sname = "${var_run_tmpdir}/$vmid.mon";
3749
3750         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3751             die "unable to connect to VM $vmid socket - $!\n";
3752
3753         my $timeout = 3;
3754
3755         # hack: migrate sometime blocks the monitor (when migrate_downtime
3756         # is set)
3757         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3758             $timeout = 60*60; # 1 hour
3759         }
3760
3761         # read banner;
3762         my $data = __read_avail($sock, $timeout);
3763
3764         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3765             die "got unexpected qemu monitor banner\n";
3766         }
3767
3768         my $sel = new IO::Select;
3769         $sel->add($sock);
3770
3771         if (!scalar(my @ready = $sel->can_write($timeout))) {
3772             die "monitor write error - timeout";
3773         }
3774
3775         my $fullcmd = "$cmdstr\r";
3776
3777         # syslog('info', "VM $vmid monitor command: $cmdstr");
3778
3779         my $b;
3780         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3781             die "monitor write error - $!";
3782         }
3783
3784         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3785
3786         $timeout = 20;
3787
3788         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3789             $timeout = 60*60; # 1 hour
3790         } elsif ($cmdstr =~ m/^(eject|change)/) {
3791             $timeout = 60; # note: cdrom mount command is slow
3792         }
3793         if ($res = __read_avail($sock, $timeout)) {
3794
3795             my @lines = split("\r?\n", $res);
3796
3797             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3798
3799             $res = join("\n", @lines);
3800             $res .= "\n";
3801         }
3802     };
3803
3804     my $err = $@;
3805
3806     if ($err) {
3807         syslog("err", "VM $vmid monitor command failed - $err");
3808         die $err;
3809     }
3810
3811     return $res;
3812 }
3813
3814 sub qemu_block_resize {
3815     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3816
3817     my $running = check_running($vmid);
3818
3819     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3820
3821     return if !$running;
3822
3823     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3824
3825 }
3826
3827 sub qemu_volume_snapshot {
3828     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3829
3830     my $running = check_running($vmid);
3831
3832     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3833         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3834     } else {
3835         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3836     }
3837 }
3838
3839 sub qemu_volume_snapshot_delete {
3840     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3841
3842     my $running = check_running($vmid);
3843
3844     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3845
3846     return if !$running;
3847
3848     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3849 }
3850
3851 sub set_migration_caps {
3852     my ($vmid) = @_;
3853
3854     my $cap_ref = [];
3855
3856     my $enabled_cap = {
3857         "auto-converge" => 1,
3858         "xbzrle" => 1,
3859         "x-rdma-pin-all" => 0,
3860         "zero-blocks" => 0,
3861         "compress" => 0
3862     };
3863
3864     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3865
3866     for my $supported_capability (@$supported_capabilities) {
3867         push @$cap_ref, {
3868             capability => $supported_capability->{capability},
3869             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3870         };
3871     }
3872
3873     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3874 }
3875
3876 my $fast_plug_option = {
3877     'lock' => 1,
3878     'name' => 1,
3879     'onboot' => 1,
3880     'shares' => 1,
3881     'startup' => 1,
3882     'description' => 1,
3883 };
3884
3885 # hotplug changes in [PENDING]
3886 # $selection hash can be used to only apply specified options, for
3887 # example: { cores => 1 } (only apply changed 'cores')
3888 # $errors ref is used to return error messages
3889 sub vmconfig_hotplug_pending {
3890     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3891
3892     my $defaults = load_defaults();
3893
3894     # commit values which do not have any impact on running VM first
3895     # Note: those option cannot raise errors, we we do not care about
3896     # $selection and always apply them.
3897
3898     my $add_error = sub {
3899         my ($opt, $msg) = @_;
3900         $errors->{$opt} = "hotplug problem - $msg";
3901     };
3902
3903     my $changes = 0;
3904     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3905         if ($fast_plug_option->{$opt}) {
3906             $conf->{$opt} = $conf->{pending}->{$opt};
3907             delete $conf->{pending}->{$opt};
3908             $changes = 1;
3909         }
3910     }
3911
3912     if ($changes) {
3913         PVE::QemuConfig->write_config($vmid, $conf);
3914         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3915     }
3916
3917     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3918
3919     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3920     while (my ($opt, $force) = each %$pending_delete_hash) {
3921         next if $selection && !$selection->{$opt};
3922         eval {
3923             if ($opt eq 'hotplug') {
3924                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3925             } elsif ($opt eq 'tablet') {
3926                 die "skip\n" if !$hotplug_features->{usb};
3927                 if ($defaults->{tablet}) {
3928                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3929                 } else {
3930                     vm_deviceunplug($vmid, $conf, $opt);
3931                 }
3932             } elsif ($opt eq 'vcpus') {
3933                 die "skip\n" if !$hotplug_features->{cpu};
3934                 qemu_cpu_hotplug($vmid, $conf, undef);
3935             } elsif ($opt eq 'balloon') {
3936                 # enable balloon device is not hotpluggable
3937                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3938             } elsif ($fast_plug_option->{$opt}) {
3939                 # do nothing
3940             } elsif ($opt =~ m/^net(\d+)$/) {
3941                 die "skip\n" if !$hotplug_features->{network};
3942                 vm_deviceunplug($vmid, $conf, $opt);
3943             } elsif (is_valid_drivename($opt)) {
3944                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3945                 vm_deviceunplug($vmid, $conf, $opt);
3946                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3947             } elsif ($opt =~ m/^memory$/) {
3948                 die "skip\n" if !$hotplug_features->{memory};
3949                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3950             } elsif ($opt eq 'cpuunits') {
3951                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3952             } elsif ($opt eq 'cpulimit') {
3953                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3954             } else {
3955                 die "skip\n";
3956             }
3957         };
3958         if (my $err = $@) {
3959             &$add_error($opt, $err) if $err ne "skip\n";
3960         } else {
3961             # save new config if hotplug was successful
3962             delete $conf->{$opt};
3963             vmconfig_undelete_pending_option($conf, $opt);
3964             PVE::QemuConfig->write_config($vmid, $conf);
3965             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3966         }
3967     }
3968
3969     foreach my $opt (keys %{$conf->{pending}}) {
3970         next if $selection && !$selection->{$opt};
3971         my $value = $conf->{pending}->{$opt};
3972         eval {
3973             if ($opt eq 'hotplug') {
3974                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3975             } elsif ($opt eq 'tablet') {
3976                 die "skip\n" if !$hotplug_features->{usb};
3977                 if ($value == 1) {
3978                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3979                 } elsif ($value == 0) {
3980                     vm_deviceunplug($vmid, $conf, $opt);
3981                 }
3982             } elsif ($opt eq 'vcpus') {
3983                 die "skip\n" if !$hotplug_features->{cpu};
3984                 qemu_cpu_hotplug($vmid, $conf, $value);
3985             } elsif ($opt eq 'balloon') {
3986                 # enable/disable balloning device is not hotpluggable
3987                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3988                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3989                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3990
3991                 # allow manual ballooning if shares is set to zero
3992                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3993                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3994                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3995                 }
3996             } elsif ($opt =~ m/^net(\d+)$/) {
3997                 # some changes can be done without hotplug
3998                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3999                                     $vmid, $opt, $value);
4000             } elsif (is_valid_drivename($opt)) {
4001                 # some changes can be done without hotplug
4002                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4003                                      $vmid, $opt, $value, 1);
4004             } elsif ($opt =~ m/^memory$/) { #dimms
4005                 die "skip\n" if !$hotplug_features->{memory};
4006                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4007             } elsif ($opt eq 'cpuunits') {
4008                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4009             } elsif ($opt eq 'cpulimit') {
4010                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4011                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4012             } else {
4013                 die "skip\n";  # skip non-hot-pluggable options
4014             }
4015         };
4016         if (my $err = $@) {
4017             &$add_error($opt, $err) if $err ne "skip\n";
4018         } else {
4019             # save new config if hotplug was successful
4020             $conf->{$opt} = $value;
4021             delete $conf->{pending}->{$opt};
4022             PVE::QemuConfig->write_config($vmid, $conf);
4023             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4024         }
4025     }
4026 }
4027
4028 sub try_deallocate_drive {
4029     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4030
4031     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4032         my $volid = $drive->{file};
4033         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4034             my $sid = PVE::Storage::parse_volume_id($volid);
4035             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4036
4037             # check if the disk is really unused
4038             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4039                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4040             PVE::Storage::vdisk_free($storecfg, $volid);
4041             return 1;
4042         } else {
4043             # If vm is not owner of this disk remove from config
4044             return 1;
4045         }
4046     }
4047
4048     return undef;
4049 }
4050
4051 sub vmconfig_delete_or_detach_drive {
4052     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4053
4054     my $drive = parse_drive($opt, $conf->{$opt});
4055
4056     my $rpcenv = PVE::RPCEnvironment::get();
4057     my $authuser = $rpcenv->get_user();
4058
4059     if ($force) {
4060         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4061         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4062     } else {
4063         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4064     }
4065 }
4066
4067 sub vmconfig_apply_pending {
4068     my ($vmid, $conf, $storecfg) = @_;
4069
4070     # cold plug
4071
4072     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4073     while (my ($opt, $force) = each %$pending_delete_hash) {
4074         die "internal error" if $opt =~ m/^unused/;
4075         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4076         if (!defined($conf->{$opt})) {
4077             vmconfig_undelete_pending_option($conf, $opt);
4078             PVE::QemuConfig->write_config($vmid, $conf);
4079         } elsif (is_valid_drivename($opt)) {
4080             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4081             vmconfig_undelete_pending_option($conf, $opt);
4082             delete $conf->{$opt};
4083             PVE::QemuConfig->write_config($vmid, $conf);
4084         } else {
4085             vmconfig_undelete_pending_option($conf, $opt);
4086             delete $conf->{$opt};
4087             PVE::QemuConfig->write_config($vmid, $conf);
4088         }
4089     }
4090
4091     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4092
4093     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4094         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4095
4096         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4097             # skip if nothing changed
4098         } elsif (is_valid_drivename($opt)) {
4099             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4100                 if defined($conf->{$opt});
4101             $conf->{$opt} = $conf->{pending}->{$opt};
4102         } else {
4103             $conf->{$opt} = $conf->{pending}->{$opt};
4104         }
4105
4106         delete $conf->{pending}->{$opt};
4107         PVE::QemuConfig->write_config($vmid, $conf);
4108     }
4109 }
4110
4111 my $safe_num_ne = sub {
4112     my ($a, $b) = @_;
4113
4114     return 0 if !defined($a) && !defined($b);
4115     return 1 if !defined($a);
4116     return 1 if !defined($b);
4117
4118     return $a != $b;
4119 };
4120
4121 my $safe_string_ne = sub {
4122     my ($a, $b) = @_;
4123
4124     return 0 if !defined($a) && !defined($b);
4125     return 1 if !defined($a);
4126     return 1 if !defined($b);
4127
4128     return $a ne $b;
4129 };
4130
4131 sub vmconfig_update_net {
4132     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4133
4134     my $newnet = parse_net($value);
4135
4136     if ($conf->{$opt}) {
4137         my $oldnet = parse_net($conf->{$opt});
4138
4139         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4140             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4141             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4142             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4143
4144             # for non online change, we try to hot-unplug
4145             die "skip\n" if !$hotplug;
4146             vm_deviceunplug($vmid, $conf, $opt);
4147         } else {
4148
4149             die "internal error" if $opt !~ m/net(\d+)/;
4150             my $iface = "tap${vmid}i$1";
4151
4152             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4153                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4154                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4155                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4156                 PVE::Network::tap_unplug($iface);
4157                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4158             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4159                 # Rate can be applied on its own but any change above needs to
4160                 # include the rate in tap_plug since OVS resets everything.
4161                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4162             }
4163
4164             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4165                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4166             }
4167
4168             return 1;
4169         }
4170     }
4171
4172     if ($hotplug) {
4173         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4174     } else {
4175         die "skip\n";
4176     }
4177 }
4178
4179 sub vmconfig_update_disk {
4180     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4181
4182     # fixme: do we need force?
4183
4184     my $drive = parse_drive($opt, $value);
4185
4186     if ($conf->{$opt}) {
4187
4188         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4189
4190             my $media = $drive->{media} || 'disk';
4191             my $oldmedia = $old_drive->{media} || 'disk';
4192             die "unable to change media type\n" if $media ne $oldmedia;
4193
4194             if (!drive_is_cdrom($old_drive)) {
4195
4196                 if ($drive->{file} ne $old_drive->{file}) {
4197
4198                     die "skip\n" if !$hotplug;
4199
4200                     # unplug and register as unused
4201                     vm_deviceunplug($vmid, $conf, $opt);
4202                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4203
4204                 } else {
4205                     # update existing disk
4206
4207                     # skip non hotpluggable value
4208                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4209                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4210                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4211                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4212                         die "skip\n";
4213                     }
4214
4215                     # apply throttle
4216                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4217                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4218                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4219                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4220                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4221                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4222                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4223                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4224                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4225                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4226                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4227                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4228
4229                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4230                                                    ($drive->{mbps} || 0)*1024*1024,
4231                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4232                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4233                                                    $drive->{iops} || 0,
4234                                                    $drive->{iops_rd} || 0,
4235                                                    $drive->{iops_wr} || 0,
4236                                                    ($drive->{mbps_max} || 0)*1024*1024,
4237                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4238                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4239                                                    $drive->{iops_max} || 0,
4240                                                    $drive->{iops_rd_max} || 0,
4241                                                    $drive->{iops_wr_max} || 0);
4242
4243                     }
4244
4245                     return 1;
4246                 }
4247
4248             } else { # cdrom
4249
4250                 if ($drive->{file} eq 'none') {
4251                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4252                 } else {
4253                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4254                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4255                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4256                 }
4257
4258                 return 1;
4259             }
4260         }
4261     }
4262
4263     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4264     # hotplug new disks
4265     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4266     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4267 }
4268
4269 sub vm_start {
4270     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4271         $forcemachine, $spice_ticket) = @_;
4272
4273     PVE::QemuConfig->lock_config($vmid, sub {
4274         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4275
4276         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4277
4278         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4279
4280         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4281
4282         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4283             vmconfig_apply_pending($vmid, $conf, $storecfg);
4284             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4285         }
4286
4287         my $defaults = load_defaults();
4288
4289         # set environment variable useful inside network script
4290         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4291
4292         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4293
4294         my $migrate_port = 0;
4295         my $migrate_uri;
4296         if ($statefile) {
4297             if ($statefile eq 'tcp') {
4298                 my $localip = "localhost";
4299                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4300                 my $nodename = PVE::INotify::nodename();
4301                 if ($datacenterconf->{migration_unsecure}) {
4302                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4303                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4304                 }
4305                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4306                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4307                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4308                 push @$cmd, '-incoming', $migrate_uri;
4309                 push @$cmd, '-S';
4310
4311             } elsif ($statefile eq 'unix') {
4312                 # should be default for secure migrations as a ssh TCP forward
4313                 # tunnel is not deterministic reliable ready and fails regurarly
4314                 # to set up in time, so use UNIX socket forwards
4315                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4316                 unlink $socket_addr;
4317
4318                 $migrate_uri = "unix:$socket_addr";
4319
4320                 push @$cmd, '-incoming', $migrate_uri;
4321                 push @$cmd, '-S';
4322
4323             } else {
4324                 push @$cmd, '-loadstate', $statefile;
4325             }
4326         } elsif ($paused) {
4327             push @$cmd, '-S';
4328         }
4329
4330         # host pci devices
4331         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4332           my $d = parse_hostpci($conf->{"hostpci$i"});
4333           next if !$d;
4334           my $pcidevices = $d->{pciid};
4335           foreach my $pcidevice (@$pcidevices) {
4336                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4337
4338                 my $info = pci_device_info("0000:$pciid");
4339                 die "IOMMU not present\n" if !check_iommu_support();
4340                 die "no pci device info for device '$pciid'\n" if !$info;
4341                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4342                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4343           }
4344         }
4345
4346         PVE::Storage::activate_volumes($storecfg, $vollist);
4347
4348         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4349             my $cmd = [];
4350             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4351             eval  { run_command($cmd); };
4352         }
4353
4354         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4355                                                   : $defaults->{cpuunits};
4356
4357         eval  {
4358             my %properties = (
4359                 Slice => 'qemu.slice',
4360                 KillMode => 'none',
4361                 CPUShares => $cpuunits
4362             );
4363             if (my $cpulimit = $conf->{cpulimit}) {
4364                 $properties{CPUQuota} = int($cpulimit * 100);
4365             }
4366             $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4367             PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4368             run_command($cmd, timeout => $statefile ? undef : 30, umask => 0077);
4369         };
4370
4371         if (my $err = $@) {
4372             # deactivate volumes if start fails
4373             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4374             die "start failed: $err";
4375         }
4376
4377         print "migration listens on $migrate_uri\n" if $migrate_uri;
4378
4379         if ($statefile && $statefile ne 'tcp')  {
4380             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4381             warn $@ if $@;
4382         }
4383
4384         if ($migratedfrom) {
4385
4386             eval {
4387                 set_migration_caps($vmid);
4388             };
4389             warn $@ if $@;
4390
4391             if ($spice_port) {
4392                 print "spice listens on port $spice_port\n";
4393                 if ($spice_ticket) {
4394                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4395                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4396                 }
4397             }
4398
4399         } else {
4400
4401             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4402                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4403                     if $conf->{balloon};
4404             }
4405
4406             foreach my $opt (keys %$conf) {
4407                 next if $opt !~  m/^net\d+$/;
4408                 my $nicconf = parse_net($conf->{$opt});
4409                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4410             }
4411         }
4412
4413         vm_mon_cmd_nocheck($vmid, 'qom-set',
4414                     path => "machine/peripheral/balloon0",
4415                     property => "guest-stats-polling-interval",
4416                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4417
4418     });
4419 }
4420
4421 sub vm_mon_cmd {
4422     my ($vmid, $execute, %params) = @_;
4423
4424     my $cmd = { execute => $execute, arguments => \%params };
4425     vm_qmp_command($vmid, $cmd);
4426 }
4427
4428 sub vm_mon_cmd_nocheck {
4429     my ($vmid, $execute, %params) = @_;
4430
4431     my $cmd = { execute => $execute, arguments => \%params };
4432     vm_qmp_command($vmid, $cmd, 1);
4433 }
4434
4435 sub vm_qmp_command {
4436     my ($vmid, $cmd, $nocheck) = @_;
4437
4438     my $res;
4439
4440     my $timeout;
4441     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4442         $timeout = $cmd->{arguments}->{timeout};
4443         delete $cmd->{arguments}->{timeout};
4444     }
4445
4446     eval {
4447         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4448         my $sname = qmp_socket($vmid);
4449         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4450             my $qmpclient = PVE::QMPClient->new();
4451
4452             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4453         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4454             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4455                 if scalar(%{$cmd->{arguments}});
4456             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4457         } else {
4458             die "unable to open monitor socket\n";
4459         }
4460     };
4461     if (my $err = $@) {
4462         syslog("err", "VM $vmid qmp command failed - $err");
4463         die $err;
4464     }
4465
4466     return $res;
4467 }
4468
4469 sub vm_human_monitor_command {
4470     my ($vmid, $cmdline) = @_;
4471
4472     my $res;
4473
4474     my $cmd = {
4475         execute => 'human-monitor-command',
4476         arguments => { 'command-line' => $cmdline},
4477     };
4478
4479     return vm_qmp_command($vmid, $cmd);
4480 }
4481
4482 sub vm_commandline {
4483     my ($storecfg, $vmid) = @_;
4484
4485     my $conf = PVE::QemuConfig->load_config($vmid);
4486
4487     my $defaults = load_defaults();
4488
4489     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4490
4491     return join(' ', @$cmd);
4492 }
4493
4494 sub vm_reset {
4495     my ($vmid, $skiplock) = @_;
4496
4497     PVE::QemuConfig->lock_config($vmid, sub {
4498
4499         my $conf = PVE::QemuConfig->load_config($vmid);
4500
4501         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4502
4503         vm_mon_cmd($vmid, "system_reset");
4504     });
4505 }
4506
4507 sub get_vm_volumes {
4508     my ($conf) = @_;
4509
4510     my $vollist = [];
4511     foreach_volid($conf, sub {
4512         my ($volid, $is_cdrom) = @_;
4513
4514         return if $volid =~ m|^/|;
4515
4516         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4517         return if !$sid;
4518
4519         push @$vollist, $volid;
4520     });
4521
4522     return $vollist;
4523 }
4524
4525 sub vm_stop_cleanup {
4526     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4527
4528     eval {
4529
4530         if (!$keepActive) {
4531             my $vollist = get_vm_volumes($conf);
4532             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4533         }
4534
4535         foreach my $ext (qw(mon qmp pid vnc qga)) {
4536             unlink "/var/run/qemu-server/${vmid}.$ext";
4537         }
4538
4539         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4540     };
4541     warn $@ if $@; # avoid errors - just warn
4542 }
4543
4544 # Note: use $nockeck to skip tests if VM configuration file exists.
4545 # We need that when migration VMs to other nodes (files already moved)
4546 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4547 sub vm_stop {
4548     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4549
4550     $force = 1 if !defined($force) && !$shutdown;
4551
4552     if ($migratedfrom){
4553         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4554         kill 15, $pid if $pid;
4555         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4556         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4557         return;
4558     }
4559
4560     PVE::QemuConfig->lock_config($vmid, sub {
4561
4562         my $pid = check_running($vmid, $nocheck);
4563         return if !$pid;
4564
4565         my $conf;
4566         if (!$nocheck) {
4567             $conf = PVE::QemuConfig->load_config($vmid);
4568             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4569             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4570                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4571                 $timeout = $opts->{down} if $opts->{down};
4572             }
4573         }
4574
4575         $timeout = 60 if !defined($timeout);
4576
4577         eval {
4578             if ($shutdown) {
4579                 if (defined($conf) && $conf->{agent}) {
4580                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4581                 } else {
4582                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4583                 }
4584             } else {
4585                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4586             }
4587         };
4588         my $err = $@;
4589
4590         if (!$err) {
4591             my $count = 0;
4592             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4593                 $count++;
4594                 sleep 1;
4595             }
4596
4597             if ($count >= $timeout) {
4598                 if ($force) {
4599                     warn "VM still running - terminating now with SIGTERM\n";
4600                     kill 15, $pid;
4601                 } else {
4602                     die "VM quit/powerdown failed - got timeout\n";
4603                 }
4604             } else {
4605                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4606                 return;
4607             }
4608         } else {
4609             if ($force) {
4610                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4611                 kill 15, $pid;
4612             } else {
4613                 die "VM quit/powerdown failed\n";
4614             }
4615         }
4616
4617         # wait again
4618         $timeout = 10;
4619
4620         my $count = 0;
4621         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4622             $count++;
4623             sleep 1;
4624         }
4625
4626         if ($count >= $timeout) {
4627             warn "VM still running - terminating now with SIGKILL\n";
4628             kill 9, $pid;
4629             sleep 1;
4630         }
4631
4632         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4633    });
4634 }
4635
4636 sub vm_suspend {
4637     my ($vmid, $skiplock) = @_;
4638
4639     PVE::QemuConfig->lock_config($vmid, sub {
4640
4641         my $conf = PVE::QemuConfig->load_config($vmid);
4642
4643         PVE::QemuConfig->check_lock($conf)
4644             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4645
4646         vm_mon_cmd($vmid, "stop");
4647     });
4648 }
4649
4650 sub vm_resume {
4651     my ($vmid, $skiplock, $nocheck) = @_;
4652
4653     PVE::QemuConfig->lock_config($vmid, sub {
4654
4655         if (!$nocheck) {
4656
4657             my $conf = PVE::QemuConfig->load_config($vmid);
4658
4659             PVE::QemuConfig->check_lock($conf)
4660                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4661
4662             vm_mon_cmd($vmid, "cont");
4663
4664         } else {
4665             vm_mon_cmd_nocheck($vmid, "cont");
4666         }
4667     });
4668 }
4669
4670 sub vm_sendkey {
4671     my ($vmid, $skiplock, $key) = @_;
4672
4673     PVE::QemuConfig->lock_config($vmid, sub {
4674
4675         my $conf = PVE::QemuConfig->load_config($vmid);
4676
4677         # there is no qmp command, so we use the human monitor command
4678         vm_human_monitor_command($vmid, "sendkey $key");
4679     });
4680 }
4681
4682 sub vm_destroy {
4683     my ($storecfg, $vmid, $skiplock) = @_;
4684
4685     PVE::QemuConfig->lock_config($vmid, sub {
4686
4687         my $conf = PVE::QemuConfig->load_config($vmid);
4688
4689         if (!check_running($vmid)) {
4690             destroy_vm($storecfg, $vmid, undef, $skiplock);
4691         } else {
4692             die "VM $vmid is running - destroy failed\n";
4693         }
4694     });
4695 }
4696
4697 # pci helpers
4698
4699 sub file_write {
4700     my ($filename, $buf) = @_;
4701
4702     my $fh = IO::File->new($filename, "w");
4703     return undef if !$fh;
4704
4705     my $res = print $fh $buf;
4706
4707     $fh->close();
4708
4709     return $res;
4710 }
4711
4712 sub pci_device_info {
4713     my ($name) = @_;
4714
4715     my $res;
4716
4717     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4718     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4719
4720     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4721     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4722
4723     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4724     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4725
4726     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4727     return undef if !defined($product) || $product !~ s/^0x//;
4728
4729     $res = {
4730         name => $name,
4731         vendor => $vendor,
4732         product => $product,
4733         domain => $domain,
4734         bus => $bus,
4735         slot => $slot,
4736         func => $func,
4737         irq => $irq,
4738         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4739     };
4740
4741     return $res;
4742 }
4743
4744 sub pci_dev_reset {
4745     my ($dev) = @_;
4746
4747     my $name = $dev->{name};
4748
4749     my $fn = "$pcisysfs/devices/$name/reset";
4750
4751     return file_write($fn, "1");
4752 }
4753
4754 sub pci_dev_bind_to_vfio {
4755     my ($dev) = @_;
4756
4757     my $name = $dev->{name};
4758
4759     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4760
4761     if (!-d $vfio_basedir) {
4762         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4763     }
4764     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4765
4766     my $testdir = "$vfio_basedir/$name";
4767     return 1 if -d $testdir;
4768
4769     my $data = "$dev->{vendor} $dev->{product}";
4770     return undef if !file_write("$vfio_basedir/new_id", $data);
4771
4772     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4773     if (!file_write($fn, $name)) {
4774         return undef if -f $fn;
4775     }
4776
4777     $fn = "$vfio_basedir/bind";
4778     if (! -d $testdir) {
4779         return undef if !file_write($fn, $name);
4780     }
4781
4782     return -d $testdir;
4783 }
4784
4785 sub pci_dev_group_bind_to_vfio {
4786     my ($pciid) = @_;
4787
4788     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4789
4790     if (!-d $vfio_basedir) {
4791         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4792     }
4793     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4794
4795     # get IOMMU group devices
4796     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4797       my @devs = grep /^0000:/, readdir($D);
4798     closedir($D);
4799
4800     foreach my $pciid (@devs) {
4801         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4802
4803         # pci bridges, switches or root ports are not supported
4804         # they have a pci_bus subdirectory so skip them
4805         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4806
4807         my $info = pci_device_info($1);
4808         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4809     }
4810
4811     return 1;
4812 }
4813
4814 sub print_pci_addr {
4815     my ($id, $bridges) = @_;
4816
4817     my $res = '';
4818     my $devices = {
4819         piix3 => { bus => 0, addr => 1 },
4820         #addr2 : first videocard
4821         balloon0 => { bus => 0, addr => 3 },
4822         watchdog => { bus => 0, addr => 4 },
4823         scsihw0 => { bus => 0, addr => 5 },
4824         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4825         scsihw1 => { bus => 0, addr => 6 },
4826         ahci0 => { bus => 0, addr => 7 },
4827         qga0 => { bus => 0, addr => 8 },
4828         spice => { bus => 0, addr => 9 },
4829         virtio0 => { bus => 0, addr => 10 },
4830         virtio1 => { bus => 0, addr => 11 },
4831         virtio2 => { bus => 0, addr => 12 },
4832         virtio3 => { bus => 0, addr => 13 },
4833         virtio4 => { bus => 0, addr => 14 },
4834         virtio5 => { bus => 0, addr => 15 },
4835         hostpci0 => { bus => 0, addr => 16 },
4836         hostpci1 => { bus => 0, addr => 17 },
4837         net0 => { bus => 0, addr => 18 },
4838         net1 => { bus => 0, addr => 19 },
4839         net2 => { bus => 0, addr => 20 },
4840         net3 => { bus => 0, addr => 21 },
4841         net4 => { bus => 0, addr => 22 },
4842         net5 => { bus => 0, addr => 23 },
4843         vga1 => { bus => 0, addr => 24 },
4844         vga2 => { bus => 0, addr => 25 },
4845         vga3 => { bus => 0, addr => 26 },
4846         hostpci2 => { bus => 0, addr => 27 },
4847         hostpci3 => { bus => 0, addr => 28 },
4848         #addr29 : usb-host (pve-usb.cfg)
4849         'pci.1' => { bus => 0, addr => 30 },
4850         'pci.2' => { bus => 0, addr => 31 },
4851         'net6' => { bus => 1, addr => 1 },
4852         'net7' => { bus => 1, addr => 2 },
4853         'net8' => { bus => 1, addr => 3 },
4854         'net9' => { bus => 1, addr => 4 },
4855         'net10' => { bus => 1, addr => 5 },
4856         'net11' => { bus => 1, addr => 6 },
4857         'net12' => { bus => 1, addr => 7 },
4858         'net13' => { bus => 1, addr => 8 },
4859         'net14' => { bus => 1, addr => 9 },
4860         'net15' => { bus => 1, addr => 10 },
4861         'net16' => { bus => 1, addr => 11 },
4862         'net17' => { bus => 1, addr => 12 },
4863         'net18' => { bus => 1, addr => 13 },
4864         'net19' => { bus => 1, addr => 14 },
4865         'net20' => { bus => 1, addr => 15 },
4866         'net21' => { bus => 1, addr => 16 },
4867         'net22' => { bus => 1, addr => 17 },
4868         'net23' => { bus => 1, addr => 18 },
4869         'net24' => { bus => 1, addr => 19 },
4870         'net25' => { bus => 1, addr => 20 },
4871         'net26' => { bus => 1, addr => 21 },
4872         'net27' => { bus => 1, addr => 22 },
4873         'net28' => { bus => 1, addr => 23 },
4874         'net29' => { bus => 1, addr => 24 },
4875         'net30' => { bus => 1, addr => 25 },
4876         'net31' => { bus => 1, addr => 26 },
4877         'xhci' => { bus => 1, addr => 27 },
4878         'virtio6' => { bus => 2, addr => 1 },
4879         'virtio7' => { bus => 2, addr => 2 },
4880         'virtio8' => { bus => 2, addr => 3 },
4881         'virtio9' => { bus => 2, addr => 4 },
4882         'virtio10' => { bus => 2, addr => 5 },
4883         'virtio11' => { bus => 2, addr => 6 },
4884         'virtio12' => { bus => 2, addr => 7 },
4885         'virtio13' => { bus => 2, addr => 8 },
4886         'virtio14' => { bus => 2, addr => 9 },
4887         'virtio15' => { bus => 2, addr => 10 },
4888         'virtioscsi0' => { bus => 3, addr => 1 },
4889         'virtioscsi1' => { bus => 3, addr => 2 },
4890         'virtioscsi2' => { bus => 3, addr => 3 },
4891         'virtioscsi3' => { bus => 3, addr => 4 },
4892         'virtioscsi4' => { bus => 3, addr => 5 },
4893         'virtioscsi5' => { bus => 3, addr => 6 },
4894         'virtioscsi6' => { bus => 3, addr => 7 },
4895         'virtioscsi7' => { bus => 3, addr => 8 },
4896         'virtioscsi8' => { bus => 3, addr => 9 },
4897         'virtioscsi9' => { bus => 3, addr => 10 },
4898         'virtioscsi10' => { bus => 3, addr => 11 },
4899         'virtioscsi11' => { bus => 3, addr => 12 },
4900         'virtioscsi12' => { bus => 3, addr => 13 },
4901         'virtioscsi13' => { bus => 3, addr => 14 },
4902         'virtioscsi14' => { bus => 3, addr => 15 },
4903         'virtioscsi15' => { bus => 3, addr => 16 },
4904         'virtioscsi16' => { bus => 3, addr => 17 },
4905         'virtioscsi17' => { bus => 3, addr => 18 },
4906         'virtioscsi18' => { bus => 3, addr => 19 },
4907         'virtioscsi19' => { bus => 3, addr => 20 },
4908         'virtioscsi20' => { bus => 3, addr => 21 },
4909         'virtioscsi21' => { bus => 3, addr => 22 },
4910         'virtioscsi22' => { bus => 3, addr => 23 },
4911         'virtioscsi23' => { bus => 3, addr => 24 },
4912         'virtioscsi24' => { bus => 3, addr => 25 },
4913         'virtioscsi25' => { bus => 3, addr => 26 },
4914         'virtioscsi26' => { bus => 3, addr => 27 },
4915         'virtioscsi27' => { bus => 3, addr => 28 },
4916         'virtioscsi28' => { bus => 3, addr => 29 },
4917         'virtioscsi29' => { bus => 3, addr => 30 },
4918         'virtioscsi30' => { bus => 3, addr => 31 },
4919
4920     };
4921
4922     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4923            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4924            my $bus = $devices->{$id}->{bus};
4925            $res = ",bus=pci.$bus,addr=$addr";
4926            $bridges->{$bus} = 1 if $bridges;
4927     }
4928     return $res;
4929
4930 }
4931
4932 sub print_pcie_addr {
4933     my ($id) = @_;
4934
4935     my $res = '';
4936     my $devices = {
4937         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4938         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4939         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4940         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4941     };
4942
4943     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4944            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4945            my $bus = $devices->{$id}->{bus};
4946            $res = ",bus=$bus,addr=$addr";
4947     }
4948     return $res;
4949
4950 }
4951
4952 # vzdump restore implementaion
4953
4954 sub tar_archive_read_firstfile {
4955     my $archive = shift;
4956
4957     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4958
4959     # try to detect archive type first
4960     my $pid = open (TMP, "tar tf '$archive'|") ||
4961         die "unable to open file '$archive'\n";
4962     my $firstfile = <TMP>;
4963     kill 15, $pid;
4964     close TMP;
4965
4966     die "ERROR: archive contaions no data\n" if !$firstfile;
4967     chomp $firstfile;
4968
4969     return $firstfile;
4970 }
4971
4972 sub tar_restore_cleanup {
4973     my ($storecfg, $statfile) = @_;
4974
4975     print STDERR "starting cleanup\n";
4976
4977     if (my $fd = IO::File->new($statfile, "r")) {
4978         while (defined(my $line = <$fd>)) {
4979             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4980                 my $volid = $2;
4981                 eval {
4982                     if ($volid =~ m|^/|) {
4983                         unlink $volid || die 'unlink failed\n';
4984                     } else {
4985                         PVE::Storage::vdisk_free($storecfg, $volid);
4986                     }
4987                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4988                 };
4989                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4990             } else {
4991                 print STDERR "unable to parse line in statfile - $line";
4992             }
4993         }
4994         $fd->close();
4995     }
4996 }
4997
4998 sub restore_archive {
4999     my ($archive, $vmid, $user, $opts) = @_;
5000
5001     my $format = $opts->{format};
5002     my $comp;
5003
5004     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5005         $format = 'tar' if !$format;
5006         $comp = 'gzip';
5007     } elsif ($archive =~ m/\.tar$/) {
5008         $format = 'tar' if !$format;
5009     } elsif ($archive =~ m/.tar.lzo$/) {
5010         $format = 'tar' if !$format;
5011         $comp = 'lzop';
5012     } elsif ($archive =~ m/\.vma$/) {
5013         $format = 'vma' if !$format;
5014     } elsif ($archive =~ m/\.vma\.gz$/) {
5015         $format = 'vma' if !$format;
5016         $comp = 'gzip';
5017     } elsif ($archive =~ m/\.vma\.lzo$/) {
5018         $format = 'vma' if !$format;
5019         $comp = 'lzop';
5020     } else {
5021         $format = 'vma' if !$format; # default
5022     }
5023
5024     # try to detect archive format
5025     if ($format eq 'tar') {
5026         return restore_tar_archive($archive, $vmid, $user, $opts);
5027     } else {
5028         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5029     }
5030 }
5031
5032 sub restore_update_config_line {
5033     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5034
5035     return if $line =~ m/^\#qmdump\#/;
5036     return if $line =~ m/^\#vzdump\#/;
5037     return if $line =~ m/^lock:/;
5038     return if $line =~ m/^unused\d+:/;
5039     return if $line =~ m/^parent:/;
5040     return if $line =~ m/^template:/; # restored VM is never a template
5041
5042     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5043         # try to convert old 1.X settings
5044         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5045         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5046             my ($model, $macaddr) = split(/\=/, $devconfig);
5047             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5048             my $net = {
5049                 model => $model,
5050                 bridge => "vmbr$ind",
5051                 macaddr => $macaddr,
5052             };
5053             my $netstr = print_net($net);
5054
5055             print $outfd "net$cookie->{netcount}: $netstr\n";
5056             $cookie->{netcount}++;
5057         }
5058     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5059         my ($id, $netstr) = ($1, $2);
5060         my $net = parse_net($netstr);
5061         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5062         $netstr = print_net($net);
5063         print $outfd "$id: $netstr\n";
5064     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5065         my $virtdev = $1;
5066         my $value = $3;
5067         my $di = parse_drive($virtdev, $value);
5068         if (defined($di->{backup}) && !$di->{backup}) {
5069             print $outfd "#$line";
5070         } elsif ($map->{$virtdev}) {
5071             delete $di->{format}; # format can change on restore
5072             $di->{file} = $map->{$virtdev};
5073             $value = print_drive($vmid, $di);
5074             print $outfd "$virtdev: $value\n";
5075         } else {
5076             print $outfd $line;
5077         }
5078     } else {
5079         print $outfd $line;
5080     }
5081 }
5082
5083 sub scan_volids {
5084     my ($cfg, $vmid) = @_;
5085
5086     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5087
5088     my $volid_hash = {};
5089     foreach my $storeid (keys %$info) {
5090         foreach my $item (@{$info->{$storeid}}) {
5091             next if !($item->{volid} && $item->{size});
5092             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5093             $volid_hash->{$item->{volid}} = $item;
5094         }
5095     }
5096
5097     return $volid_hash;
5098 }
5099
5100 sub is_volume_in_use {
5101     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5102
5103     my $path = PVE::Storage::path($storecfg, $volid);
5104
5105     my $scan_config = sub {
5106         my ($cref, $snapname) = @_;
5107
5108         foreach my $key (keys %$cref) {
5109             my $value = $cref->{$key};
5110             if (is_valid_drivename($key)) {
5111                 next if $skip_drive && $key eq $skip_drive;
5112                 my $drive = parse_drive($key, $value);
5113                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5114                 return 1 if $volid eq $drive->{file};
5115                 if ($drive->{file} =~ m!^/!) {
5116                     return 1 if $drive->{file} eq $path;
5117                 } else {
5118                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5119                     next if !$storeid;
5120                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5121                     next if !$scfg;
5122                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5123                 }
5124             }
5125         }
5126
5127         return 0;
5128     };
5129
5130     return 1 if &$scan_config($conf);
5131
5132     undef $skip_drive;
5133
5134     foreach my $snapname (keys %{$conf->{snapshots}}) {
5135         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5136     }
5137
5138     return 0;
5139 }
5140
5141 sub update_disksize {
5142     my ($vmid, $conf, $volid_hash) = @_;
5143
5144     my $changes;
5145
5146     my $used = {};
5147
5148     # Note: it is allowed to define multiple storages with same path (alias), so
5149     # we need to check both 'volid' and real 'path' (two different volid can point
5150     # to the same path).
5151
5152     my $usedpath = {};
5153
5154     # update size info
5155     foreach my $opt (keys %$conf) {
5156         if (is_valid_drivename($opt)) {
5157             my $drive = parse_drive($opt, $conf->{$opt});
5158             my $volid = $drive->{file};
5159             next if !$volid;
5160
5161             $used->{$volid} = 1;
5162             if ($volid_hash->{$volid} &&
5163                 (my $path = $volid_hash->{$volid}->{path})) {
5164                 $usedpath->{$path} = 1;
5165             }
5166
5167             next if drive_is_cdrom($drive);
5168             next if !$volid_hash->{$volid};
5169
5170             $drive->{size} = $volid_hash->{$volid}->{size};
5171             my $new = print_drive($vmid, $drive);
5172             if ($new ne $conf->{$opt}) {
5173                 $changes = 1;
5174                 $conf->{$opt} = $new;
5175             }
5176         }
5177     }
5178
5179     # remove 'unusedX' entry if volume is used
5180     foreach my $opt (keys %$conf) {
5181         next if $opt !~ m/^unused\d+$/;
5182         my $volid = $conf->{$opt};
5183         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5184         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5185             $changes = 1;
5186             delete $conf->{$opt};
5187         }
5188     }
5189
5190     foreach my $volid (sort keys %$volid_hash) {
5191         next if $volid =~ m/vm-$vmid-state-/;
5192         next if $used->{$volid};
5193         my $path = $volid_hash->{$volid}->{path};
5194         next if !$path; # just to be sure
5195         next if $usedpath->{$path};
5196         $changes = 1;
5197         PVE::QemuConfig->add_unused_volume($conf, $volid);
5198         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5199     }
5200
5201     return $changes;
5202 }
5203
5204 sub rescan {
5205     my ($vmid, $nolock) = @_;
5206
5207     my $cfg = PVE::Storage::config();
5208
5209     my $volid_hash = scan_volids($cfg, $vmid);
5210
5211     my $updatefn =  sub {
5212         my ($vmid) = @_;
5213
5214         my $conf = PVE::QemuConfig->load_config($vmid);
5215
5216         PVE::QemuConfig->check_lock($conf);
5217
5218         my $vm_volids = {};
5219         foreach my $volid (keys %$volid_hash) {
5220             my $info = $volid_hash->{$volid};
5221             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5222         }
5223
5224         my $changes = update_disksize($vmid, $conf, $vm_volids);
5225
5226         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5227     };
5228
5229     if (defined($vmid)) {
5230         if ($nolock) {
5231             &$updatefn($vmid);
5232         } else {
5233             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5234         }
5235     } else {
5236         my $vmlist = config_list();
5237         foreach my $vmid (keys %$vmlist) {
5238             if ($nolock) {
5239                 &$updatefn($vmid);
5240             } else {
5241                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5242             }
5243         }
5244     }
5245 }
5246
5247 sub restore_vma_archive {
5248     my ($archive, $vmid, $user, $opts, $comp) = @_;
5249
5250     my $input = $archive eq '-' ? "<&STDIN" : undef;
5251     my $readfrom = $archive;
5252
5253     my $uncomp = '';
5254     if ($comp) {
5255         $readfrom = '-';
5256         my $qarchive = PVE::Tools::shellquote($archive);
5257         if ($comp eq 'gzip') {
5258             $uncomp = "zcat $qarchive|";
5259         } elsif ($comp eq 'lzop') {
5260             $uncomp = "lzop -d -c $qarchive|";
5261         } else {
5262             die "unknown compression method '$comp'\n";
5263         }
5264
5265     }
5266
5267     my $tmpdir = "/var/tmp/vzdumptmp$$";
5268     rmtree $tmpdir;
5269
5270     # disable interrupts (always do cleanups)
5271     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5272         warn "got interrupt - ignored\n";
5273     };
5274
5275     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5276     POSIX::mkfifo($mapfifo, 0600);
5277     my $fifofh;
5278
5279     my $openfifo = sub {
5280         open($fifofh, '>', $mapfifo) || die $!;
5281     };
5282
5283     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5284
5285     my $oldtimeout;
5286     my $timeout = 5;
5287
5288     my $devinfo = {};
5289
5290     my $rpcenv = PVE::RPCEnvironment::get();
5291
5292     my $conffile = PVE::QemuConfig->config_file($vmid);
5293     my $tmpfn = "$conffile.$$.tmp";
5294
5295     # Note: $oldconf is undef if VM does not exists
5296     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5297     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5298
5299     my $print_devmap = sub {
5300         my $virtdev_hash = {};
5301
5302         my $cfgfn = "$tmpdir/qemu-server.conf";
5303
5304         # we can read the config - that is already extracted
5305         my $fh = IO::File->new($cfgfn, "r") ||
5306             "unable to read qemu-server.conf - $!\n";
5307
5308         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5309         if (-f $fwcfgfn) {
5310             my $pve_firewall_dir = '/etc/pve/firewall';
5311             mkdir $pve_firewall_dir; # make sure the dir exists
5312             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5313         }
5314
5315         while (defined(my $line = <$fh>)) {
5316             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5317                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5318                 die "archive does not contain data for drive '$virtdev'\n"
5319                     if !$devinfo->{$devname};
5320                 if (defined($opts->{storage})) {
5321                     $storeid = $opts->{storage} || 'local';
5322                 } elsif (!$storeid) {
5323                     $storeid = 'local';
5324                 }
5325                 $format = 'raw' if !$format;
5326                 $devinfo->{$devname}->{devname} = $devname;
5327                 $devinfo->{$devname}->{virtdev} = $virtdev;
5328                 $devinfo->{$devname}->{format} = $format;
5329                 $devinfo->{$devname}->{storeid} = $storeid;
5330
5331                 # check permission on storage
5332                 my $pool = $opts->{pool}; # todo: do we need that?
5333                 if ($user ne 'root@pam') {
5334                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5335                 }
5336
5337                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5338             }
5339         }
5340
5341         foreach my $devname (keys %$devinfo) {
5342             die "found no device mapping information for device '$devname'\n"
5343                 if !$devinfo->{$devname}->{virtdev};
5344         }
5345
5346         my $cfg = PVE::Storage::config();
5347
5348         # create empty/temp config
5349         if ($oldconf) {
5350             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5351             foreach_drive($oldconf, sub {
5352                 my ($ds, $drive) = @_;
5353
5354                 return if drive_is_cdrom($drive);
5355
5356                 my $volid = $drive->{file};
5357
5358                 return if !$volid || $volid =~ m|^/|;
5359
5360                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5361                 return if !$path || !$owner || ($owner != $vmid);
5362
5363                 # Note: only delete disk we want to restore
5364                 # other volumes will become unused
5365                 if ($virtdev_hash->{$ds}) {
5366                     PVE::Storage::vdisk_free($cfg, $volid);
5367                 }
5368             });
5369
5370             # delete vmstate files
5371             # since after the restore we have no snapshots anymore
5372             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5373                 my $snap = $oldconf->{snapshots}->{$snapname};
5374                 if ($snap->{vmstate}) {
5375                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5376                     if (my $err = $@) {
5377                         warn $err;
5378                     }
5379                 }
5380             }
5381         }
5382
5383         my $map = {};
5384         foreach my $virtdev (sort keys %$virtdev_hash) {
5385             my $d = $virtdev_hash->{$virtdev};
5386             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5387             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5388
5389             # test if requested format is supported
5390             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5391             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5392             $d->{format} = $defFormat if !$supported;
5393
5394             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5395                                                   $d->{format}, undef, $alloc_size);
5396             print STDERR "new volume ID is '$volid'\n";
5397             $d->{volid} = $volid;
5398             my $path = PVE::Storage::path($cfg, $volid);
5399
5400             PVE::Storage::activate_volumes($cfg,[$volid]);
5401
5402             my $write_zeros = 1;
5403             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5404                 $write_zeros = 0;
5405             }
5406
5407             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5408
5409             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5410             $map->{$virtdev} = $volid;
5411         }
5412
5413         $fh->seek(0, 0) || die "seek failed - $!\n";
5414
5415         my $outfd = new IO::File ($tmpfn, "w") ||
5416             die "unable to write config for VM $vmid\n";
5417
5418         my $cookie = { netcount => 0 };
5419         while (defined(my $line = <$fh>)) {
5420             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5421         }
5422
5423         $fh->close();
5424         $outfd->close();
5425     };
5426
5427     eval {
5428         # enable interrupts
5429         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5430             die "interrupted by signal\n";
5431         };
5432         local $SIG{ALRM} = sub { die "got timeout\n"; };
5433
5434         $oldtimeout = alarm($timeout);
5435
5436         my $parser = sub {
5437             my $line = shift;
5438
5439             print "$line\n";
5440
5441             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5442                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5443                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5444             } elsif ($line =~ m/^CTIME: /) {
5445                 # we correctly received the vma config, so we can disable
5446                 # the timeout now for disk allocation (set to 10 minutes, so
5447                 # that we always timeout if something goes wrong)
5448                 alarm(600);
5449                 &$print_devmap();
5450                 print $fifofh "done\n";
5451                 my $tmp = $oldtimeout || 0;
5452                 $oldtimeout = undef;
5453                 alarm($tmp);
5454                 close($fifofh);
5455             }
5456         };
5457
5458         print "restore vma archive: $cmd\n";
5459         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5460     };
5461     my $err = $@;
5462
5463     alarm($oldtimeout) if $oldtimeout;
5464
5465     my $vollist = [];
5466     foreach my $devname (keys %$devinfo) {
5467         my $volid = $devinfo->{$devname}->{volid};
5468         push @$vollist, $volid if $volid;
5469     }
5470
5471     my $cfg = PVE::Storage::config();
5472     PVE::Storage::deactivate_volumes($cfg, $vollist);
5473
5474     unlink $mapfifo;
5475
5476     if ($err) {
5477         rmtree $tmpdir;
5478         unlink $tmpfn;
5479
5480         foreach my $devname (keys %$devinfo) {
5481             my $volid = $devinfo->{$devname}->{volid};
5482             next if !$volid;
5483             eval {
5484                 if ($volid =~ m|^/|) {
5485                     unlink $volid || die 'unlink failed\n';
5486                 } else {
5487                     PVE::Storage::vdisk_free($cfg, $volid);
5488                 }
5489                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5490             };
5491             print STDERR "unable to cleanup '$volid' - $@" if $@;
5492         }
5493         die $err;
5494     }
5495
5496     rmtree $tmpdir;
5497
5498     rename($tmpfn, $conffile) ||
5499         die "unable to commit configuration file '$conffile'\n";
5500
5501     PVE::Cluster::cfs_update(); # make sure we read new file
5502
5503     eval { rescan($vmid, 1); };
5504     warn $@ if $@;
5505 }
5506
5507 sub restore_tar_archive {
5508     my ($archive, $vmid, $user, $opts) = @_;
5509
5510     if ($archive ne '-') {
5511         my $firstfile = tar_archive_read_firstfile($archive);
5512         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5513             if $firstfile ne 'qemu-server.conf';
5514     }
5515
5516     my $storecfg = PVE::Storage::config();
5517
5518     # destroy existing data - keep empty config
5519     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5520     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5521
5522     my $tocmd = "/usr/lib/qemu-server/qmextract";
5523
5524     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5525     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5526     $tocmd .= ' --prealloc' if $opts->{prealloc};
5527     $tocmd .= ' --info' if $opts->{info};
5528
5529     # tar option "xf" does not autodetect compression when read from STDIN,
5530     # so we pipe to zcat
5531     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5532         PVE::Tools::shellquote("--to-command=$tocmd");
5533
5534     my $tmpdir = "/var/tmp/vzdumptmp$$";
5535     mkpath $tmpdir;
5536
5537     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5538     local $ENV{VZDUMP_VMID} = $vmid;
5539     local $ENV{VZDUMP_USER} = $user;
5540
5541     my $conffile = PVE::QemuConfig->config_file($vmid);
5542     my $tmpfn = "$conffile.$$.tmp";
5543
5544     # disable interrupts (always do cleanups)
5545     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5546         print STDERR "got interrupt - ignored\n";
5547     };
5548
5549     eval {
5550         # enable interrupts
5551         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5552             die "interrupted by signal\n";
5553         };
5554
5555         if ($archive eq '-') {
5556             print "extracting archive from STDIN\n";
5557             run_command($cmd, input => "<&STDIN");
5558         } else {
5559             print "extracting archive '$archive'\n";
5560             run_command($cmd);
5561         }
5562
5563         return if $opts->{info};
5564
5565         # read new mapping
5566         my $map = {};
5567         my $statfile = "$tmpdir/qmrestore.stat";
5568         if (my $fd = IO::File->new($statfile, "r")) {
5569             while (defined (my $line = <$fd>)) {
5570                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5571                     $map->{$1} = $2 if $1;
5572                 } else {
5573                     print STDERR "unable to parse line in statfile - $line\n";
5574                 }
5575             }
5576             $fd->close();
5577         }
5578
5579         my $confsrc = "$tmpdir/qemu-server.conf";
5580
5581         my $srcfd = new IO::File($confsrc, "r") ||
5582             die "unable to open file '$confsrc'\n";
5583
5584         my $outfd = new IO::File ($tmpfn, "w") ||
5585             die "unable to write config for VM $vmid\n";
5586
5587         my $cookie = { netcount => 0 };
5588         while (defined (my $line = <$srcfd>)) {
5589             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5590         }
5591
5592         $srcfd->close();
5593         $outfd->close();
5594     };
5595     my $err = $@;
5596
5597     if ($err) {
5598
5599         unlink $tmpfn;
5600
5601         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5602
5603         die $err;
5604     }
5605
5606     rmtree $tmpdir;
5607
5608     rename $tmpfn, $conffile ||
5609         die "unable to commit configuration file '$conffile'\n";
5610
5611     PVE::Cluster::cfs_update(); # make sure we read new file
5612
5613     eval { rescan($vmid, 1); };
5614     warn $@ if $@;
5615 };
5616
5617 sub foreach_writable_storage {
5618     my ($conf, $func) = @_;
5619
5620     my $sidhash = {};
5621
5622     foreach my $ds (keys %$conf) {
5623         next if !is_valid_drivename($ds);
5624
5625         my $drive = parse_drive($ds, $conf->{$ds});
5626         next if !$drive;
5627         next if drive_is_cdrom($drive);
5628
5629         my $volid = $drive->{file};
5630
5631         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5632         $sidhash->{$sid} = $sid if $sid;
5633     }
5634
5635     foreach my $sid (sort keys %$sidhash) {
5636         &$func($sid);
5637     }
5638 }
5639
5640 sub do_snapshots_with_qemu {
5641     my ($storecfg, $volid) = @_;
5642
5643     my $storage_name = PVE::Storage::parse_volume_id($volid);
5644
5645     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5646         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5647         return 1;
5648     }
5649
5650     if ($volid =~ m/\.(qcow2|qed)$/){
5651         return 1;
5652     }
5653
5654     return undef;
5655 }
5656
5657 sub qga_check_running {
5658     my ($vmid) = @_;
5659
5660     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5661     if ($@) {
5662         warn "Qemu Guest Agent are not running - $@";
5663         return 0;
5664     }
5665     return 1;
5666 }
5667
5668 sub template_create {
5669     my ($vmid, $conf, $disk) = @_;
5670
5671     my $storecfg = PVE::Storage::config();
5672
5673     foreach_drive($conf, sub {
5674         my ($ds, $drive) = @_;
5675
5676         return if drive_is_cdrom($drive);
5677         return if $disk && $ds ne $disk;
5678
5679         my $volid = $drive->{file};
5680         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5681
5682         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5683         $drive->{file} = $voliddst;
5684         $conf->{$ds} = print_drive($vmid, $drive);
5685         PVE::QemuConfig->write_config($vmid, $conf);
5686     });
5687 }
5688
5689 sub qemu_img_convert {
5690     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5691
5692     my $storecfg = PVE::Storage::config();
5693     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5694     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5695
5696     if ($src_storeid && $dst_storeid) {
5697
5698         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5699
5700         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5701         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5702
5703         my $src_format = qemu_img_format($src_scfg, $src_volname);
5704         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5705
5706         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5707         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5708
5709         my $cmd = [];
5710         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5711         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5712         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5713         if ($is_zero_initialized) {
5714             push @$cmd, "zeroinit:$dst_path";
5715         } else {
5716             push @$cmd, $dst_path;
5717         }
5718
5719         my $parser = sub {
5720             my $line = shift;
5721             if($line =~ m/\((\S+)\/100\%\)/){
5722                 my $percent = $1;
5723                 my $transferred = int($size * $percent / 100);
5724                 my $remaining = $size - $transferred;
5725
5726                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5727             }
5728
5729         };
5730
5731         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5732         my $err = $@;
5733         die "copy failed: $err" if $err;
5734     }
5735 }
5736
5737 sub qemu_img_format {
5738     my ($scfg, $volname) = @_;
5739
5740     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5741         return $1;
5742     } else {
5743         return "raw";
5744     }
5745 }
5746
5747 sub qemu_drive_mirror {
5748     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5749
5750     my $storecfg = PVE::Storage::config();
5751     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5752
5753     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5754
5755     my $format = qemu_img_format($dst_scfg, $dst_volname);
5756
5757     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5758
5759     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5760
5761     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5762     $opts->{format} = $format if $format;
5763
5764     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5765
5766     my $finish_job = sub {
5767         while (1) {
5768             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5769             my $stat = @$stats[0];
5770             last if !$stat;
5771             sleep 1;
5772         }
5773     };
5774
5775     eval {
5776     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5777         while (1) {
5778             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5779             my $stat = @$stats[0];
5780             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5781             die "error job is not mirroring" if $stat->{type} ne "mirror";
5782
5783             my $busy = $stat->{busy};
5784             my $ready = $stat->{ready};
5785
5786             if (my $total = $stat->{len}) {
5787                 my $transferred = $stat->{offset} || 0;
5788                 my $remaining = $total - $transferred;
5789                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5790
5791                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5792             }
5793
5794
5795             if ($stat->{ready} eq 'true') {
5796
5797                 last if $vmiddst != $vmid;
5798
5799                 # try to switch the disk if source and destination are on the same guest
5800                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5801                 if (!$@) {
5802                     &$finish_job();
5803                     last;
5804                 }
5805                 die $@ if $@ !~ m/cannot be completed/;
5806             }
5807             sleep 1;
5808         }
5809
5810
5811     };
5812     my $err = $@;
5813
5814     my $cancel_job = sub {
5815         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5816         &$finish_job();
5817     };
5818
5819     if ($err) {
5820         eval { &$cancel_job(); };
5821         die "mirroring error: $err";
5822     }
5823
5824     if ($vmiddst != $vmid) {
5825         # if we clone a disk for a new target vm, we don't switch the disk
5826         &$cancel_job(); # so we call block-job-cancel
5827     }
5828 }
5829
5830 sub clone_disk {
5831     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5832         $newvmid, $storage, $format, $full, $newvollist) = @_;
5833
5834     my $newvolid;
5835
5836     if (!$full) {
5837         print "create linked clone of drive $drivename ($drive->{file})\n";
5838         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5839         push @$newvollist, $newvolid;
5840     } else {
5841         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5842         $storeid = $storage if $storage;
5843
5844         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5845         if (!$format) {
5846             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5847             $format = qemu_img_format($scfg, $volname);
5848         }
5849
5850         # test if requested format is supported - else use default
5851         my $supported = grep { $_ eq $format } @$validFormats;
5852         $format = $defFormat if !$supported;
5853
5854         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5855
5856         print "create full clone of drive $drivename ($drive->{file})\n";
5857         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5858         push @$newvollist, $newvolid;
5859
5860         PVE::Storage::activate_volumes($storecfg, $newvollist);
5861
5862         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5863         if (!$running || $snapname) {
5864             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5865         } else {
5866             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5867         }
5868     }
5869
5870     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5871
5872     my $disk = $drive;
5873     $disk->{format} = undef;
5874     $disk->{file} = $newvolid;
5875     $disk->{size} = $size;
5876
5877     return $disk;
5878 }
5879
5880 # this only works if VM is running
5881 sub get_current_qemu_machine {
5882     my ($vmid) = @_;
5883
5884     my $cmd = { execute => 'query-machines', arguments => {} };
5885     my $res = vm_qmp_command($vmid, $cmd);
5886
5887     my ($current, $default);
5888     foreach my $e (@$res) {
5889         $default = $e->{name} if $e->{'is-default'};
5890         $current = $e->{name} if $e->{'is-current'};
5891     }
5892
5893     # fallback to the default machine if current is not supported by qemu
5894     return $current || $default || 'pc';
5895 }
5896
5897 sub qemu_machine_feature_enabled {
5898     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5899
5900     my $current_major;
5901     my $current_minor;
5902
5903     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5904
5905         $current_major = $3;
5906         $current_minor = $4;
5907
5908     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5909
5910         $current_major = $1;
5911         $current_minor = $2;
5912     }
5913
5914     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5915
5916
5917 }
5918
5919 sub qemu_machine_pxe {
5920     my ($vmid, $conf, $machine) = @_;
5921
5922     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5923
5924     foreach my $opt (keys %$conf) {
5925         next if $opt !~ m/^net(\d+)$/;
5926         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5927         next if !$net;
5928         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5929         return $machine.".pxe" if $romfile =~ m/pxe/;
5930         last;
5931     }
5932
5933     return $machine;
5934 }
5935
5936 sub qemu_use_old_bios_files {
5937     my ($machine_type) = @_;
5938
5939     return if !$machine_type;
5940
5941     my $use_old_bios_files = undef;
5942
5943     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5944         $machine_type = $1;
5945         $use_old_bios_files = 1;
5946     } else {
5947         my $kvmver = kvm_user_version();
5948         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5949         # load new efi bios files on migration. So this hack is required to allow
5950         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5951         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5952         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5953     }
5954
5955     return ($use_old_bios_files, $machine_type);
5956 }
5957
5958 sub lspci {
5959
5960     my $devices = {};
5961
5962     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5963             my (undef, $id, $function) = @_;
5964             my $res = { id => $id, function => $function};
5965             push @{$devices->{$id}}, $res;
5966     });
5967
5968     return $devices;
5969 }
5970
5971 sub vm_iothreads_list {
5972     my ($vmid) = @_;
5973
5974     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5975
5976     my $iothreads = {};
5977     foreach my $iothread (@$res) {
5978         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5979     }
5980
5981     return $iothreads;
5982 }
5983
5984 sub scsihw_infos {
5985     my ($conf, $drive) = @_;
5986
5987     my $maxdev = 0;
5988
5989     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5990         $maxdev = 7;
5991     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5992         $maxdev = 1;
5993     } else {
5994         $maxdev = 256;
5995     }
5996
5997     my $controller = int($drive->{index} / $maxdev);
5998     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5999
6000     return ($maxdev, $controller, $controller_prefix);
6001 }
6002
6003 # bash completion helper
6004
6005 sub complete_backup_archives {
6006     my ($cmdname, $pname, $cvalue) = @_;
6007
6008     my $cfg = PVE::Storage::config();
6009
6010     my $storeid;
6011
6012     if ($cvalue =~ m/^([^:]+):/) {
6013         $storeid = $1;
6014     }
6015
6016     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6017
6018     my $res = [];
6019     foreach my $id (keys %$data) {
6020         foreach my $item (@{$data->{$id}}) {
6021             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6022             push @$res, $item->{volid} if defined($item->{volid});
6023         }
6024     }
6025
6026     return $res;
6027 }
6028
6029 my $complete_vmid_full = sub {
6030     my ($running) = @_;
6031
6032     my $idlist = vmstatus();
6033
6034     my $res = [];
6035
6036     foreach my $id (keys %$idlist) {
6037         my $d = $idlist->{$id};
6038         if (defined($running)) {
6039             next if $d->{template};
6040             next if $running && $d->{status} ne 'running';
6041             next if !$running && $d->{status} eq 'running';
6042         }
6043         push @$res, $id;
6044
6045     }
6046     return $res;
6047 };
6048
6049 sub complete_vmid {
6050     return &$complete_vmid_full();
6051 }
6052
6053 sub complete_vmid_stopped {
6054     return &$complete_vmid_full(0);
6055 }
6056
6057 sub complete_vmid_running {
6058     return &$complete_vmid_full(1);
6059 }
6060
6061 sub complete_storage {
6062
6063     my $cfg = PVE::Storage::config();
6064     my $ids = $cfg->{ids};
6065
6066     my $res = [];
6067     foreach my $sid (keys %$ids) {
6068         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6069         next if !$ids->{$sid}->{content}->{images};
6070         push @$res, $sid;
6071     }
6072
6073     return $res;
6074 }
6075
6076 1;