PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35
  36 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  37
  38 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  39
  40 # Note about locking: we use flock on the config file protect
  41 # against concurent actions.
  42 # Aditionaly, we have a 'lock' setting in the config file. This
  43 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  44 # allowed when such lock is set. But you can ignore this kind of
  45 # lock with the --skiplock flag.
  46
  47 cfs_register_file('/qemu-server/',
  48                   \&parse_vm_config,
  49                   \&write_vm_config);
  50
  51 PVE::JSONSchema::register_standard_option('skiplock', {
  52     description => "Ignore locks - only root is allowed to use this option.",
  53     type => 'boolean',
  54     optional => 1,
  55 });
  56
  57 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  58     description => "Some command save/restore state from this location.",
  59     type => 'string',
  60     maxLength => 128,
  61     optional => 1,
  62 });
  63
  64 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  65     description => "The name of the snapshot.",
  66     type => 'string', format => 'pve-configid',
  67     maxLength => 40,
  68 });
  69
  70 #no warnings 'redefine';
  71
  72 sub cgroups_write {
  73    my ($controller, $vmid, $option, $value) = @_;
  74
  75    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  76    PVE::ProcFSTools::write_proc_entry($path, $value);
  77
  78 }
  79
  80 my $nodename = PVE::INotify::nodename();
  81
  82 mkdir "/etc/pve/nodes/$nodename";
  83 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  84 mkdir $confdir;
  85
  86 my $var_run_tmpdir = "/var/run/qemu-server";
  87 mkdir $var_run_tmpdir;
  88
  89 my $lock_dir = "/var/lock/qemu-server";
  90 mkdir $lock_dir;
  91
  92 my $pcisysfs = "/sys/bus/pci";
  93
  94 my $cpudesc = {
  95     cputype => {
  96         description => "Emulated CPU type.",
  97         type => 'string',
  98         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  99         default => 'kvm64',
 100         default_key => 1,
 101     },
 102     hidden => {
 103         description => "Do not identify as a KVM virtual machine.",
 104         type => 'boolean',
 105         optional => 1,
 106         default => 0
 107     },
 108 };
 109
 110 my $confdesc = {
 111     onboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Specifies whether a VM will be started during system bootup.",
 115         default => 0,
 116     },
 117     autostart => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Automatic restart after crash (currently ignored).",
 121         default => 0,
 122     },
 123     hotplug => {
 124         optional => 1,
 125         type => 'string', format => 'pve-hotplug-features',
 126         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 127         default => 'network,disk,usb',
 128     },
 129     reboot => {
 130         optional => 1,
 131         type => 'boolean',
 132         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 133         default => 1,
 134     },
 135     lock => {
 136         optional => 1,
 137         type => 'string',
 138         description => "Lock/unlock the VM.",
 139         enum => [qw(migrate backup snapshot rollback)],
 140     },
 141     cpulimit => {
 142         optional => 1,
 143         type => 'number',
 144         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 145         minimum => 0,
 146         maximum => 128,
 147         default => 0,
 148     },
 149     cpuunits => {
 150         optional => 1,
 151         type => 'integer',
 152         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 153         minimum => 0,
 154         maximum => 500000,
 155         default => 1000,
 156     },
 157     memory => {
 158         optional => 1,
 159         type => 'integer',
 160         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 161         minimum => 16,
 162         default => 512,
 163     },
 164     balloon => {
 165         optional => 1,
 166         type => 'integer',
 167         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 168         minimum => 0,
 169     },
 170     shares => {
 171         optional => 1,
 172         type => 'integer',
 173         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 174         minimum => 0,
 175         maximum => 50000,
 176         default => 1000,
 177     },
 178     keyboard => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 182         enum => PVE::Tools::kvmkeymaplist(),
 183         default => 'en-us',
 184     },
 185     name => {
 186         optional => 1,
 187         type => 'string', format => 'dns-name',
 188         description => "Set a name for the VM. Only used on the configuration web interface.",
 189     },
 190     scsihw => {
 191         optional => 1,
 192         type => 'string',
 193         description => "scsi controller model",
 194         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 195         default => 'lsi',
 196     },
 197     description => {
 198         optional => 1,
 199         type => 'string',
 200         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 201     },
 202     ostype => {
 203         optional => 1,
 204         type => 'string',
 205         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 206         description => <<EODESC,
 207 Used to enable special optimization/features for specific
 208 operating systems:
 209
 210 other  => unspecified OS
 211 wxp    => Microsoft Windows XP
 212 w2k    => Microsoft Windows 2000
 213 w2k3   => Microsoft Windows 2003
 214 w2k8   => Microsoft Windows 2008
 215 wvista => Microsoft Windows Vista
 216 win7   => Microsoft Windows 7
 217 win8   => Microsoft Windows 8/2012
 218 l24    => Linux 2.4 Kernel
 219 l26    => Linux 2.6/3.X Kernel
 220 solaris => solaris/opensolaris/openindiania kernel
 221
 222 other|l24|l26|solaris                       ... no special behaviour
 223 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 224 EODESC
 225     },
 226     boot => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 230         pattern => '[acdn]{1,4}',
 231         default => 'cdn',
 232     },
 233     bootdisk => {
 234         optional => 1,
 235         type => 'string', format => 'pve-qm-bootdisk',
 236         description => "Enable booting from specified disk.",
 237         pattern => '(ide|sata|scsi|virtio)\d+',
 238     },
 239     smp => {
 240         optional => 1,
 241         type => 'integer',
 242         description => "The number of CPUs. Please use option -sockets instead.",
 243         minimum => 1,
 244         default => 1,
 245     },
 246     sockets => {
 247         optional => 1,
 248         type => 'integer',
 249         description => "The number of CPU sockets.",
 250         minimum => 1,
 251         default => 1,
 252     },
 253     cores => {
 254         optional => 1,
 255         type => 'integer',
 256         description => "The number of cores per socket.",
 257         minimum => 1,
 258         default => 1,
 259     },
 260     numa => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Enable/disable NUMA.",
 264         default => 0,
 265     },
 266     vcpus => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "Number of hotplugged vcpus.",
 270         minimum => 1,
 271         default => 0,
 272     },
 273     acpi => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable ACPI.",
 277         default => 1,
 278     },
 279     agent => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Enable/disable Qemu GuestAgent.",
 283         default => 0,
 284     },
 285     kvm => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Enable/disable KVM hardware virtualization.",
 289         default => 1,
 290     },
 291     tdf => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Enable/disable time drift fix.",
 295         default => 0,
 296     },
 297     localtime => {
 298         optional => 1,
 299         type => 'boolean',
 300         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 301     },
 302     freeze => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 306     },
 307     vga => {
 308         optional => 1,
 309         type => 'string',
 310         description => "Select the VGA type. If you want to use high resolution" .
 311             " modes (>= 1280x1024x16) then you should use the options " .
 312             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 313             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 314             "display sever. For win* OS you can select how many independent " .
 315             "displays you want, Linux guests can add displays them self. " .
 316             "You can also run without any graphic card, using a serial device" .
 317             " as terminal.",
 318         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 319     },
 320     watchdog => {
 321         optional => 1,
 322         type => 'string', format => 'pve-qm-watchdog',
 323         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 324         description => "Create a virtual hardware watchdog device. Once enabled" .
 325             " (by a guest action), the watchdog must be periodically polled " .
 326             "by an agent inside the guest or else the watchdog will reset " .
 327             "the guest (or execute the respective action specified)",
 328     },
 329     startdate => {
 330         optional => 1,
 331         type => 'string',
 332         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 333         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 334         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 335         default => 'now',
 336     },
 337     startup =>  get_standard_option('pve-startup-order'),
 338     template => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Enable/disable Template.",
 342         default => 0,
 343     },
 344     args => {
 345         optional => 1,
 346         type => 'string',
 347         description => <<EODESCR,
 348 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 349
 350 args: -no-reboot -no-hpet
 351 EODESCR
 352     },
 353     tablet => {
 354         optional => 1,
 355         type => 'boolean',
 356         default => 1,
 357         description => "Enable/disable the USB tablet device. This device is " .
 358             "usually needed to allow absolute mouse positioning with VNC. " .
 359             "Else the mouse runs out of sync with normal VNC clients. " .
 360             "If you're running lots of console-only guests on one host, " .
 361             "you may consider disabling this to save some context switches. " .
 362             "This is turned off by default if you use spice (-vga=qxl).",
 363     },
 364     migrate_speed => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 368         minimum => 0,
 369         default => 0,
 370     },
 371     migrate_downtime => {
 372         optional => 1,
 373         type => 'number',
 374         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 375         minimum => 0,
 376         default => 0.1,
 377     },
 378     cdrom => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-drive',
 381         typetext => 'volume',
 382         description => "This is an alias for option -ide2",
 383     },
 384     cpu => {
 385         optional => 1,
 386         description => "Emulated CPU type.",
 387         type => 'string',
 388         format => $cpudesc,
 389     },
 390     parent => get_standard_option('pve-snapshot-name', {
 391         optional => 1,
 392         description => "Parent snapshot name. This is used internally, and should not be modified.",
 393     }),
 394     snaptime => {
 395         optional => 1,
 396         description => "Timestamp for snapshots.",
 397         type => 'integer',
 398         minimum => 0,
 399     },
 400     vmstate => {
 401         optional => 1,
 402         type => 'string', format => 'pve-volume-id',
 403         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 404     },
 405     machine => {
 406         description => "Specific the Qemu machine type.",
 407         type => 'string',
 408         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 409         maxLength => 40,
 410         optional => 1,
 411     },
 412     smbios1 => {
 413         description => "Specify SMBIOS type 1 fields.",
 414         type => 'string', format => 'pve-qm-smbios1',
 415         maxLength => 256,
 416         optional => 1,
 417     },
 418     protection => {
 419         optional => 1,
 420         type => 'boolean',
 421         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 422         default => 0,
 423     },
 424     bios => {
 425         optional => 1,
 426         type => 'string',
 427         enum => [ qw(seabios ovmf) ],
 428         description => "Select BIOS implementation.",
 429         default => 'seabios',
 430     },
 431 };
 432
 433 # what about other qemu settings ?
 434 #cpu => 'string',
 435 #machine => 'string',
 436 #fda => 'file',
 437 #fdb => 'file',
 438 #mtdblock => 'file',
 439 #sd => 'file',
 440 #pflash => 'file',
 441 #snapshot => 'bool',
 442 #bootp => 'file',
 443 ##tftp => 'dir',
 444 ##smb => 'dir',
 445 #kernel => 'file',
 446 #append => 'string',
 447 #initrd => 'file',
 448 ##soundhw => 'string',
 449
 450 while (my ($k, $v) = each %$confdesc) {
 451     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 452 }
 453
 454 my $MAX_IDE_DISKS = 4;
 455 my $MAX_SCSI_DISKS = 14;
 456 my $MAX_VIRTIO_DISKS = 16;
 457 my $MAX_SATA_DISKS = 6;
 458 my $MAX_USB_DEVICES = 5;
 459 my $MAX_NETS = 32;
 460 my $MAX_UNUSED_DISKS = 8;
 461 my $MAX_HOSTPCI_DEVICES = 4;
 462 my $MAX_SERIAL_PORTS = 4;
 463 my $MAX_PARALLEL_PORTS = 3;
 464 my $MAX_NUMA = 8;
 465 my $MAX_MEM = 4194304;
 466 my $STATICMEM = 1024;
 467
 468 my $numadesc = {
 469     optional => 1,
 470     type => 'string', format => 'pve-qm-numanode',
 471     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 472     description => "numa topology",
 473 };
 474 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 475
 476 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 477     $confdesc->{"numa$i"} = $numadesc;
 478 }
 479
 480 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 481                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 482                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 483 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 484
 485 my $netdesc = {
 486     optional => 1,
 487     type => 'string', format => 'pve-qm-net',
 488     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 489     description => <<EODESCR,
 490 Specify network devices.
 491
 492 MODEL is one of: $nic_model_list_txt
 493
 494 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 495 automatically generated if not specified.
 496
 497 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 498
 499 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 500
 501 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 502
 503 10.0.2.2   Gateway
 504 10.0.2.3   DNS Server
 505 10.0.2.4   SMB Server
 506
 507 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 508
 509 EODESCR
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 512
 513 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 514     $confdesc->{"net$i"} = $netdesc;
 515 }
 516
 517 my $drivename_hash;
 518
 519 my %drivedesc_base = (
 520     volume => { alias => 'file' },
 521     file => {
 522         type => 'pve-volume-id',
 523         default_key => 1,
 524         format_description => 'volume',
 525         description => "The drive's backing volume.",
 526     },
 527     media => {
 528         type => 'string',
 529         format_description => 'cdrom|disk',
 530         enum => [qw(cdrom disk)],
 531         description => "The drive's media type.",
 532         default => 'disk',
 533         optional => 1
 534     },
 535     cyls => {
 536         type => 'integer',
 537         format_description => 'count',
 538         description => "Force the drive's physical geometry to have a specific cylinder count.",
 539         optional => 1
 540     },
 541     heads => {
 542         type => 'integer',
 543         format_description => 'count',
 544         description => "Force the drive's physical geometry to have a specific head count.",
 545         optional => 1
 546     },
 547     secs => {
 548         type => 'integer',
 549         format_description => 'count',
 550         description => "Force the drive's physical geometry to have a specific sector count.",
 551         optional => 1
 552     },
 553     trans => {
 554         type => 'string',
 555         format_description => 'none|lba|auto',
 556         enum => [qw(none lba auto)],
 557         description => "Force disk geometry bios translation mode.",
 558         optional => 1,
 559     },
 560     snapshot => {
 561         type => 'boolean',
 562         format_description => 'on|off',
 563         description => "Whether the drive should be included when making snapshots.",
 564         optional => 1,
 565     },
 566     cache => {
 567         type => 'string',
 568         format_description => 'none|writethrough|writeback|unsafe|directsync',
 569         enum => [qw(none writethrough writeback unsafe directsync)],
 570         description => "The drive's cache mode",
 571         optional => 1,
 572     },
 573     format => {
 574         type => 'string',
 575         format_description => 'drive format',
 576         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 577         description => "The drive's backing file's data format.",
 578         optional => 1,
 579     },
 580     size => {
 581         type => 'string',
 582         format => 'disk-size',
 583         description => "Disk size. This is purely informational and has no effect.",
 584         optional => 1,
 585     },
 586     backup => {
 587         type => 'boolean',
 588         format_description => 'on|off',
 589         description => "Whether the drive should be included when making backups.",
 590         optional => 1,
 591     },
 592     werror => {
 593         type => 'string',
 594         format_description => 'enospc|ignore|report|stop',
 595         enum => [qw(enospc ignore report stop)],
 596         description => 'Write error action.',
 597         optional => 1,
 598     },
 599     aio => {
 600         type => 'string',
 601         format_description => 'native|threads',
 602         enum => [qw(native threads)],
 603         description => 'AIO type to use.',
 604         optional => 1,
 605     },
 606     discard => {
 607         type => 'string',
 608         format_description => 'ignore|on',
 609         enum => [qw(ignore on)],
 610         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 611         optional => 1,
 612     },
 613     detect_zeroes => {
 614         type => 'boolean',
 615         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 616         optional => 1,
 617     },
 618     serial => {
 619         type => 'string',
 620         format_description => 'serial',
 621         description => "The drive's reported serial number.",
 622         optional => 1,
 623     }
 624 );
 625
 626 my %rerror_fmt = (
 627     rerror => {
 628         type => 'string',
 629         format_description => 'ignore|report|stop',
 630         enum => [qw(ignore report stop)],
 631         description => 'Read error action.',
 632         optional => 1,
 633     },
 634 );
 635
 636 my %iothread_fmt = ( iothread => {
 637         type => 'boolean',
 638         format_description => 'off|on',
 639         description => "Whether to use iothreads for this drive",
 640         optional => 1,
 641 });
 642
 643 my %model_fmt = (
 644     model => {
 645         type => 'string',
 646         format_description => 'model',
 647         description => "The drive's reported model name.",
 648         optional => 1,
 649     },
 650 );
 651
 652 my %queues_fmt = (
 653     queues => {
 654         type => 'integer',
 655         format_description => 'nbqueues',
 656         description => "Number of queues.",
 657         minimum => 2,
 658         optional => 1
 659     }
 660 );
 661
 662 my $add_throttle_desc = sub {
 663     my ($key, $type, $what, $size, $longsize) = @_;
 664     $drivedesc_base{$key} = {
 665         type => $type,
 666         format_description => $size,
 667         description => "Maximum $what speed in $longsize per second.",
 668         optional => 1,
 669     };
 670 };
 671 # throughput: (leaky bucket)
 672 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 673 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 674 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 675 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 676 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 677 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 678 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 679 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 680 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 681
 682 # pools: (pool of IO before throttling starts taking effect)
 683 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 684 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 685 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 686 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 687 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 688 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 689
 690 my $ide_fmt = {
 691     %drivedesc_base,
 692     %rerror_fmt,
 693     %model_fmt,
 694 };
 695
 696 my $idedesc = {
 697     optional => 1,
 698     type => 'string', format => $ide_fmt,
 699     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 700 };
 701 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 702
 703 my $scsi_fmt = {
 704     %drivedesc_base,
 705     %iothread_fmt,
 706     %queues_fmt,
 707 };
 708 my $scsidesc = {
 709     optional => 1,
 710     type => 'string', format => $scsi_fmt,
 711     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 712 };
 713 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 714
 715 my $sata_fmt = {
 716     %drivedesc_base,
 717     %rerror_fmt,
 718 };
 719 my $satadesc = {
 720     optional => 1,
 721     type => 'string', format => $sata_fmt,
 722     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 723 };
 724 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 725
 726 my $virtio_fmt = {
 727     %drivedesc_base,
 728     %iothread_fmt,
 729     %rerror_fmt,
 730 };
 731 my $virtiodesc = {
 732     optional => 1,
 733     type => 'string', format => $virtio_fmt,
 734     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 735 };
 736 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 737
 738 my $alldrive_fmt = {
 739     %drivedesc_base,
 740     %rerror_fmt,
 741     %iothread_fmt,
 742     %model_fmt,
 743     %queues_fmt,
 744 };
 745
 746 my $usbformat = {
 747     host => {
 748         default_key => 1,
 749         type => 'string', format => 'pve-qm-usb-device',
 750         format_description => 'HOSTUSBDEVICE|spice',
 751         description => 'The Host USB device or port or the value spice',
 752     },
 753     usb3 => {
 754         optional => 1,
 755         type => 'boolean',
 756         format_description => 'yes|no',
 757         description => 'Specifies whether if given host option is a USB3 device or port',
 758     },
 759 };
 760
 761 my $usbdesc = {
 762     optional => 1,
 763     type => 'string', format => $usbformat,
 764     description => <<EODESCR,
 765 Configure an USB device (n is 0 to 4). This can be used to
 766 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 767
 768 'bus-port(.port)*' (decimal numbers) or
 769 'vendor_id:product_id' (hexadeciaml numbers) or
 770 'spice'
 771
 772 You can use the 'lsusb -t' command to list existing usb devices.
 773
 774 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 775
 776 The value 'spice' can be used to add a usb redirection devices for spice.
 777
 778 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 779
 780 EODESCR
 781 };
 782 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 783
 784 my $hostpcidesc = {
 785         optional => 1,
 786         type => 'string', format => 'pve-qm-hostpci',
 787         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 788         description => <<EODESCR,
 789 Map host pci devices. HOSTPCIDEVICE syntax is:
 790
 791 'bus:dev.func' (hexadecimal numbers)
 792
 793 You can us the 'lspci' command to list existing pci devices.
 794
 795 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 796
 797 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 798
 799 Experimental: user reported problems with this option.
 800 EODESCR
 801 };
 802 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 803
 804 my $serialdesc = {
 805         optional => 1,
 806         type => 'string',
 807         pattern => '(/dev/.+|socket)',
 808         description =>  <<EODESCR,
 809 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 810
 811 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 812
 813 Experimental: user reported problems with this option.
 814 EODESCR
 815 };
 816
 817 my $paralleldesc= {
 818         optional => 1,
 819         type => 'string',
 820         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 821         description =>  <<EODESCR,
 822 Map host parallel devices (n is 0 to 2).
 823
 824 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 825
 826 Experimental: user reported problems with this option.
 827 EODESCR
 828 };
 829
 830 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 831     $confdesc->{"parallel$i"} = $paralleldesc;
 832 }
 833
 834 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 835     $confdesc->{"serial$i"} = $serialdesc;
 836 }
 837
 838 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 839     $confdesc->{"hostpci$i"} = $hostpcidesc;
 840 }
 841
 842 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 843     $drivename_hash->{"ide$i"} = 1;
 844     $confdesc->{"ide$i"} = $idedesc;
 845 }
 846
 847 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 848     $drivename_hash->{"sata$i"} = 1;
 849     $confdesc->{"sata$i"} = $satadesc;
 850 }
 851
 852 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 853     $drivename_hash->{"scsi$i"} = 1;
 854     $confdesc->{"scsi$i"} = $scsidesc ;
 855 }
 856
 857 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 858     $drivename_hash->{"virtio$i"} = 1;
 859     $confdesc->{"virtio$i"} = $virtiodesc;
 860 }
 861
 862 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 863     $confdesc->{"usb$i"} = $usbdesc;
 864 }
 865
 866 my $unuseddesc = {
 867     optional => 1,
 868     type => 'string', format => 'pve-volume-id',
 869     description => "Reference to unused volumes.",
 870 };
 871
 872 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 873     $confdesc->{"unused$i"} = $unuseddesc;
 874 }
 875
 876 my $kvm_api_version = 0;
 877
 878 sub kvm_version {
 879
 880     return $kvm_api_version if $kvm_api_version;
 881
 882     my $fh = IO::File->new("</dev/kvm") ||
 883         return 0;
 884
 885     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 886         $kvm_api_version = $v;
 887     }
 888
 889     $fh->close();
 890
 891     return  $kvm_api_version;
 892 }
 893
 894 my $kvm_user_version;
 895
 896 sub kvm_user_version {
 897
 898     return $kvm_user_version if $kvm_user_version;
 899
 900     $kvm_user_version = 'unknown';
 901
 902     my $code = sub {
 903         my $line = shift;
 904         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 905             $kvm_user_version = $2;
 906         }
 907     };
 908
 909     eval { run_command("kvm -version", outfunc => $code); };
 910     warn $@ if $@;
 911
 912     return $kvm_user_version;
 913
 914 }
 915
 916 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 917
 918 sub valid_drive_names {
 919     # order is important - used to autoselect boot disk
 920     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 921             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 922             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 923             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 924 }
 925
 926 sub is_valid_drivename {
 927     my $dev = shift;
 928
 929     return defined($drivename_hash->{$dev});
 930 }
 931
 932 sub option_exists {
 933     my $key = shift;
 934     return defined($confdesc->{$key});
 935 }
 936
 937 sub nic_models {
 938     return $nic_model_list;
 939 }
 940
 941 sub os_list_description {
 942
 943     return {
 944         other => 'Other',
 945         wxp => 'Windows XP',
 946         w2k => 'Windows 2000',
 947         w2k3 =>, 'Windows 2003',
 948         w2k8 => 'Windows 2008',
 949         wvista => 'Windows Vista',
 950         win7 => 'Windows 7',
 951         win8 => 'Windows 8/2012',
 952         l24 => 'Linux 2.4',
 953         l26 => 'Linux 2.6',
 954     };
 955 }
 956
 957 my $cdrom_path;
 958
 959 sub get_cdrom_path {
 960
 961     return  $cdrom_path if $cdrom_path;
 962
 963     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 964     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 965     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 966 }
 967
 968 sub get_iso_path {
 969     my ($storecfg, $vmid, $cdrom) = @_;
 970
 971     if ($cdrom eq 'cdrom') {
 972         return get_cdrom_path();
 973     } elsif ($cdrom eq 'none') {
 974         return '';
 975     } elsif ($cdrom =~ m|^/|) {
 976         return $cdrom;
 977     } else {
 978         return PVE::Storage::path($storecfg, $cdrom);
 979     }
 980 }
 981
 982 # try to convert old style file names to volume IDs
 983 sub filename_to_volume_id {
 984     my ($vmid, $file, $media) = @_;
 985
 986     if (!($file eq 'none' || $file eq 'cdrom' ||
 987           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 988
 989         return undef if $file =~ m|/|;
 990
 991         if ($media && $media eq 'cdrom') {
 992             $file = "local:iso/$file";
 993         } else {
 994             $file = "local:$vmid/$file";
 995         }
 996     }
 997
 998     return $file;
 999 }
1000
1001 sub verify_media_type {
1002     my ($opt, $vtype, $media) = @_;
1003
1004     return if !$media;
1005
1006     my $etype;
1007     if ($media eq 'disk') {
1008         $etype = 'images';
1009     } elsif ($media eq 'cdrom') {
1010         $etype = 'iso';
1011     } else {
1012         die "internal error";
1013     }
1014
1015     return if ($vtype eq $etype);
1016
1017     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1018 }
1019
1020 sub cleanup_drive_path {
1021     my ($opt, $storecfg, $drive) = @_;
1022
1023     # try to convert filesystem paths to volume IDs
1024
1025     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1026         ($drive->{file} !~ m|^/dev/.+|) &&
1027         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1028         ($drive->{file} !~ m/^\d+$/)) {
1029         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1030         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1031         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1032         verify_media_type($opt, $vtype, $drive->{media});
1033         $drive->{file} = $volid;
1034     }
1035
1036     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1037 }
1038
1039 sub parse_hotplug_features {
1040     my ($data) = @_;
1041
1042     my $res = {};
1043
1044     return $res if $data eq '0';
1045
1046     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1047
1048     foreach my $feature (PVE::Tools::split_list($data)) {
1049         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1050             $res->{$1} = 1;
1051         } else {
1052             warn "ignoring unknown hotplug feature '$feature'\n";
1053         }
1054     }
1055     return $res;
1056 }
1057
1058 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1059 sub pve_verify_hotplug_features {
1060     my ($value, $noerr) = @_;
1061
1062     return $value if parse_hotplug_features($value);
1063
1064     return undef if $noerr;
1065
1066     die "unable to parse hotplug option\n";
1067 }
1068
1069 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1070 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1071 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1072 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1073 #        [,iothread=on][,serial=serial][,model=model]
1074
1075 sub parse_drive {
1076     my ($key, $data) = @_;
1077
1078     my ($interface, $index);
1079
1080     if ($key =~ m/^([^\d]+)(\d+)$/) {
1081         $interface = $1;
1082         $index = $2;
1083     } else {
1084         return undef;
1085     }
1086
1087     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1088                                      : $confdesc->{$key}->{format};
1089     if (!$desc) {
1090         warn "invalid drive key: $key\n";
1091         return undef;
1092     }
1093     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1094     return undef if !$res;
1095     $res->{interface} = $interface;
1096     $res->{index} = $index;
1097
1098     my $error = 0;
1099     foreach my $opt (qw(bps bps_rd bps_wr)) {
1100         if (my $bps = defined(delete $res->{$opt})) {
1101             if (defined($res->{"m$opt"})) {
1102                 warn "both $opt and m$opt specified\n";
1103                 ++$error;
1104                 next;
1105             }
1106             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1107         }
1108     }
1109     return undef if $error;
1110
1111     return undef if $res->{mbps_rd} && $res->{mbps};
1112     return undef if $res->{mbps_wr} && $res->{mbps};
1113     return undef if $res->{iops_rd} && $res->{iops};
1114     return undef if $res->{iops_wr} && $res->{iops};
1115
1116     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1117         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1118         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1119         return undef if $res->{interface} eq 'virtio';
1120     }
1121
1122     if (my $size = $res->{size}) {
1123         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1124     }
1125
1126     return $res;
1127 }
1128
1129 sub print_drive {
1130     my ($vmid, $drive) = @_;
1131     my $data = { %$drive };
1132     delete $data->{$_} for qw(index interface);
1133     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1134 }
1135
1136 sub scsi_inquiry {
1137     my($fh, $noerr) = @_;
1138
1139     my $SG_IO = 0x2285;
1140     my $SG_GET_VERSION_NUM = 0x2282;
1141
1142     my $versionbuf = "\x00" x 8;
1143     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1144     if (!$ret) {
1145         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1146         return undef;
1147     }
1148     my $version = unpack("I", $versionbuf);
1149     if ($version < 30000) {
1150         die "scsi generic interface too old\n"  if !$noerr;
1151         return undef;
1152     }
1153
1154     my $buf = "\x00" x 36;
1155     my $sensebuf = "\x00" x 8;
1156     my $cmd = pack("C x3 C x1", 0x12, 36);
1157
1158     # see /usr/include/scsi/sg.h
1159     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1160
1161     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1162                       length($sensebuf), 0, length($buf), $buf,
1163                       $cmd, $sensebuf, 6000);
1164
1165     $ret = ioctl($fh, $SG_IO, $packet);
1166     if (!$ret) {
1167         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1168         return undef;
1169     }
1170
1171     my @res = unpack($sg_io_hdr_t, $packet);
1172     if ($res[17] || $res[18]) {
1173         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1174         return undef;
1175     }
1176
1177     my $res = {};
1178     (my $byte0, my $byte1, $res->{vendor},
1179      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1180
1181     $res->{removable} = $byte1 & 128 ? 1 : 0;
1182     $res->{type} = $byte0 & 31;
1183
1184     return $res;
1185 }
1186
1187 sub path_is_scsi {
1188     my ($path) = @_;
1189
1190     my $fh = IO::File->new("+<$path") || return undef;
1191     my $res = scsi_inquiry($fh, 1);
1192     close($fh);
1193
1194     return $res;
1195 }
1196
1197 sub machine_type_is_q35 {
1198     my ($conf) = @_;
1199
1200     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1201 }
1202
1203 sub print_tabletdevice_full {
1204     my ($conf) = @_;
1205
1206     my $q35 = machine_type_is_q35($conf);
1207
1208     # we use uhci for old VMs because tablet driver was buggy in older qemu
1209     my $usbbus = $q35 ? "ehci" : "uhci";
1210
1211     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1212 }
1213
1214 sub print_drivedevice_full {
1215     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1216
1217     my $device = '';
1218     my $maxdev = 0;
1219
1220     if ($drive->{interface} eq 'virtio') {
1221         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1222         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1223         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1224     } elsif ($drive->{interface} eq 'scsi') {
1225
1226         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1227         my $unit = $drive->{index} % $maxdev;
1228         my $devicetype = 'hd';
1229         my $path = '';
1230         if (drive_is_cdrom($drive)) {
1231             $devicetype = 'cd';
1232         } else {
1233             if ($drive->{file} =~ m|^/|) {
1234                 $path = $drive->{file};
1235                 if (my $info = path_is_scsi($path)) {
1236                     if ($info->{type} == 0) {
1237                         $devicetype = 'block';
1238                     } elsif ($info->{type} == 1) { # tape
1239                         $devicetype = 'generic';
1240                     }
1241                 }
1242             } else {
1243                  $path = PVE::Storage::path($storecfg, $drive->{file});
1244             }
1245
1246             if($path =~ m/^iscsi\:\/\//){
1247                 $devicetype = 'generic';
1248             }
1249         }
1250
1251         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1252             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1253         } else {
1254             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1255         }
1256
1257     } elsif ($drive->{interface} eq 'ide'){
1258         $maxdev = 2;
1259         my $controller = int($drive->{index} / $maxdev);
1260         my $unit = $drive->{index} % $maxdev;
1261         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1262
1263         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1264         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1265             $device .= ",model=$model";
1266         }
1267     } elsif ($drive->{interface} eq 'sata'){
1268         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1269         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1270         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1271     } elsif ($drive->{interface} eq 'usb') {
1272         die "implement me";
1273         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1274     } else {
1275         die "unsupported interface type";
1276     }
1277
1278     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1279
1280     return $device;
1281 }
1282
1283 sub get_initiator_name {
1284     my $initiator;
1285
1286     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1287     while (defined(my $line = <$fh>)) {
1288         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1289         $initiator = $1;
1290         last;
1291     }
1292     $fh->close();
1293
1294     return $initiator;
1295 }
1296
1297 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1298 sub print_drive_full {
1299     my ($storecfg, $vmid, $drive) = @_;
1300
1301     my $path;
1302     my $volid = $drive->{file};
1303     my $format;
1304
1305     if (drive_is_cdrom($drive)) {
1306         $path = get_iso_path($storecfg, $vmid, $volid);
1307     } else {
1308         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1309         if ($storeid) {
1310             $path = PVE::Storage::path($storecfg, $volid);
1311             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1312             $format = qemu_img_format($scfg, $volname);
1313         } else {
1314             $path = $volid;
1315             $format = "raw";
1316         }
1317    }
1318
1319     my $opts = '';
1320     foreach my $o (@qemu_drive_options) {
1321         next if $o eq 'bootindex';
1322         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1323     }
1324
1325     $opts .= ",format=$format" if $format && !$drive->{format};
1326
1327     foreach my $o (qw(bps bps_rd bps_wr)) {
1328         my $v = $drive->{"m$o"};
1329         $opts .= ",$o=" . int($v*1024*1024) if $v;
1330     }
1331
1332     my $cache_direct = 0;
1333
1334     if (my $cache = $drive->{cache}) {
1335         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1336     } elsif (!drive_is_cdrom($drive)) {
1337         $opts .= ",cache=none";
1338         $cache_direct = 1;
1339     }
1340
1341     # aio native works only with O_DIRECT
1342     if (!$drive->{aio}) {
1343         if($cache_direct) {
1344             $opts .= ",aio=native";
1345         } else {
1346             $opts .= ",aio=threads";
1347         }
1348     }
1349
1350     if (!drive_is_cdrom($drive)) {
1351         my $detectzeroes;
1352         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1353             $detectzeroes = 'off';
1354         } elsif ($drive->{discard}) {
1355             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1356         } else {
1357             # This used to be our default with discard not being specified:
1358             $detectzeroes = 'on';
1359         }
1360         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1361     }
1362
1363     my $pathinfo = $path ? "file=$path," : '';
1364
1365     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1366 }
1367
1368 sub print_netdevice_full {
1369     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1370
1371     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1372
1373     my $device = $net->{model};
1374     if ($net->{model} eq 'virtio') {
1375          $device = 'virtio-net-pci';
1376      };
1377
1378     my $pciaddr = print_pci_addr("$netid", $bridges);
1379     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1380     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1381         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1382         my $vectors = $net->{queues} * 2 + 2;
1383         $tmpstr .= ",vectors=$vectors,mq=on";
1384     }
1385     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1386
1387     if ($use_old_bios_files) {
1388         my $romfile;
1389         if ($device eq 'virtio-net-pci') {
1390             $romfile = 'pxe-virtio.rom';
1391         } elsif ($device eq 'e1000') {
1392             $romfile = 'pxe-e1000.rom';
1393         } elsif ($device eq 'ne2k') {
1394             $romfile = 'pxe-ne2k_pci.rom';
1395         } elsif ($device eq 'pcnet') {
1396             $romfile = 'pxe-pcnet.rom';
1397         } elsif ($device eq 'rtl8139') {
1398             $romfile = 'pxe-rtl8139.rom';
1399         }
1400         $tmpstr .= ",romfile=$romfile" if $romfile;
1401     }
1402
1403     return $tmpstr;
1404 }
1405
1406 sub print_netdev_full {
1407     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1408
1409     my $i = '';
1410     if ($netid =~ m/^net(\d+)$/) {
1411         $i = int($1);
1412     }
1413
1414     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1415
1416     my $ifname = "tap${vmid}i$i";
1417
1418     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1419     die "interface name '$ifname' is too long (max 15 character)\n"
1420         if length($ifname) >= 16;
1421
1422     my $vhostparam = '';
1423     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1424
1425     my $vmname = $conf->{name} || "vm$vmid";
1426
1427     my $netdev = "";
1428     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1429
1430     if ($net->{bridge}) {
1431         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1432     } else {
1433         $netdev = "type=user,id=$netid,hostname=$vmname";
1434     }
1435
1436     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1437
1438     return $netdev;
1439 }
1440
1441 sub drive_is_cdrom {
1442     my ($drive) = @_;
1443
1444     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1445
1446 }
1447
1448 sub parse_numa {
1449     my ($data) = @_;
1450
1451     my $res = {};
1452
1453     foreach my $kvp (split(/,/, $data)) {
1454
1455         if ($kvp =~ m/^memory=(\S+)$/) {
1456             $res->{memory} = $1;
1457         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1458             $res->{policy} = $1;
1459         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1460             $res->{cpus}->{start} = $1;
1461             $res->{cpus}->{end} = $3;
1462         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1463             $res->{hostnodes}->{start} = $1;
1464             $res->{hostnodes}->{end} = $3;
1465         } else {
1466             return undef;
1467         }
1468     }
1469
1470     return $res;
1471 }
1472
1473 sub parse_hostpci {
1474     my ($value) = @_;
1475
1476     return undef if !$value;
1477
1478
1479     my @list = split(/,/, $value);
1480     my $found;
1481
1482     my $res = {};
1483     foreach my $kv (@list) {
1484
1485         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1486             $found = 1;
1487             if(defined($4)){
1488                 push @{$res->{pciid}}, { id => $2 , function => $4};
1489
1490             }else{
1491                 my $pcidevices = lspci($2);
1492                 $res->{pciid} = $pcidevices->{$2};
1493             }
1494         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1495             $res->{rombar} = $1;
1496         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1497             $res->{'x-vga'} = $1;
1498         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1499             $res->{pcie} = 1 if $1 == 1;
1500         } else {
1501             warn "unknown hostpci setting '$kv'\n";
1502         }
1503     }
1504
1505     return undef if !$found;
1506
1507     return $res;
1508 }
1509
1510 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1511 sub parse_net {
1512     my ($data) = @_;
1513
1514     my $res = {};
1515
1516     foreach my $kvp (split(/,/, $data)) {
1517
1518         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1519             my $model = lc($1);
1520             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1521             $res->{model} = $model;
1522             $res->{macaddr} = $mac;
1523         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1524             $res->{bridge} = $1;
1525         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1526             $res->{queues} = $1;
1527         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1528             $res->{rate} = $1;
1529         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1530             $res->{tag} = $1;
1531         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1532             $res->{trunks} = $1;
1533         } elsif ($kvp =~ m/^firewall=([01])$/) {
1534             $res->{firewall} = $1;
1535         } elsif ($kvp =~ m/^link_down=([01])$/) {
1536             $res->{link_down} = $1;
1537         } else {
1538             return undef;
1539         }
1540
1541     }
1542
1543     return undef if !$res->{model};
1544
1545     return $res;
1546 }
1547
1548 sub print_net {
1549     my $net = shift;
1550
1551     my $res = "$net->{model}";
1552     $res .= "=$net->{macaddr}" if $net->{macaddr};
1553     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1554     $res .= ",rate=$net->{rate}" if $net->{rate};
1555     $res .= ",tag=$net->{tag}" if $net->{tag};
1556     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1557     $res .= ",firewall=1" if $net->{firewall};
1558     $res .= ",link_down=1" if $net->{link_down};
1559     $res .= ",queues=$net->{queues}" if $net->{queues};
1560
1561     return $res;
1562 }
1563
1564 sub add_random_macs {
1565     my ($settings) = @_;
1566
1567     foreach my $opt (keys %$settings) {
1568         next if $opt !~ m/^net(\d+)$/;
1569         my $net = parse_net($settings->{$opt});
1570         next if !$net;
1571         $settings->{$opt} = print_net($net);
1572     }
1573 }
1574
1575 sub vm_is_volid_owner {
1576     my ($storecfg, $vmid, $volid) = @_;
1577
1578     if ($volid !~  m|^/|) {
1579         my ($path, $owner);
1580         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1581         if ($owner && ($owner == $vmid)) {
1582             return 1;
1583         }
1584     }
1585
1586     return undef;
1587 }
1588
1589 sub split_flagged_list {
1590     my $text = shift || '';
1591     $text =~ s/[,;]/ /g;
1592     $text =~ s/^\s+//;
1593     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1594 }
1595
1596 sub join_flagged_list {
1597     my ($how, $lst) = @_;
1598     join $how, map { $lst->{$_} . $_ } keys %$lst;
1599 }
1600
1601 sub vmconfig_delete_pending_option {
1602     my ($conf, $key, $force) = @_;
1603
1604     delete $conf->{pending}->{$key};
1605     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1606     $pending_delete_hash->{$key} = $force ? '!' : '';
1607     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1608 }
1609
1610 sub vmconfig_undelete_pending_option {
1611     my ($conf, $key) = @_;
1612
1613     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1614     delete $pending_delete_hash->{$key};
1615
1616     if (%$pending_delete_hash) {
1617         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1618     } else {
1619         delete $conf->{pending}->{delete};
1620     }
1621 }
1622
1623 sub vmconfig_register_unused_drive {
1624     my ($storecfg, $vmid, $conf, $drive) = @_;
1625
1626     if (!drive_is_cdrom($drive)) {
1627         my $volid = $drive->{file};
1628         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1629             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1630         }
1631     }
1632 }
1633
1634 sub vmconfig_cleanup_pending {
1635     my ($conf) = @_;
1636
1637     # remove pending changes when nothing changed
1638     my $changes;
1639     foreach my $opt (keys %{$conf->{pending}}) {
1640         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1641             $changes = 1;
1642             delete $conf->{pending}->{$opt};
1643         }
1644     }
1645
1646     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1647     my $pending_delete_hash = {};
1648     while (my ($opt, $force) = each %$current_delete_hash) {
1649         if (defined($conf->{$opt})) {
1650             $pending_delete_hash->{$opt} = $force;
1651         } else {
1652             $changes = 1;
1653         }
1654     }
1655
1656     if (%$pending_delete_hash) {
1657         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1658     } else {
1659         delete $conf->{pending}->{delete};
1660     }
1661
1662     return $changes;
1663 }
1664
1665 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1666 my $smbios1_desc = {
1667     uuid => {
1668         type => 'string',
1669         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1670         format_description => 'UUID',
1671         optional => 1,
1672     },
1673     version => {
1674         type => 'string',
1675         pattern => '\S+',
1676         format_description => 'str',
1677         optional => 1,
1678     },
1679     serial => {
1680         type => 'string',
1681         pattern => '\S+',
1682         format_description => 'str',
1683         optional => 1,
1684     },
1685     manufacturer => {
1686         type => 'string',
1687         pattern => '\S+',
1688         format_description => 'name',
1689         optional => 1,
1690     },
1691     product => {
1692         type => 'string',
1693         pattern => '\S+',
1694         format_description => 'name',
1695         optional => 1,
1696     },
1697     sku => {
1698         type => 'string',
1699         pattern => '\S+',
1700         format_description => 'str',
1701         optional => 1,
1702     },
1703     family => {
1704         type => 'string',
1705         pattern => '\S+',
1706         format_description => 'str',
1707         optional => 1,
1708     },
1709 };
1710
1711 sub parse_smbios1 {
1712     my ($data) = @_;
1713
1714     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1715     warn $@ if $@;
1716     return $res;
1717 }
1718
1719 sub print_smbios1 {
1720     my ($smbios1) = @_;
1721     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1722 }
1723
1724 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1725
1726 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1727 sub verify_bootdisk {
1728     my ($value, $noerr) = @_;
1729
1730     return $value if is_valid_drivename($value);
1731
1732     return undef if $noerr;
1733
1734     die "invalid boot disk '$value'\n";
1735 }
1736
1737 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1738 sub verify_numa {
1739     my ($value, $noerr) = @_;
1740
1741     return $value if parse_numa($value);
1742
1743     return undef if $noerr;
1744
1745     die "unable to parse numa options\n";
1746 }
1747
1748 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1749 sub verify_net {
1750     my ($value, $noerr) = @_;
1751
1752     return $value if parse_net($value);
1753
1754     return undef if $noerr;
1755
1756     die "unable to parse network options\n";
1757 }
1758
1759 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1760 sub verify_hostpci {
1761     my ($value, $noerr) = @_;
1762
1763     return $value if parse_hostpci($value);
1764
1765     return undef if $noerr;
1766
1767     die "unable to parse pci id\n";
1768 }
1769
1770 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1771 sub verify_watchdog {
1772     my ($value, $noerr) = @_;
1773
1774     return $value if parse_watchdog($value);
1775
1776     return undef if $noerr;
1777
1778     die "unable to parse watchdog options\n";
1779 }
1780
1781 sub parse_watchdog {
1782     my ($value) = @_;
1783
1784     return undef if !$value;
1785
1786     my $res = {};
1787
1788     foreach my $p (split(/,/, $value)) {
1789         next if $p =~ m/^\s*$/;
1790
1791         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1792             $res->{model} = $2;
1793         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1794             $res->{action} = $2;
1795         } else {
1796             return undef;
1797         }
1798     }
1799
1800     return $res;
1801 }
1802
1803 sub parse_usb_device {
1804     my ($value) = @_;
1805
1806     return undef if !$value;
1807
1808     my $res = {};
1809     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1810         $res->{vendorid} = $2;
1811         $res->{productid} = $4;
1812     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1813         $res->{hostbus} = $1;
1814         $res->{hostport} = $2;
1815     } elsif ($value =~ m/^spice$/i) {
1816         $res->{spice} = 1;
1817     } else {
1818         return undef;
1819     }
1820
1821     return $res;
1822 }
1823
1824 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1825 sub verify_usb_device {
1826     my ($value, $noerr) = @_;
1827
1828     return $value if parse_usb_device($value);
1829
1830     return undef if $noerr;
1831
1832     die "unable to parse usb device\n";
1833 }
1834
1835 # add JSON properties for create and set function
1836 sub json_config_properties {
1837     my $prop = shift;
1838
1839     foreach my $opt (keys %$confdesc) {
1840         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1841         $prop->{$opt} = $confdesc->{$opt};
1842     }
1843
1844     return $prop;
1845 }
1846
1847 sub check_type {
1848     my ($key, $value) = @_;
1849
1850     die "unknown setting '$key'\n" if !$confdesc->{$key};
1851
1852     my $type = $confdesc->{$key}->{type};
1853
1854     if (!defined($value)) {
1855         die "got undefined value\n";
1856     }
1857
1858     if ($value =~ m/[\n\r]/) {
1859         die "property contains a line feed\n";
1860     }
1861
1862     if ($type eq 'boolean') {
1863         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1864         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1865         die "type check ('boolean') failed - got '$value'\n";
1866     } elsif ($type eq 'integer') {
1867         return int($1) if $value =~ m/^(\d+)$/;
1868         die "type check ('integer') failed - got '$value'\n";
1869     } elsif ($type eq 'number') {
1870         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1871         die "type check ('number') failed - got '$value'\n";
1872     } elsif ($type eq 'string') {
1873         if (my $fmt = $confdesc->{$key}->{format}) {
1874             if ($fmt eq 'pve-qm-drive') {
1875                 # special case - we need to pass $key to parse_drive()
1876                 my $drive = parse_drive($key, $value);
1877                 return $value if $drive;
1878                 die "unable to parse drive options\n";
1879             }
1880             PVE::JSONSchema::check_format($fmt, $value);
1881             return $value;
1882         }
1883         $value =~ s/^\"(.*)\"$/$1/;
1884         return $value;
1885     } else {
1886         die "internal error"
1887     }
1888 }
1889
1890 sub check_iommu_support{
1891     #fixme : need to check IOMMU support
1892     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1893
1894     my $iommu=1;
1895     return $iommu;
1896
1897 }
1898
1899 sub touch_config {
1900     my ($vmid) = @_;
1901
1902     my $conf = PVE::QemuConfig->config_file($vmid);
1903     utime undef, undef, $conf;
1904 }
1905
1906 sub destroy_vm {
1907     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1908
1909     my $conffile = PVE::QemuConfig->config_file($vmid);
1910
1911     my $conf = PVE::QemuConfig->load_config($vmid);
1912
1913     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1914
1915     # only remove disks owned by this VM
1916     foreach_drive($conf, sub {
1917         my ($ds, $drive) = @_;
1918
1919         return if drive_is_cdrom($drive);
1920
1921         my $volid = $drive->{file};
1922
1923         return if !$volid || $volid =~ m|^/|;
1924
1925         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1926         return if !$path || !$owner || ($owner != $vmid);
1927
1928         PVE::Storage::vdisk_free($storecfg, $volid);
1929     });
1930
1931     if ($keep_empty_config) {
1932         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1933     } else {
1934         unlink $conffile;
1935     }
1936
1937     # also remove unused disk
1938     eval {
1939         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1940
1941         eval {
1942             PVE::Storage::foreach_volid($dl, sub {
1943                 my ($volid, $sid, $volname, $d) = @_;
1944                 PVE::Storage::vdisk_free($storecfg, $volid);
1945             });
1946         };
1947         warn $@ if $@;
1948
1949     };
1950     warn $@ if $@;
1951 }
1952
1953 sub parse_vm_config {
1954     my ($filename, $raw) = @_;
1955
1956     return undef if !defined($raw);
1957
1958     my $res = {
1959         digest => Digest::SHA::sha1_hex($raw),
1960         snapshots => {},
1961         pending => {},
1962     };
1963
1964     $filename =~ m|/qemu-server/(\d+)\.conf$|
1965         || die "got strange filename '$filename'";
1966
1967     my $vmid = $1;
1968
1969     my $conf = $res;
1970     my $descr;
1971     my $section = '';
1972
1973     my @lines = split(/\n/, $raw);
1974     foreach my $line (@lines) {
1975         next if $line =~ m/^\s*$/;
1976
1977         if ($line =~ m/^\[PENDING\]\s*$/i) {
1978             $section = 'pending';
1979             if (defined($descr)) {
1980                 $descr =~ s/\s+$//;
1981                 $conf->{description} = $descr;
1982             }
1983             $descr = undef;
1984             $conf = $res->{$section} = {};
1985             next;
1986
1987         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1988             $section = $1;
1989             if (defined($descr)) {
1990                 $descr =~ s/\s+$//;
1991                 $conf->{description} = $descr;
1992             }
1993             $descr = undef;
1994             $conf = $res->{snapshots}->{$section} = {};
1995             next;
1996         }
1997
1998         if ($line =~ m/^\#(.*)\s*$/) {
1999             $descr = '' if !defined($descr);
2000             $descr .= PVE::Tools::decode_text($1) . "\n";
2001             next;
2002         }
2003
2004         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2005             $descr = '' if !defined($descr);
2006             $descr .= PVE::Tools::decode_text($2);
2007         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2008             $conf->{snapstate} = $1;
2009         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2010             my $key = $1;
2011             my $value = $2;
2012             $conf->{$key} = $value;
2013         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2014             my $value = $1;
2015             if ($section eq 'pending') {
2016                 $conf->{delete} = $value; # we parse this later
2017             } else {
2018                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2019             }
2020         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2021             my $key = $1;
2022             my $value = $2;
2023             eval { $value = check_type($key, $value); };
2024             if ($@) {
2025                 warn "vm $vmid - unable to parse value of '$key' - $@";
2026             } else {
2027                 my $fmt = $confdesc->{$key}->{format};
2028                 if ($fmt && $fmt eq 'pve-qm-drive') {
2029                     my $v = parse_drive($key, $value);
2030                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2031                         $v->{file} = $volid;
2032                         $value = print_drive($vmid, $v);
2033                     } else {
2034                         warn "vm $vmid - unable to parse value of '$key'\n";
2035                         next;
2036                     }
2037                 }
2038
2039                 if ($key eq 'cdrom') {
2040                     $conf->{ide2} = $value;
2041                 } else {
2042                     $conf->{$key} = $value;
2043                 }
2044             }
2045         }
2046     }
2047
2048     if (defined($descr)) {
2049         $descr =~ s/\s+$//;
2050         $conf->{description} = $descr;
2051     }
2052     delete $res->{snapstate}; # just to be sure
2053
2054     return $res;
2055 }
2056
2057 sub write_vm_config {
2058     my ($filename, $conf) = @_;
2059
2060     delete $conf->{snapstate}; # just to be sure
2061
2062     if ($conf->{cdrom}) {
2063         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2064         $conf->{ide2} = $conf->{cdrom};
2065         delete $conf->{cdrom};
2066     }
2067
2068     # we do not use 'smp' any longer
2069     if ($conf->{sockets}) {
2070         delete $conf->{smp};
2071     } elsif ($conf->{smp}) {
2072         $conf->{sockets} = $conf->{smp};
2073         delete $conf->{cores};
2074         delete $conf->{smp};
2075     }
2076
2077     my $used_volids = {};
2078
2079     my $cleanup_config = sub {
2080         my ($cref, $pending, $snapname) = @_;
2081
2082         foreach my $key (keys %$cref) {
2083             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2084                 $key eq 'snapstate' || $key eq 'pending';
2085             my $value = $cref->{$key};
2086             if ($key eq 'delete') {
2087                 die "propertry 'delete' is only allowed in [PENDING]\n"
2088                     if !$pending;
2089                 # fixme: check syntax?
2090                 next;
2091             }
2092             eval { $value = check_type($key, $value); };
2093             die "unable to parse value of '$key' - $@" if $@;
2094
2095             $cref->{$key} = $value;
2096
2097             if (!$snapname && is_valid_drivename($key)) {
2098                 my $drive = parse_drive($key, $value);
2099                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2100             }
2101         }
2102     };
2103
2104     &$cleanup_config($conf);
2105
2106     &$cleanup_config($conf->{pending}, 1);
2107
2108     foreach my $snapname (keys %{$conf->{snapshots}}) {
2109         die "internal error" if $snapname eq 'pending';
2110         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2111     }
2112
2113     # remove 'unusedX' settings if we re-add a volume
2114     foreach my $key (keys %$conf) {
2115         my $value = $conf->{$key};
2116         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2117             delete $conf->{$key};
2118         }
2119     }
2120
2121     my $generate_raw_config = sub {
2122         my ($conf, $pending) = @_;
2123
2124         my $raw = '';
2125
2126         # add description as comment to top of file
2127         if (defined(my $descr = $conf->{description})) {
2128             if ($descr) {
2129                 foreach my $cl (split(/\n/, $descr)) {
2130                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2131                 }
2132             } else {
2133                 $raw .= "#\n" if $pending;
2134             }
2135         }
2136
2137         foreach my $key (sort keys %$conf) {
2138             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2139             $raw .= "$key: $conf->{$key}\n";
2140         }
2141         return $raw;
2142     };
2143
2144     my $raw = &$generate_raw_config($conf);
2145
2146     if (scalar(keys %{$conf->{pending}})){
2147         $raw .= "\n[PENDING]\n";
2148         $raw .= &$generate_raw_config($conf->{pending}, 1);
2149     }
2150
2151     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2152         $raw .= "\n[$snapname]\n";
2153         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2154     }
2155
2156     return $raw;
2157 }
2158
2159 sub load_defaults {
2160
2161     my $res = {};
2162
2163     # we use static defaults from our JSON schema configuration
2164     foreach my $key (keys %$confdesc) {
2165         if (defined(my $default = $confdesc->{$key}->{default})) {
2166             $res->{$key} = $default;
2167         }
2168     }
2169
2170     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2171     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2172
2173     return $res;
2174 }
2175
2176 sub config_list {
2177     my $vmlist = PVE::Cluster::get_vmlist();
2178     my $res = {};
2179     return $res if !$vmlist || !$vmlist->{ids};
2180     my $ids = $vmlist->{ids};
2181
2182     foreach my $vmid (keys %$ids) {
2183         my $d = $ids->{$vmid};
2184         next if !$d->{node} || $d->{node} ne $nodename;
2185         next if !$d->{type} || $d->{type} ne 'qemu';
2186         $res->{$vmid}->{exists} = 1;
2187     }
2188     return $res;
2189 }
2190
2191 # test if VM uses local resources (to prevent migration)
2192 sub check_local_resources {
2193     my ($conf, $noerr) = @_;
2194
2195     my $loc_res = 0;
2196
2197     $loc_res = 1 if $conf->{hostusb}; # old syntax
2198     $loc_res = 1 if $conf->{hostpci}; # old syntax
2199
2200     foreach my $k (keys %$conf) {
2201         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2202         # sockets are safe: they will recreated be on the target side post-migrate
2203         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2204         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2205     }
2206
2207     die "VM uses local resources\n" if $loc_res && !$noerr;
2208
2209     return $loc_res;
2210 }
2211
2212 # check if used storages are available on all nodes (use by migrate)
2213 sub check_storage_availability {
2214     my ($storecfg, $conf, $node) = @_;
2215
2216     foreach_drive($conf, sub {
2217         my ($ds, $drive) = @_;
2218
2219         my $volid = $drive->{file};
2220         return if !$volid;
2221
2222         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2223         return if !$sid;
2224
2225         # check if storage is available on both nodes
2226         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2227         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2228    });
2229 }
2230
2231 # list nodes where all VM images are available (used by has_feature API)
2232 sub shared_nodes {
2233     my ($conf, $storecfg) = @_;
2234
2235     my $nodelist = PVE::Cluster::get_nodelist();
2236     my $nodehash = { map { $_ => 1 } @$nodelist };
2237     my $nodename = PVE::INotify::nodename();
2238
2239     foreach_drive($conf, sub {
2240         my ($ds, $drive) = @_;
2241
2242         my $volid = $drive->{file};
2243         return if !$volid;
2244
2245         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2246         if ($storeid) {
2247             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2248             if ($scfg->{disable}) {
2249                 $nodehash = {};
2250             } elsif (my $avail = $scfg->{nodes}) {
2251                 foreach my $node (keys %$nodehash) {
2252                     delete $nodehash->{$node} if !$avail->{$node};
2253                 }
2254             } elsif (!$scfg->{shared}) {
2255                 foreach my $node (keys %$nodehash) {
2256                     delete $nodehash->{$node} if $node ne $nodename
2257                 }
2258             }
2259         }
2260     });
2261
2262     return $nodehash
2263 }
2264
2265 sub check_cmdline {
2266     my ($pidfile, $pid) = @_;
2267
2268     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2269     if (defined($fh)) {
2270         my $line = <$fh>;
2271         $fh->close;
2272         return undef if !$line;
2273         my @param = split(/\0/, $line);
2274
2275         my $cmd = $param[0];
2276         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2277
2278         for (my $i = 0; $i < scalar (@param); $i++) {
2279             my $p = $param[$i];
2280             next if !$p;
2281             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2282                 my $p = $param[$i+1];
2283                 return 1 if $p && ($p eq $pidfile);
2284                 return undef;
2285             }
2286         }
2287     }
2288     return undef;
2289 }
2290
2291 sub check_running {
2292     my ($vmid, $nocheck, $node) = @_;
2293
2294     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2295
2296     die "unable to find configuration file for VM $vmid - no such machine\n"
2297         if !$nocheck && ! -f $filename;
2298
2299     my $pidfile = pidfile_name($vmid);
2300
2301     if (my $fd = IO::File->new("<$pidfile")) {
2302         my $st = stat($fd);
2303         my $line = <$fd>;
2304         close($fd);
2305
2306         my $mtime = $st->mtime;
2307         if ($mtime > time()) {
2308             warn "file '$filename' modified in future\n";
2309         }
2310
2311         if ($line =~ m/^(\d+)$/) {
2312             my $pid = $1;
2313             if (check_cmdline($pidfile, $pid)) {
2314                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2315                     return $pid;
2316                 }
2317             }
2318         }
2319     }
2320
2321     return undef;
2322 }
2323
2324 sub vzlist {
2325
2326     my $vzlist = config_list();
2327
2328     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2329
2330     while (defined(my $de = $fd->read)) {
2331         next if $de !~ m/^(\d+)\.pid$/;
2332         my $vmid = $1;
2333         next if !defined($vzlist->{$vmid});
2334         if (my $pid = check_running($vmid)) {
2335             $vzlist->{$vmid}->{pid} = $pid;
2336         }
2337     }
2338
2339     return $vzlist;
2340 }
2341
2342 sub disksize {
2343     my ($storecfg, $conf) = @_;
2344
2345     my $bootdisk = $conf->{bootdisk};
2346     return undef if !$bootdisk;
2347     return undef if !is_valid_drivename($bootdisk);
2348
2349     return undef if !$conf->{$bootdisk};
2350
2351     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2352     return undef if !defined($drive);
2353
2354     return undef if drive_is_cdrom($drive);
2355
2356     my $volid = $drive->{file};
2357     return undef if !$volid;
2358
2359     return $drive->{size};
2360 }
2361
2362 my $last_proc_pid_stat;
2363
2364 # get VM status information
2365 # This must be fast and should not block ($full == false)
2366 # We only query KVM using QMP if $full == true (this can be slow)
2367 sub vmstatus {
2368     my ($opt_vmid, $full) = @_;
2369
2370     my $res = {};
2371
2372     my $storecfg = PVE::Storage::config();
2373
2374     my $list = vzlist();
2375     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2376
2377     my $cpucount = $cpuinfo->{cpus} || 1;
2378
2379     foreach my $vmid (keys %$list) {
2380         next if $opt_vmid && ($vmid ne $opt_vmid);
2381
2382         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2383         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2384
2385         my $d = {};
2386         $d->{pid} = $list->{$vmid}->{pid};
2387
2388         # fixme: better status?
2389         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2390
2391         my $size = disksize($storecfg, $conf);
2392         if (defined($size)) {
2393             $d->{disk} = 0; # no info available
2394             $d->{maxdisk} = $size;
2395         } else {
2396             $d->{disk} = 0;
2397             $d->{maxdisk} = 0;
2398         }
2399
2400         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2401         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2402         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2403
2404         $d->{name} = $conf->{name} || "VM $vmid";
2405         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2406
2407         if ($conf->{balloon}) {
2408             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2409             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2410         }
2411
2412         $d->{uptime} = 0;
2413         $d->{cpu} = 0;
2414         $d->{mem} = 0;
2415
2416         $d->{netout} = 0;
2417         $d->{netin} = 0;
2418
2419         $d->{diskread} = 0;
2420         $d->{diskwrite} = 0;
2421
2422         $d->{template} = PVE::QemuConfig->is_template($conf);
2423
2424         $res->{$vmid} = $d;
2425     }
2426
2427     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2428     foreach my $dev (keys %$netdev) {
2429         next if $dev !~ m/^tap([1-9]\d*)i/;
2430         my $vmid = $1;
2431         my $d = $res->{$vmid};
2432         next if !$d;
2433
2434         $d->{netout} += $netdev->{$dev}->{receive};
2435         $d->{netin} += $netdev->{$dev}->{transmit};
2436
2437         if ($full) {
2438             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2439             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2440         }
2441
2442     }
2443
2444     my $ctime = gettimeofday;
2445
2446     foreach my $vmid (keys %$list) {
2447
2448         my $d = $res->{$vmid};
2449         my $pid = $d->{pid};
2450         next if !$pid;
2451
2452         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2453         next if !$pstat; # not running
2454
2455         my $used = $pstat->{utime} + $pstat->{stime};
2456
2457         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2458
2459         if ($pstat->{vsize}) {
2460             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2461         }
2462
2463         my $old = $last_proc_pid_stat->{$pid};
2464         if (!$old) {
2465             $last_proc_pid_stat->{$pid} = {
2466                 time => $ctime,
2467                 used => $used,
2468                 cpu => 0,
2469             };
2470             next;
2471         }
2472
2473         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2474
2475         if ($dtime > 1000) {
2476             my $dutime = $used -  $old->{used};
2477
2478             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2479             $last_proc_pid_stat->{$pid} = {
2480                 time => $ctime,
2481                 used => $used,
2482                 cpu => $d->{cpu},
2483             };
2484         } else {
2485             $d->{cpu} = $old->{cpu};
2486         }
2487     }
2488
2489     return $res if !$full;
2490
2491     my $qmpclient = PVE::QMPClient->new();
2492
2493     my $ballooncb = sub {
2494         my ($vmid, $resp) = @_;
2495
2496         my $info = $resp->{'return'};
2497         return if !$info->{max_mem};
2498
2499         my $d = $res->{$vmid};
2500
2501         # use memory assigned to VM
2502         $d->{maxmem} = $info->{max_mem};
2503         $d->{balloon} = $info->{actual};
2504
2505         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2506             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2507             $d->{freemem} = $info->{free_mem};
2508         }
2509
2510         $d->{ballooninfo} = $info;
2511     };
2512
2513     my $blockstatscb = sub {
2514         my ($vmid, $resp) = @_;
2515         my $data = $resp->{'return'} || [];
2516         my $totalrdbytes = 0;
2517         my $totalwrbytes = 0;
2518
2519         for my $blockstat (@$data) {
2520             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2521             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2522
2523             $blockstat->{device} =~ s/drive-//;
2524             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2525         }
2526         $res->{$vmid}->{diskread} = $totalrdbytes;
2527         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2528     };
2529
2530     my $statuscb = sub {
2531         my ($vmid, $resp) = @_;
2532
2533         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2534         # this fails if ballon driver is not loaded, so this must be
2535         # the last commnand (following command are aborted if this fails).
2536         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2537
2538         my $status = 'unknown';
2539         if (!defined($status = $resp->{'return'}->{status})) {
2540             warn "unable to get VM status\n";
2541             return;
2542         }
2543
2544         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2545     };
2546
2547     foreach my $vmid (keys %$list) {
2548         next if $opt_vmid && ($vmid ne $opt_vmid);
2549         next if !$res->{$vmid}->{pid}; # not running
2550         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2551     }
2552
2553     $qmpclient->queue_execute(undef, 1);
2554
2555     foreach my $vmid (keys %$list) {
2556         next if $opt_vmid && ($vmid ne $opt_vmid);
2557         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2558     }
2559
2560     return $res;
2561 }
2562
2563 sub foreach_dimm {
2564     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2565
2566     my $dimm_id = 0;
2567     my $current_size = 1024;
2568     my $dimm_size = 512;
2569     return if $current_size == $memory;
2570
2571     for (my $j = 0; $j < 8; $j++) {
2572         for (my $i = 0; $i < 32; $i++) {
2573             my $name = "dimm${dimm_id}";
2574             $dimm_id++;
2575             my $numanode = $i % $sockets;
2576             $current_size += $dimm_size;
2577             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2578             return  $current_size if $current_size >= $memory;
2579         }
2580         $dimm_size *= 2;
2581     }
2582 }
2583
2584 sub foreach_reverse_dimm {
2585     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2586
2587     my $dimm_id = 253;
2588     my $current_size = 4177920;
2589     my $dimm_size = 65536;
2590     return if $current_size == $memory;
2591
2592     for (my $j = 0; $j < 8; $j++) {
2593         for (my $i = 0; $i < 32; $i++) {
2594             my $name = "dimm${dimm_id}";
2595             $dimm_id--;
2596             my $numanode = $i % $sockets;
2597             $current_size -= $dimm_size;
2598             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2599             return  $current_size if $current_size <= $memory;
2600         }
2601         $dimm_size /= 2;
2602     }
2603 }
2604
2605 sub foreach_drive {
2606     my ($conf, $func) = @_;
2607
2608     foreach my $ds (valid_drive_names()) {
2609         next if !defined($conf->{$ds});
2610
2611         my $drive = parse_drive($ds, $conf->{$ds});
2612         next if !$drive;
2613
2614         &$func($ds, $drive);
2615     }
2616 }
2617
2618 sub foreach_volid {
2619     my ($conf, $func) = @_;
2620
2621     my $volhash = {};
2622
2623     my $test_volid = sub {
2624         my ($volid, $is_cdrom) = @_;
2625
2626         return if !$volid;
2627
2628         $volhash->{$volid} = $is_cdrom || 0;
2629     };
2630
2631     foreach_drive($conf, sub {
2632         my ($ds, $drive) = @_;
2633         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2634     });
2635
2636     foreach my $snapname (keys %{$conf->{snapshots}}) {
2637         my $snap = $conf->{snapshots}->{$snapname};
2638         &$test_volid($snap->{vmstate}, 0);
2639         foreach_drive($snap, sub {
2640             my ($ds, $drive) = @_;
2641             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2642         });
2643     }
2644
2645     foreach my $volid (keys %$volhash) {
2646         &$func($volid, $volhash->{$volid});
2647     }
2648 }
2649
2650 sub vga_conf_has_spice {
2651     my ($vga) = @_;
2652
2653     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2654
2655     return $1 || 1;
2656 }
2657
2658 sub config_to_command {
2659     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2660
2661     my $cmd = [];
2662     my $globalFlags = [];
2663     my $machineFlags = [];
2664     my $rtcFlags = [];
2665     my $cpuFlags = [];
2666     my $devices = [];
2667     my $pciaddr = '';
2668     my $bridges = {};
2669     my $kvmver = kvm_user_version();
2670     my $vernum = 0; # unknown
2671     my $ostype = $conf->{ostype};
2672     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2673         $vernum = $1*1000000+$2*1000;
2674     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2675         $vernum = $1*1000000+$2*1000+$3;
2676     }
2677
2678     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2679
2680     my $have_ovz = -f '/proc/vz/vestat';
2681
2682     my $q35 = machine_type_is_q35($conf);
2683     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2684     my $machine_type = $forcemachine || $conf->{machine};
2685     my $use_old_bios_files = undef;
2686     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2687
2688     my $cpuunits = defined($conf->{cpuunits}) ?
2689             $conf->{cpuunits} : $defaults->{cpuunits};
2690
2691     push @$cmd, '/usr/bin/systemd-run';
2692     push @$cmd, '--scope';
2693     push @$cmd, '--slice', "qemu";
2694     push @$cmd, '--unit', $vmid;
2695     # set KillMode=none, so that systemd don't kill those scopes
2696     # at shutdown (pve-manager service should stop the VMs instead)
2697     push @$cmd, '-p', "KillMode=none";
2698     push @$cmd, '-p', "CPUShares=$cpuunits";
2699     if ($conf->{cpulimit}) {
2700         my $cpulimit = int($conf->{cpulimit} * 100);
2701         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2702     }
2703
2704     push @$cmd, '/usr/bin/kvm';
2705
2706     push @$cmd, '-id', $vmid;
2707
2708     my $use_virtio = 0;
2709
2710     my $qmpsocket = qmp_socket($vmid);
2711     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2712     push @$cmd, '-mon', "chardev=qmp,mode=control";
2713
2714
2715     push @$cmd, '-pidfile' , pidfile_name($vmid);
2716
2717     push @$cmd, '-daemonize';
2718
2719     if ($conf->{smbios1}) {
2720         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2721     }
2722
2723     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2724         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2725         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2726         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2727         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2728         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2729         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2730     }
2731
2732     if ($q35) {
2733         # the q35 chipset support native usb2, so we enable usb controller
2734         # by default for this machine type
2735         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2736     } else {
2737         $pciaddr = print_pci_addr("piix3", $bridges);
2738         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2739
2740         my $use_usb2 = 0;
2741         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2742             next if !$conf->{"usb$i"};
2743             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2744             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2745             $use_usb2 = 1;
2746         }
2747         # include usb device config
2748         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2749     }
2750
2751     # add usb3 controller if needed
2752
2753     my $use_usb3 = 0;
2754     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2755         next if !$conf->{"usb$i"};
2756         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2757         next if !$d || !$d->{usb3};
2758         $use_usb3 = 1;
2759     }
2760
2761     $pciaddr = print_pci_addr("xhci", $bridges);
2762     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2763
2764     my $vga = $conf->{vga};
2765
2766     my $qxlnum = vga_conf_has_spice($vga);
2767     $vga = 'qxl' if $qxlnum;
2768
2769     if (!$vga) {
2770         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2771                                 $conf->{ostype} eq 'win7' ||
2772                                 $conf->{ostype} eq 'w2k8')) {
2773             $vga = 'std';
2774         } else {
2775             $vga = 'cirrus';
2776         }
2777     }
2778
2779     # enable absolute mouse coordinates (needed by vnc)
2780     my $tablet;
2781     if (defined($conf->{tablet})) {
2782         $tablet = $conf->{tablet};
2783     } else {
2784         $tablet = $defaults->{tablet};
2785         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2786         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2787     }
2788
2789     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2790
2791     my $kvm_off = 0;
2792     # host pci devices
2793     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2794         my $d = parse_hostpci($conf->{"hostpci$i"});
2795         next if !$d;
2796
2797         my $pcie = $d->{pcie};
2798         if($pcie){
2799             die "q35 machine model is not enabled" if !$q35;
2800             $pciaddr = print_pcie_addr("hostpci$i");
2801         }else{
2802             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2803         }
2804
2805         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2806         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2807         if ($xvga && $xvga ne '') {
2808             $kvm_off = 1;
2809             $vga = 'none';
2810             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2811                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2812             }
2813             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2814                 $xvga = "";
2815             }
2816         }
2817         my $pcidevices = $d->{pciid};
2818         my $multifunction = 1 if @$pcidevices > 1;
2819
2820         my $j=0;
2821         foreach my $pcidevice (@$pcidevices) {
2822
2823             my $id = "hostpci$i";
2824             $id .= ".$j" if $multifunction;
2825             my $addr = $pciaddr;
2826             $addr .= ".$j" if $multifunction;
2827             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2828
2829             if($j == 0){
2830                 $devicestr .= "$rombar$xvga";
2831                 $devicestr .= ",multifunction=on" if $multifunction;
2832             }
2833
2834             push @$devices, '-device', $devicestr;
2835             $j++;
2836         }
2837     }
2838
2839     # usb devices
2840     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2841         next if !$conf->{"usb$i"};
2842         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2843         next if !$d;
2844
2845         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2846         my $usbbus = '';
2847         if (defined($d->{usb3}) && $d->{usb3}) {
2848             $usbbus = ',bus=xhci.0';
2849         }
2850
2851         if (defined($d->{host})) {
2852             $d = parse_usb_device($d->{host});
2853             if (defined($d->{vendorid}) && defined($d->{productid})) {
2854                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2855             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2856                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2857             } elsif (defined($d->{spice}) && $d->{spice}) {
2858                 # usb redir support for spice, currently no usb3
2859                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2860                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2861             }
2862         }
2863     }
2864
2865     # serial devices
2866     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2867         if (my $path = $conf->{"serial$i"}) {
2868             if ($path eq 'socket') {
2869                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2870                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2871                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2872             } else {
2873                 die "no such serial device\n" if ! -c $path;
2874                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2875                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2876             }
2877         }
2878     }
2879
2880     # parallel devices
2881     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2882         if (my $path = $conf->{"parallel$i"}) {
2883             die "no such parallel device\n" if ! -c $path;
2884             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2885             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2886             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2887         }
2888     }
2889
2890     my $vmname = $conf->{name} || "vm$vmid";
2891
2892     push @$cmd, '-name', $vmname;
2893
2894     my $sockets = 1;
2895     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2896     $sockets = $conf->{sockets} if  $conf->{sockets};
2897
2898     my $cores = $conf->{cores} || 1;
2899
2900     my $maxcpus = $sockets * $cores;
2901
2902     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2903
2904     my $allowed_vcpus = $cpuinfo->{cpus};
2905
2906     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2907         if ($allowed_vcpus < $maxcpus);
2908
2909     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2910
2911     push @$cmd, '-nodefaults';
2912
2913     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2914
2915     my $bootindex_hash = {};
2916     my $i = 1;
2917     foreach my $o (split(//, $bootorder)) {
2918         $bootindex_hash->{$o} = $i*100;
2919         $i++;
2920     }
2921
2922     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2923
2924     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2925
2926     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2927
2928     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2929
2930     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2931         my $socket = vnc_socket($vmid);
2932         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2933     } else {
2934         push @$cmd, '-nographic';
2935     }
2936
2937     # time drift fix
2938     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2939
2940     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2941     my $useLocaltime = $conf->{localtime};
2942
2943     if ($ostype) {
2944         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2945
2946         if ($ostype =~ m/^w/) { # windows
2947             $useLocaltime = 1 if !defined($conf->{localtime});
2948
2949             # use time drift fix when acpi is enabled
2950             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2951                 $tdf = 1 if !defined($conf->{tdf});
2952             }
2953         }
2954
2955         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2956             $ostype eq 'wvista') {
2957             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2958             push @$cmd, '-no-hpet';
2959             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2960                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2961                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2962                 push @$cpuFlags , 'hv_time' if !$nokvm;
2963
2964             } else {
2965                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2966             }
2967         }
2968
2969         if ($ostype eq 'win7' || $ostype eq 'win8') {
2970             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2971         }
2972     }
2973
2974     push @$rtcFlags, 'driftfix=slew' if $tdf;
2975
2976     if ($nokvm) {
2977         push @$machineFlags, 'accel=tcg';
2978     } else {
2979         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2980     }
2981
2982     if ($machine_type) {
2983         push @$machineFlags, "type=${machine_type}";
2984     }
2985
2986     if ($conf->{startdate}) {
2987         push @$rtcFlags, "base=$conf->{startdate}";
2988     } elsif ($useLocaltime) {
2989         push @$rtcFlags, 'base=localtime';
2990     }
2991
2992     my $cpu = $nokvm ? "qemu64" : "kvm64";
2993     if (my $cputype = $conf->{cpu}) {
2994         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
2995             or die "Cannot parse cpu description: $cputype\n";
2996         $cpu = $cpuconf->{cputype};
2997         $kvm_off = 1 if $cpuconf->{hidden};
2998     }
2999
3000     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3001
3002     push @$cpuFlags , '-x2apic'
3003         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3004
3005     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3006
3007     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3008
3009     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3010
3011         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3012         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3013     }
3014
3015     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3016
3017     push @$cpuFlags, 'kvm=off' if $kvm_off;
3018
3019     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3020
3021     push @$cmd, '-cpu', $cpu;
3022
3023     my $memory = $conf->{memory} || $defaults->{memory};
3024     my $static_memory = 0;
3025     my $dimm_memory = 0;
3026
3027     if ($hotplug_features->{memory}) {
3028         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3029         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3030         $static_memory = $STATICMEM;
3031         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3032         $dimm_memory = $memory - $static_memory;
3033         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3034
3035     } else {
3036
3037         $static_memory = $memory;
3038         push @$cmd, '-m', $static_memory;
3039     }
3040
3041     if ($conf->{numa}) {
3042
3043         my $numa_totalmemory = undef;
3044         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3045             next if !$conf->{"numa$i"};
3046             my $numa = parse_numa($conf->{"numa$i"});
3047             next if !$numa;
3048             # memory
3049             die "missing numa node$i memory value\n" if !$numa->{memory};
3050             my $numa_memory = $numa->{memory};
3051             $numa_totalmemory += $numa_memory;
3052             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3053
3054             # cpus
3055             my $cpus_start = $numa->{cpus}->{start};
3056             die "missing numa node$i cpus\n" if !defined($cpus_start);
3057             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3058             my $cpus = $cpus_start;
3059             if (defined($cpus_end)) {
3060                 $cpus .= "-$cpus_end";
3061                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3062             }
3063
3064             # hostnodes
3065             my $hostnodes_start = $numa->{hostnodes}->{start};
3066             if (defined($hostnodes_start)) {
3067                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3068                 my $hostnodes = $hostnodes_start;
3069                 if (defined($hostnodes_end)) {
3070                     $hostnodes .= "-$hostnodes_end";
3071                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3072                 }
3073
3074                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3075                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3076                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3077                 }
3078
3079                 # policy
3080                 my $policy = $numa->{policy};
3081                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3082                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3083             }
3084
3085             push @$cmd, '-object', $numa_object;
3086             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3087         }
3088
3089         die "total memory for NUMA nodes must be equal to vm static memory\n"
3090             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3091
3092         #if no custom tology, we split memory and cores across numa nodes
3093         if(!$numa_totalmemory) {
3094
3095             my $numa_memory = ($static_memory / $sockets) . "M";
3096
3097             for (my $i = 0; $i < $sockets; $i++)  {
3098
3099                 my $cpustart = ($cores * $i);
3100                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3101                 my $cpus = $cpustart;
3102                 $cpus .= "-$cpuend" if $cpuend;
3103
3104                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3105                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3106             }
3107         }
3108     }
3109
3110     if ($hotplug_features->{memory}) {
3111         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3112             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3113             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3114             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3115
3116             #if dimm_memory is not aligned to dimm map
3117             if($current_size > $memory) {
3118                  $conf->{memory} = $current_size;
3119                  PVE::QemuConfig->write_config($vmid, $conf);
3120             }
3121         });
3122     }
3123
3124     push @$cmd, '-S' if $conf->{freeze};
3125
3126     # set keyboard layout
3127     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3128     push @$cmd, '-k', $kb if $kb;
3129
3130     # enable sound
3131     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3132     #push @$cmd, '-soundhw', 'es1370';
3133     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3134
3135     if($conf->{agent}) {
3136         my $qgasocket = qmp_socket($vmid, 1);
3137         my $pciaddr = print_pci_addr("qga0", $bridges);
3138         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3139         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3140         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3141     }
3142
3143     my $spice_port;
3144
3145     if ($qxlnum) {
3146         if ($qxlnum > 1) {
3147             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3148                 for(my $i = 1; $i < $qxlnum; $i++){
3149                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3150                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3151                 }
3152             } else {
3153                 # assume other OS works like Linux
3154                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3155                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3156             }
3157         }
3158
3159         my $pciaddr = print_pci_addr("spice", $bridges);
3160
3161         my $nodename = PVE::INotify::nodename();
3162         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3163         $spice_port = PVE::Tools::next_spice_port($pfamily);
3164
3165         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3166
3167         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3168         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3169         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3170     }
3171
3172     # enable balloon by default, unless explicitly disabled
3173     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3174         $pciaddr = print_pci_addr("balloon0", $bridges);
3175         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3176     }
3177
3178     if ($conf->{watchdog}) {
3179         my $wdopts = parse_watchdog($conf->{watchdog});
3180         $pciaddr = print_pci_addr("watchdog", $bridges);
3181         my $watchdog = $wdopts->{model} || 'i6300esb';
3182         push @$devices, '-device', "$watchdog$pciaddr";
3183         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3184     }
3185
3186     my $vollist = [];
3187     my $scsicontroller = {};
3188     my $ahcicontroller = {};
3189     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3190
3191     # Add iscsi initiator name if available
3192     if (my $initiator = get_initiator_name()) {
3193         push @$devices, '-iscsi', "initiator-name=$initiator";
3194     }
3195
3196     foreach_drive($conf, sub {
3197         my ($ds, $drive) = @_;
3198
3199         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3200             push @$vollist, $drive->{file};
3201         }
3202
3203         $use_virtio = 1 if $ds =~ m/^virtio/;
3204
3205         if (drive_is_cdrom ($drive)) {
3206             if ($bootindex_hash->{d}) {
3207                 $drive->{bootindex} = $bootindex_hash->{d};
3208                 $bootindex_hash->{d} += 1;
3209             }
3210         } else {
3211             if ($bootindex_hash->{c}) {
3212                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3213                 $bootindex_hash->{c} += 1;
3214             }
3215         }
3216
3217         if($drive->{interface} eq 'virtio'){
3218            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3219         }
3220
3221         if ($drive->{interface} eq 'scsi') {
3222
3223             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3224
3225             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3226             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3227
3228             my $iothread = '';
3229             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3230                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3231                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3232             }
3233
3234             my $queues = '';
3235             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3236                 $queues = ",num_queues=$drive->{queues}";
3237             }
3238
3239             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3240             $scsicontroller->{$controller}=1;
3241         }
3242
3243         if ($drive->{interface} eq 'sata') {
3244            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3245            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3246            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3247            $ahcicontroller->{$controller}=1;
3248         }
3249
3250         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3251         push @$devices, '-drive',$drive_cmd;
3252         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3253     });
3254
3255     for (my $i = 0; $i < $MAX_NETS; $i++) {
3256          next if !$conf->{"net$i"};
3257          my $d = parse_net($conf->{"net$i"});
3258          next if !$d;
3259
3260          $use_virtio = 1 if $d->{model} eq 'virtio';
3261
3262          if ($bootindex_hash->{n}) {
3263             $d->{bootindex} = $bootindex_hash->{n};
3264             $bootindex_hash->{n} += 1;
3265          }
3266
3267          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3268          push @$devices, '-netdev', $netdevfull;
3269
3270          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3271          push @$devices, '-device', $netdevicefull;
3272     }
3273
3274     if (!$q35) {
3275         # add pci bridges
3276         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3277            $bridges->{1} = 1;
3278            $bridges->{2} = 1;
3279         }
3280
3281         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3282
3283         while (my ($k, $v) = each %$bridges) {
3284             $pciaddr = print_pci_addr("pci.$k");
3285             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3286         }
3287     }
3288
3289     # add custom args
3290     if ($conf->{args}) {
3291         my $aa = PVE::Tools::split_args($conf->{args});
3292         push @$cmd, @$aa;
3293     }
3294
3295     push @$cmd, @$devices;
3296     push @$cmd, '-rtc', join(',', @$rtcFlags)
3297         if scalar(@$rtcFlags);
3298     push @$cmd, '-machine', join(',', @$machineFlags)
3299         if scalar(@$machineFlags);
3300     push @$cmd, '-global', join(',', @$globalFlags)
3301         if scalar(@$globalFlags);
3302
3303     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3304 }
3305
3306 sub vnc_socket {
3307     my ($vmid) = @_;
3308     return "${var_run_tmpdir}/$vmid.vnc";
3309 }
3310
3311 sub spice_port {
3312     my ($vmid) = @_;
3313
3314     my $res = vm_mon_cmd($vmid, 'query-spice');
3315
3316     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3317 }
3318
3319 sub qmp_socket {
3320     my ($vmid, $qga) = @_;
3321     my $sockettype = $qga ? 'qga' : 'qmp';
3322     return "${var_run_tmpdir}/$vmid.$sockettype";
3323 }
3324
3325 sub pidfile_name {
3326     my ($vmid) = @_;
3327     return "${var_run_tmpdir}/$vmid.pid";
3328 }
3329
3330 sub vm_devices_list {
3331     my ($vmid) = @_;
3332
3333     my $res = vm_mon_cmd($vmid, 'query-pci');
3334     my $devices = {};
3335     foreach my $pcibus (@$res) {
3336         foreach my $device (@{$pcibus->{devices}}) {
3337             next if !$device->{'qdev_id'};
3338             if ($device->{'pci_bridge'}) {
3339                 $devices->{$device->{'qdev_id'}} = 1;
3340                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3341                     next if !$bridge_device->{'qdev_id'};
3342                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3343                     $devices->{$device->{'qdev_id'}}++;
3344                 }
3345             } else {
3346                 $devices->{$device->{'qdev_id'}} = 1;
3347             }
3348         }
3349     }
3350
3351     my $resblock = vm_mon_cmd($vmid, 'query-block');
3352     foreach my $block (@$resblock) {
3353         if($block->{device} =~ m/^drive-(\S+)/){
3354                 $devices->{$1} = 1;
3355         }
3356     }
3357
3358     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3359     foreach my $mice (@$resmice) {
3360         if ($mice->{name} eq 'QEMU HID Tablet') {
3361             $devices->{tablet} = 1;
3362             last;
3363         }
3364     }
3365
3366     return $devices;
3367 }
3368
3369 sub vm_deviceplug {
3370     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3371
3372     my $q35 = machine_type_is_q35($conf);
3373
3374     my $devices_list = vm_devices_list($vmid);
3375     return 1 if defined($devices_list->{$deviceid});
3376
3377     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3378
3379     if ($deviceid eq 'tablet') {
3380
3381         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3382
3383     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3384
3385         qemu_iothread_add($vmid, $deviceid, $device);
3386
3387         qemu_driveadd($storecfg, $vmid, $device);
3388         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3389
3390         qemu_deviceadd($vmid, $devicefull);
3391         eval { qemu_deviceaddverify($vmid, $deviceid); };
3392         if (my $err = $@) {
3393             eval { qemu_drivedel($vmid, $deviceid); };
3394             warn $@ if $@;
3395             die $err;
3396         }
3397
3398     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3399
3400
3401         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3402         my $pciaddr = print_pci_addr($deviceid);
3403         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3404
3405         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3406
3407         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3408             qemu_iothread_add($vmid, $deviceid, $device);
3409             $devicefull .= ",iothread=iothread-$deviceid";
3410         }
3411
3412         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3413             $devicefull .= ",num_queues=$device->{queues}";
3414         }
3415
3416         qemu_deviceadd($vmid, $devicefull);
3417         qemu_deviceaddverify($vmid, $deviceid);
3418
3419     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3420
3421         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3422         qemu_driveadd($storecfg, $vmid, $device);
3423
3424         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3425         eval { qemu_deviceadd($vmid, $devicefull); };
3426         if (my $err = $@) {
3427             eval { qemu_drivedel($vmid, $deviceid); };
3428             warn $@ if $@;
3429             die $err;
3430         }
3431
3432     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3433
3434         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3435
3436         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3437         my $use_old_bios_files = undef;
3438         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3439
3440         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3441         qemu_deviceadd($vmid, $netdevicefull);
3442         eval { qemu_deviceaddverify($vmid, $deviceid); };
3443         if (my $err = $@) {
3444             eval { qemu_netdevdel($vmid, $deviceid); };
3445             warn $@ if $@;
3446             die $err;
3447         }
3448
3449     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3450
3451         my $bridgeid = $2;
3452         my $pciaddr = print_pci_addr($deviceid);
3453         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3454
3455         qemu_deviceadd($vmid, $devicefull);
3456         qemu_deviceaddverify($vmid, $deviceid);
3457
3458     } else {
3459         die "can't hotplug device '$deviceid'\n";
3460     }
3461
3462     return 1;
3463 }
3464
3465 # fixme: this should raise exceptions on error!
3466 sub vm_deviceunplug {
3467     my ($vmid, $conf, $deviceid) = @_;
3468
3469     my $devices_list = vm_devices_list($vmid);
3470     return 1 if !defined($devices_list->{$deviceid});
3471
3472     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3473
3474     if ($deviceid eq 'tablet') {
3475
3476         qemu_devicedel($vmid, $deviceid);
3477
3478     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3479
3480         qemu_devicedel($vmid, $deviceid);
3481         qemu_devicedelverify($vmid, $deviceid);
3482         qemu_drivedel($vmid, $deviceid);
3483         qemu_iothread_del($conf, $vmid, $deviceid);
3484
3485     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3486
3487         qemu_devicedel($vmid, $deviceid);
3488         qemu_devicedelverify($vmid, $deviceid);
3489         qemu_iothread_del($conf, $vmid, $deviceid);
3490
3491     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3492
3493         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3494         my $device = parse_drive($deviceid, $conf->{$deviceid});
3495         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3496
3497         qemu_devicedel($vmid, $deviceid);
3498         qemu_drivedel($vmid, $deviceid);
3499         qemu_deletescsihw($conf, $vmid, $deviceid);
3500
3501     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3502
3503         qemu_devicedel($vmid, $deviceid);
3504         qemu_devicedelverify($vmid, $deviceid);
3505         qemu_netdevdel($vmid, $deviceid);
3506
3507     } else {
3508         die "can't unplug device '$deviceid'\n";
3509     }
3510
3511     return 1;
3512 }
3513
3514 sub qemu_deviceadd {
3515     my ($vmid, $devicefull) = @_;
3516
3517     $devicefull = "driver=".$devicefull;
3518     my %options =  split(/[=,]/, $devicefull);
3519
3520     vm_mon_cmd($vmid, "device_add" , %options);
3521 }
3522
3523 sub qemu_devicedel {
3524     my ($vmid, $deviceid) = @_;
3525
3526     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3527 }
3528
3529 sub qemu_iothread_add {
3530     my($vmid, $deviceid, $device) = @_;
3531
3532     if ($device->{iothread}) {
3533         my $iothreads = vm_iothreads_list($vmid);
3534         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3535     }
3536 }
3537
3538 sub qemu_iothread_del {
3539     my($conf, $vmid, $deviceid) = @_;
3540
3541     my $device = parse_drive($deviceid, $conf->{$deviceid});
3542     if ($device->{iothread}) {
3543         my $iothreads = vm_iothreads_list($vmid);
3544         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3545     }
3546 }
3547
3548 sub qemu_objectadd {
3549     my($vmid, $objectid, $qomtype) = @_;
3550
3551     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3552
3553     return 1;
3554 }
3555
3556 sub qemu_objectdel {
3557     my($vmid, $objectid) = @_;
3558
3559     vm_mon_cmd($vmid, "object-del", id => $objectid);
3560
3561     return 1;
3562 }
3563
3564 sub qemu_driveadd {
3565     my ($storecfg, $vmid, $device) = @_;
3566
3567     my $drive = print_drive_full($storecfg, $vmid, $device);
3568     $drive =~ s/\\/\\\\/g;
3569     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3570
3571     # If the command succeeds qemu prints: "OK"
3572     return 1 if $ret =~ m/OK/s;
3573
3574     die "adding drive failed: $ret\n";
3575 }
3576
3577 sub qemu_drivedel {
3578     my($vmid, $deviceid) = @_;
3579
3580     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3581     $ret =~ s/^\s+//;
3582
3583     return 1 if $ret eq "";
3584
3585     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3586     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3587
3588     die "deleting drive $deviceid failed : $ret\n";
3589 }
3590
3591 sub qemu_deviceaddverify {
3592     my ($vmid, $deviceid) = @_;
3593
3594     for (my $i = 0; $i <= 5; $i++) {
3595          my $devices_list = vm_devices_list($vmid);
3596          return 1 if defined($devices_list->{$deviceid});
3597          sleep 1;
3598     }
3599
3600     die "error on hotplug device '$deviceid'\n";
3601 }
3602
3603
3604 sub qemu_devicedelverify {
3605     my ($vmid, $deviceid) = @_;
3606
3607     # need to verify that the device is correctly removed as device_del
3608     # is async and empty return is not reliable
3609
3610     for (my $i = 0; $i <= 5; $i++) {
3611          my $devices_list = vm_devices_list($vmid);
3612          return 1 if !defined($devices_list->{$deviceid});
3613          sleep 1;
3614     }
3615
3616     die "error on hot-unplugging device '$deviceid'\n";
3617 }
3618
3619 sub qemu_findorcreatescsihw {
3620     my ($storecfg, $conf, $vmid, $device) = @_;
3621
3622     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3623
3624     my $scsihwid="$controller_prefix$controller";
3625     my $devices_list = vm_devices_list($vmid);
3626
3627     if(!defined($devices_list->{$scsihwid})) {
3628         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3629     }
3630
3631     return 1;
3632 }
3633
3634 sub qemu_deletescsihw {
3635     my ($conf, $vmid, $opt) = @_;
3636
3637     my $device = parse_drive($opt, $conf->{$opt});
3638
3639     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3640         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3641         return 1;
3642     }
3643
3644     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3645
3646     my $devices_list = vm_devices_list($vmid);
3647     foreach my $opt (keys %{$devices_list}) {
3648         if (PVE::QemuServer::is_valid_drivename($opt)) {
3649             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3650             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3651                 return 1;
3652             }
3653         }
3654     }
3655
3656     my $scsihwid="scsihw$controller";
3657
3658     vm_deviceunplug($vmid, $conf, $scsihwid);
3659
3660     return 1;
3661 }
3662
3663 sub qemu_add_pci_bridge {
3664     my ($storecfg, $conf, $vmid, $device) = @_;
3665
3666     my $bridges = {};
3667
3668     my $bridgeid;
3669
3670     print_pci_addr($device, $bridges);
3671
3672     while (my ($k, $v) = each %$bridges) {
3673         $bridgeid = $k;
3674     }
3675     return 1 if !defined($bridgeid) || $bridgeid < 1;
3676
3677     my $bridge = "pci.$bridgeid";
3678     my $devices_list = vm_devices_list($vmid);
3679
3680     if (!defined($devices_list->{$bridge})) {
3681         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3682     }
3683
3684     return 1;
3685 }
3686
3687 sub qemu_set_link_status {
3688     my ($vmid, $device, $up) = @_;
3689
3690     vm_mon_cmd($vmid, "set_link", name => $device,
3691                up => $up ? JSON::true : JSON::false);
3692 }
3693
3694 sub qemu_netdevadd {
3695     my ($vmid, $conf, $device, $deviceid) = @_;
3696
3697     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3698     my %options =  split(/[=,]/, $netdev);
3699
3700     vm_mon_cmd($vmid, "netdev_add",  %options);
3701     return 1;
3702 }
3703
3704 sub qemu_netdevdel {
3705     my ($vmid, $deviceid) = @_;
3706
3707     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3708 }
3709
3710 sub qemu_cpu_hotplug {
3711     my ($vmid, $conf, $vcpus) = @_;
3712
3713     my $sockets = 1;
3714     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3715     $sockets = $conf->{sockets} if  $conf->{sockets};
3716     my $cores = $conf->{cores} || 1;
3717     my $maxcpus = $sockets * $cores;
3718
3719     $vcpus = $maxcpus if !$vcpus;
3720
3721     die "you can't add more vcpus than maxcpus\n"
3722         if $vcpus > $maxcpus;
3723
3724     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3725     die "online cpu unplug is not yet possible\n"
3726         if $vcpus < $currentvcpus;
3727
3728     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3729     die "vcpus in running vm is different than configuration\n"
3730         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3731
3732     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3733         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3734     }
3735 }
3736
3737 sub qemu_memory_hotplug {
3738     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3739
3740     return $value if !check_running($vmid);
3741
3742     my $memory = $conf->{memory} || $defaults->{memory};
3743     $value = $defaults->{memory} if !$value;
3744     return $value if $value == $memory;
3745
3746     my $static_memory = $STATICMEM;
3747     my $dimm_memory = $memory - $static_memory;
3748
3749     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3750     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3751
3752
3753     my $sockets = 1;
3754     $sockets = $conf->{sockets} if $conf->{sockets};
3755
3756     if($value > $memory) {
3757
3758         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3759             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3760
3761                 return if $current_size <= $conf->{memory};
3762
3763                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3764                 if (my $err = $@) {
3765                     eval { qemu_objectdel($vmid, "mem-$name"); };
3766                     die $err;
3767                 }
3768
3769                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3770                 if (my $err = $@) {
3771                     eval { qemu_objectdel($vmid, "mem-$name"); };
3772                     die $err;
3773                 }
3774                 #update conf after each succesful module hotplug
3775                 $conf->{memory} = $current_size;
3776                 PVE::QemuConfig->write_config($vmid, $conf);
3777         });
3778
3779     } else {
3780
3781         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3782             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3783
3784                 return if $current_size >= $conf->{memory};
3785                 print "try to unplug memory dimm $name\n";
3786
3787                 my $retry = 0;
3788                 while (1) {
3789                     eval { qemu_devicedel($vmid, $name) };
3790                     sleep 3;
3791                     my $dimm_list = qemu_dimm_list($vmid);
3792                     last if !$dimm_list->{$name};
3793                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3794                     $retry++;
3795                 }
3796
3797                 #update conf after each succesful module unplug
3798                 $conf->{memory} = $current_size;
3799
3800                 eval { qemu_objectdel($vmid, "mem-$name"); };
3801                 PVE::QemuConfig->write_config($vmid, $conf);
3802         });
3803     }
3804 }
3805
3806 sub qemu_dimm_list {
3807     my ($vmid) = @_;
3808
3809     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3810     my $dimms = {};
3811
3812     foreach my $dimm (@$dimmarray) {
3813
3814         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3815         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3816         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3817         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3818         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3819     }
3820     return $dimms;
3821 }
3822
3823 sub qemu_block_set_io_throttle {
3824     my ($vmid, $deviceid,
3825         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3826         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3827
3828     return if !check_running($vmid) ;
3829
3830     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3831         bps => int($bps),
3832         bps_rd => int($bps_rd),
3833         bps_wr => int($bps_wr),
3834         iops => int($iops),
3835         iops_rd => int($iops_rd),
3836         iops_wr => int($iops_wr),
3837         bps_max => int($bps_max),
3838         bps_rd_max => int($bps_rd_max),
3839         bps_wr_max => int($bps_wr_max),
3840         iops_max => int($iops_max),
3841         iops_rd_max => int($iops_rd_max),
3842         iops_wr_max => int($iops_wr_max)
3843     );
3844
3845 }
3846
3847 # old code, only used to shutdown old VM after update
3848 sub __read_avail {
3849     my ($fh, $timeout) = @_;
3850
3851     my $sel = new IO::Select;
3852     $sel->add($fh);
3853
3854     my $res = '';
3855     my $buf;
3856
3857     my @ready;
3858     while (scalar (@ready = $sel->can_read($timeout))) {
3859         my $count;
3860         if ($count = $fh->sysread($buf, 8192)) {
3861             if ($buf =~ /^(.*)\(qemu\) $/s) {
3862                 $res .= $1;
3863                 last;
3864             } else {
3865                 $res .= $buf;
3866             }
3867         } else {
3868             if (!defined($count)) {
3869                 die "$!\n";
3870             }
3871             last;
3872         }
3873     }
3874
3875     die "monitor read timeout\n" if !scalar(@ready);
3876
3877     return $res;
3878 }
3879
3880 # old code, only used to shutdown old VM after update
3881 sub vm_monitor_command {
3882     my ($vmid, $cmdstr, $nocheck) = @_;
3883
3884     my $res;
3885
3886     eval {
3887         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3888
3889         my $sname = "${var_run_tmpdir}/$vmid.mon";
3890
3891         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3892             die "unable to connect to VM $vmid socket - $!\n";
3893
3894         my $timeout = 3;
3895
3896         # hack: migrate sometime blocks the monitor (when migrate_downtime
3897         # is set)
3898         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3899             $timeout = 60*60; # 1 hour
3900         }
3901
3902         # read banner;
3903         my $data = __read_avail($sock, $timeout);
3904
3905         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3906             die "got unexpected qemu monitor banner\n";
3907         }
3908
3909         my $sel = new IO::Select;
3910         $sel->add($sock);
3911
3912         if (!scalar(my @ready = $sel->can_write($timeout))) {
3913             die "monitor write error - timeout";
3914         }
3915
3916         my $fullcmd = "$cmdstr\r";
3917
3918         # syslog('info', "VM $vmid monitor command: $cmdstr");
3919
3920         my $b;
3921         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3922             die "monitor write error - $!";
3923         }
3924
3925         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3926
3927         $timeout = 20;
3928
3929         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3930             $timeout = 60*60; # 1 hour
3931         } elsif ($cmdstr =~ m/^(eject|change)/) {
3932             $timeout = 60; # note: cdrom mount command is slow
3933         }
3934         if ($res = __read_avail($sock, $timeout)) {
3935
3936             my @lines = split("\r?\n", $res);
3937
3938             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3939
3940             $res = join("\n", @lines);
3941             $res .= "\n";
3942         }
3943     };
3944
3945     my $err = $@;
3946
3947     if ($err) {
3948         syslog("err", "VM $vmid monitor command failed - $err");
3949         die $err;
3950     }
3951
3952     return $res;
3953 }
3954
3955 sub qemu_block_resize {
3956     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3957
3958     my $running = check_running($vmid);
3959
3960     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3961
3962     return if !$running;
3963
3964     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3965
3966 }
3967
3968 sub qemu_volume_snapshot {
3969     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3970
3971     my $running = check_running($vmid);
3972
3973     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3974         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3975     } else {
3976         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3977     }
3978 }
3979
3980 sub qemu_volume_snapshot_delete {
3981     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3982
3983     my $running = check_running($vmid);
3984
3985     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3986
3987     return if !$running;
3988
3989     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3990 }
3991
3992 sub set_migration_caps {
3993     my ($vmid) = @_;
3994
3995     my $cap_ref = [];
3996
3997     my $enabled_cap = {
3998         "auto-converge" => 1,
3999         "xbzrle" => 1,
4000         "x-rdma-pin-all" => 0,
4001         "zero-blocks" => 0,
4002         "compress" => 0
4003     };
4004
4005     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4006
4007     for my $supported_capability (@$supported_capabilities) {
4008         push @$cap_ref, {
4009             capability => $supported_capability->{capability},
4010             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4011         };
4012     }
4013
4014     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4015 }
4016
4017 my $fast_plug_option = {
4018     'lock' => 1,
4019     'name' => 1,
4020     'onboot' => 1,
4021     'shares' => 1,
4022     'startup' => 1,
4023     'description' => 1,
4024 };
4025
4026 # hotplug changes in [PENDING]
4027 # $selection hash can be used to only apply specified options, for
4028 # example: { cores => 1 } (only apply changed 'cores')
4029 # $errors ref is used to return error messages
4030 sub vmconfig_hotplug_pending {
4031     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4032
4033     my $defaults = load_defaults();
4034
4035     # commit values which do not have any impact on running VM first
4036     # Note: those option cannot raise errors, we we do not care about
4037     # $selection and always apply them.
4038
4039     my $add_error = sub {
4040         my ($opt, $msg) = @_;
4041         $errors->{$opt} = "hotplug problem - $msg";
4042     };
4043
4044     my $changes = 0;
4045     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4046         if ($fast_plug_option->{$opt}) {
4047             $conf->{$opt} = $conf->{pending}->{$opt};
4048             delete $conf->{pending}->{$opt};
4049             $changes = 1;
4050         }
4051     }
4052
4053     if ($changes) {
4054         PVE::QemuConfig->write_config($vmid, $conf);
4055         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4056     }
4057
4058     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4059
4060     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4061     while (my ($opt, $force) = each %$pending_delete_hash) {
4062         next if $selection && !$selection->{$opt};
4063         eval {
4064             if ($opt eq 'hotplug') {
4065                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4066             } elsif ($opt eq 'tablet') {
4067                 die "skip\n" if !$hotplug_features->{usb};
4068                 if ($defaults->{tablet}) {
4069                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4070                 } else {
4071                     vm_deviceunplug($vmid, $conf, $opt);
4072                 }
4073             } elsif ($opt eq 'vcpus') {
4074                 die "skip\n" if !$hotplug_features->{cpu};
4075                 qemu_cpu_hotplug($vmid, $conf, undef);
4076             } elsif ($opt eq 'balloon') {
4077                 # enable balloon device is not hotpluggable
4078                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4079             } elsif ($fast_plug_option->{$opt}) {
4080                 # do nothing
4081             } elsif ($opt =~ m/^net(\d+)$/) {
4082                 die "skip\n" if !$hotplug_features->{network};
4083                 vm_deviceunplug($vmid, $conf, $opt);
4084             } elsif (is_valid_drivename($opt)) {
4085                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4086                 vm_deviceunplug($vmid, $conf, $opt);
4087                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4088             } elsif ($opt =~ m/^memory$/) {
4089                 die "skip\n" if !$hotplug_features->{memory};
4090                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4091             } elsif ($opt eq 'cpuunits') {
4092                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4093             } elsif ($opt eq 'cpulimit') {
4094                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4095             } else {
4096                 die "skip\n";
4097             }
4098         };
4099         if (my $err = $@) {
4100             &$add_error($opt, $err) if $err ne "skip\n";
4101         } else {
4102             # save new config if hotplug was successful
4103             delete $conf->{$opt};
4104             vmconfig_undelete_pending_option($conf, $opt);
4105             PVE::QemuConfig->write_config($vmid, $conf);
4106             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4107         }
4108     }
4109
4110     foreach my $opt (keys %{$conf->{pending}}) {
4111         next if $selection && !$selection->{$opt};
4112         my $value = $conf->{pending}->{$opt};
4113         eval {
4114             if ($opt eq 'hotplug') {
4115                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4116             } elsif ($opt eq 'tablet') {
4117                 die "skip\n" if !$hotplug_features->{usb};
4118                 if ($value == 1) {
4119                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4120                 } elsif ($value == 0) {
4121                     vm_deviceunplug($vmid, $conf, $opt);
4122                 }
4123             } elsif ($opt eq 'vcpus') {
4124                 die "skip\n" if !$hotplug_features->{cpu};
4125                 qemu_cpu_hotplug($vmid, $conf, $value);
4126             } elsif ($opt eq 'balloon') {
4127                 # enable/disable balloning device is not hotpluggable
4128                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4129                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4130                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4131
4132                 # allow manual ballooning if shares is set to zero
4133                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4134                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4135                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4136                 }
4137             } elsif ($opt =~ m/^net(\d+)$/) {
4138                 # some changes can be done without hotplug
4139                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4140                                     $vmid, $opt, $value);
4141             } elsif (is_valid_drivename($opt)) {
4142                 # some changes can be done without hotplug
4143                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4144                                      $vmid, $opt, $value, 1);
4145             } elsif ($opt =~ m/^memory$/) { #dimms
4146                 die "skip\n" if !$hotplug_features->{memory};
4147                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4148             } elsif ($opt eq 'cpuunits') {
4149                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4150             } elsif ($opt eq 'cpulimit') {
4151                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4152                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4153             } else {
4154                 die "skip\n";  # skip non-hot-pluggable options
4155             }
4156         };
4157         if (my $err = $@) {
4158             &$add_error($opt, $err) if $err ne "skip\n";
4159         } else {
4160             # save new config if hotplug was successful
4161             $conf->{$opt} = $value;
4162             delete $conf->{pending}->{$opt};
4163             PVE::QemuConfig->write_config($vmid, $conf);
4164             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4165         }
4166     }
4167 }
4168
4169 sub try_deallocate_drive {
4170     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4171
4172     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4173         my $volid = $drive->{file};
4174         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4175             my $sid = PVE::Storage::parse_volume_id($volid);
4176             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4177
4178             # check if the disk is really unused
4179             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4180                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4181             PVE::Storage::vdisk_free($storecfg, $volid);
4182             return 1;
4183         } else {
4184             # If vm is not owner of this disk remove from config
4185             return 1;
4186         }
4187     }
4188
4189     return undef;
4190 }
4191
4192 sub vmconfig_delete_or_detach_drive {
4193     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4194
4195     my $drive = parse_drive($opt, $conf->{$opt});
4196
4197     my $rpcenv = PVE::RPCEnvironment::get();
4198     my $authuser = $rpcenv->get_user();
4199
4200     if ($force) {
4201         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4202         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4203     } else {
4204         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4205     }
4206 }
4207
4208 sub vmconfig_apply_pending {
4209     my ($vmid, $conf, $storecfg) = @_;
4210
4211     # cold plug
4212
4213     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4214     while (my ($opt, $force) = each %$pending_delete_hash) {
4215         die "internal error" if $opt =~ m/^unused/;
4216         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4217         if (!defined($conf->{$opt})) {
4218             vmconfig_undelete_pending_option($conf, $opt);
4219             PVE::QemuConfig->write_config($vmid, $conf);
4220         } elsif (is_valid_drivename($opt)) {
4221             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4222             vmconfig_undelete_pending_option($conf, $opt);
4223             delete $conf->{$opt};
4224             PVE::QemuConfig->write_config($vmid, $conf);
4225         } else {
4226             vmconfig_undelete_pending_option($conf, $opt);
4227             delete $conf->{$opt};
4228             PVE::QemuConfig->write_config($vmid, $conf);
4229         }
4230     }
4231
4232     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4233
4234     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4235         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4236
4237         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4238             # skip if nothing changed
4239         } elsif (is_valid_drivename($opt)) {
4240             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4241                 if defined($conf->{$opt});
4242             $conf->{$opt} = $conf->{pending}->{$opt};
4243         } else {
4244             $conf->{$opt} = $conf->{pending}->{$opt};
4245         }
4246
4247         delete $conf->{pending}->{$opt};
4248         PVE::QemuConfig->write_config($vmid, $conf);
4249     }
4250 }
4251
4252 my $safe_num_ne = sub {
4253     my ($a, $b) = @_;
4254
4255     return 0 if !defined($a) && !defined($b);
4256     return 1 if !defined($a);
4257     return 1 if !defined($b);
4258
4259     return $a != $b;
4260 };
4261
4262 my $safe_string_ne = sub {
4263     my ($a, $b) = @_;
4264
4265     return 0 if !defined($a) && !defined($b);
4266     return 1 if !defined($a);
4267     return 1 if !defined($b);
4268
4269     return $a ne $b;
4270 };
4271
4272 sub vmconfig_update_net {
4273     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4274
4275     my $newnet = parse_net($value);
4276
4277     if ($conf->{$opt}) {
4278         my $oldnet = parse_net($conf->{$opt});
4279
4280         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4281             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4282             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4283             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4284
4285             # for non online change, we try to hot-unplug
4286             die "skip\n" if !$hotplug;
4287             vm_deviceunplug($vmid, $conf, $opt);
4288         } else {
4289
4290             die "internal error" if $opt !~ m/net(\d+)/;
4291             my $iface = "tap${vmid}i$1";
4292
4293             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4294                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4295                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4296                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4297                 PVE::Network::tap_unplug($iface);
4298                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4299             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4300                 # Rate can be applied on its own but any change above needs to
4301                 # include the rate in tap_plug since OVS resets everything.
4302                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4303             }
4304
4305             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4306                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4307             }
4308
4309             return 1;
4310         }
4311     }
4312
4313     if ($hotplug) {
4314         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4315     } else {
4316         die "skip\n";
4317     }
4318 }
4319
4320 sub vmconfig_update_disk {
4321     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4322
4323     # fixme: do we need force?
4324
4325     my $drive = parse_drive($opt, $value);
4326
4327     if ($conf->{$opt}) {
4328
4329         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4330
4331             my $media = $drive->{media} || 'disk';
4332             my $oldmedia = $old_drive->{media} || 'disk';
4333             die "unable to change media type\n" if $media ne $oldmedia;
4334
4335             if (!drive_is_cdrom($old_drive)) {
4336
4337                 if ($drive->{file} ne $old_drive->{file}) {
4338
4339                     die "skip\n" if !$hotplug;
4340
4341                     # unplug and register as unused
4342                     vm_deviceunplug($vmid, $conf, $opt);
4343                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4344
4345                 } else {
4346                     # update existing disk
4347
4348                     # skip non hotpluggable value
4349                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4350                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4351                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4352                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4353                         die "skip\n";
4354                     }
4355
4356                     # apply throttle
4357                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4358                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4359                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4360                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4361                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4362                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4363                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4364                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4365                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4366                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4367                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4368                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4369
4370                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4371                                                    ($drive->{mbps} || 0)*1024*1024,
4372                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4373                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4374                                                    $drive->{iops} || 0,
4375                                                    $drive->{iops_rd} || 0,
4376                                                    $drive->{iops_wr} || 0,
4377                                                    ($drive->{mbps_max} || 0)*1024*1024,
4378                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4379                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4380                                                    $drive->{iops_max} || 0,
4381                                                    $drive->{iops_rd_max} || 0,
4382                                                    $drive->{iops_wr_max} || 0);
4383
4384                     }
4385
4386                     return 1;
4387                 }
4388
4389             } else { # cdrom
4390
4391                 if ($drive->{file} eq 'none') {
4392                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4393                 } else {
4394                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4395                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4396                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4397                 }
4398
4399                 return 1;
4400             }
4401         }
4402     }
4403
4404     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4405     # hotplug new disks
4406     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4407     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4408 }
4409
4410 sub vm_start {
4411     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4412         $forcemachine, $spice_ticket) = @_;
4413
4414     PVE::QemuConfig->lock_config($vmid, sub {
4415         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4416
4417         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4418
4419         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4420
4421         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4422
4423         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4424             vmconfig_apply_pending($vmid, $conf, $storecfg);
4425             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4426         }
4427
4428         my $defaults = load_defaults();
4429
4430         # set environment variable useful inside network script
4431         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4432
4433         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4434
4435         my $migrate_port = 0;
4436         my $migrate_uri;
4437         if ($statefile) {
4438             if ($statefile eq 'tcp') {
4439                 my $localip = "localhost";
4440                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4441                 my $nodename = PVE::INotify::nodename();
4442                 if ($datacenterconf->{migration_unsecure}) {
4443                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4444                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4445                 }
4446                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4447                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4448                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4449                 push @$cmd, '-incoming', $migrate_uri;
4450                 push @$cmd, '-S';
4451             } else {
4452                 push @$cmd, '-loadstate', $statefile;
4453             }
4454         } elsif ($paused) {
4455             push @$cmd, '-S';
4456         }
4457
4458         # host pci devices
4459         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4460           my $d = parse_hostpci($conf->{"hostpci$i"});
4461           next if !$d;
4462           my $pcidevices = $d->{pciid};
4463           foreach my $pcidevice (@$pcidevices) {
4464                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4465
4466                 my $info = pci_device_info("0000:$pciid");
4467                 die "IOMMU not present\n" if !check_iommu_support();
4468                 die "no pci device info for device '$pciid'\n" if !$info;
4469                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4470                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4471           }
4472         }
4473
4474         PVE::Storage::activate_volumes($storecfg, $vollist);
4475
4476         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4477                     umask => 0077); };
4478
4479         if (my $err = $@) {
4480             # deactivate volumes if start fails
4481             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4482             die "start failed: $err";
4483         }
4484
4485         print "migration listens on $migrate_uri\n" if $migrate_uri;
4486
4487         if ($statefile && $statefile ne 'tcp')  {
4488             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4489             warn $@ if $@;
4490         }
4491
4492         if ($migratedfrom) {
4493
4494             eval {
4495                 set_migration_caps($vmid);
4496             };
4497             warn $@ if $@;
4498
4499             if ($spice_port) {
4500                 print "spice listens on port $spice_port\n";
4501                 if ($spice_ticket) {
4502                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4503                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4504                 }
4505             }
4506
4507         } else {
4508
4509             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4510                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4511                     if $conf->{balloon};
4512             }
4513
4514             foreach my $opt (keys %$conf) {
4515                 next if $opt !~  m/^net\d+$/;
4516                 my $nicconf = parse_net($conf->{$opt});
4517                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4518             }
4519         }
4520
4521         vm_mon_cmd_nocheck($vmid, 'qom-set',
4522                     path => "machine/peripheral/balloon0",
4523                     property => "guest-stats-polling-interval",
4524                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4525
4526     });
4527 }
4528
4529 sub vm_mon_cmd {
4530     my ($vmid, $execute, %params) = @_;
4531
4532     my $cmd = { execute => $execute, arguments => \%params };
4533     vm_qmp_command($vmid, $cmd);
4534 }
4535
4536 sub vm_mon_cmd_nocheck {
4537     my ($vmid, $execute, %params) = @_;
4538
4539     my $cmd = { execute => $execute, arguments => \%params };
4540     vm_qmp_command($vmid, $cmd, 1);
4541 }
4542
4543 sub vm_qmp_command {
4544     my ($vmid, $cmd, $nocheck) = @_;
4545
4546     my $res;
4547
4548     my $timeout;
4549     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4550         $timeout = $cmd->{arguments}->{timeout};
4551         delete $cmd->{arguments}->{timeout};
4552     }
4553
4554     eval {
4555         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4556         my $sname = qmp_socket($vmid);
4557         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4558             my $qmpclient = PVE::QMPClient->new();
4559
4560             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4561         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4562             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4563                 if scalar(%{$cmd->{arguments}});
4564             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4565         } else {
4566             die "unable to open monitor socket\n";
4567         }
4568     };
4569     if (my $err = $@) {
4570         syslog("err", "VM $vmid qmp command failed - $err");
4571         die $err;
4572     }
4573
4574     return $res;
4575 }
4576
4577 sub vm_human_monitor_command {
4578     my ($vmid, $cmdline) = @_;
4579
4580     my $res;
4581
4582     my $cmd = {
4583         execute => 'human-monitor-command',
4584         arguments => { 'command-line' => $cmdline},
4585     };
4586
4587     return vm_qmp_command($vmid, $cmd);
4588 }
4589
4590 sub vm_commandline {
4591     my ($storecfg, $vmid) = @_;
4592
4593     my $conf = PVE::QemuConfig->load_config($vmid);
4594
4595     my $defaults = load_defaults();
4596
4597     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4598
4599     return join(' ', @$cmd);
4600 }
4601
4602 sub vm_reset {
4603     my ($vmid, $skiplock) = @_;
4604
4605     PVE::QemuConfig->lock_config($vmid, sub {
4606
4607         my $conf = PVE::QemuConfig->load_config($vmid);
4608
4609         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4610
4611         vm_mon_cmd($vmid, "system_reset");
4612     });
4613 }
4614
4615 sub get_vm_volumes {
4616     my ($conf) = @_;
4617
4618     my $vollist = [];
4619     foreach_volid($conf, sub {
4620         my ($volid, $is_cdrom) = @_;
4621
4622         return if $volid =~ m|^/|;
4623
4624         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4625         return if !$sid;
4626
4627         push @$vollist, $volid;
4628     });
4629
4630     return $vollist;
4631 }
4632
4633 sub vm_stop_cleanup {
4634     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4635
4636     eval {
4637
4638         if (!$keepActive) {
4639             my $vollist = get_vm_volumes($conf);
4640             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4641         }
4642
4643         foreach my $ext (qw(mon qmp pid vnc qga)) {
4644             unlink "/var/run/qemu-server/${vmid}.$ext";
4645         }
4646
4647         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4648     };
4649     warn $@ if $@; # avoid errors - just warn
4650 }
4651
4652 # Note: use $nockeck to skip tests if VM configuration file exists.
4653 # We need that when migration VMs to other nodes (files already moved)
4654 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4655 sub vm_stop {
4656     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4657
4658     $force = 1 if !defined($force) && !$shutdown;
4659
4660     if ($migratedfrom){
4661         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4662         kill 15, $pid if $pid;
4663         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4664         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4665         return;
4666     }
4667
4668     PVE::QemuConfig->lock_config($vmid, sub {
4669
4670         my $pid = check_running($vmid, $nocheck);
4671         return if !$pid;
4672
4673         my $conf;
4674         if (!$nocheck) {
4675             $conf = PVE::QemuConfig->load_config($vmid);
4676             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4677             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4678                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4679                 $timeout = $opts->{down} if $opts->{down};
4680             }
4681         }
4682
4683         $timeout = 60 if !defined($timeout);
4684
4685         eval {
4686             if ($shutdown) {
4687                 if (defined($conf) && $conf->{agent}) {
4688                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4689                 } else {
4690                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4691                 }
4692             } else {
4693                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4694             }
4695         };
4696         my $err = $@;
4697
4698         if (!$err) {
4699             my $count = 0;
4700             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4701                 $count++;
4702                 sleep 1;
4703             }
4704
4705             if ($count >= $timeout) {
4706                 if ($force) {
4707                     warn "VM still running - terminating now with SIGTERM\n";
4708                     kill 15, $pid;
4709                 } else {
4710                     die "VM quit/powerdown failed - got timeout\n";
4711                 }
4712             } else {
4713                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4714                 return;
4715             }
4716         } else {
4717             if ($force) {
4718                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4719                 kill 15, $pid;
4720             } else {
4721                 die "VM quit/powerdown failed\n";
4722             }
4723         }
4724
4725         # wait again
4726         $timeout = 10;
4727
4728         my $count = 0;
4729         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4730             $count++;
4731             sleep 1;
4732         }
4733
4734         if ($count >= $timeout) {
4735             warn "VM still running - terminating now with SIGKILL\n";
4736             kill 9, $pid;
4737             sleep 1;
4738         }
4739
4740         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4741    });
4742 }
4743
4744 sub vm_suspend {
4745     my ($vmid, $skiplock) = @_;
4746
4747     PVE::QemuConfig->lock_config($vmid, sub {
4748
4749         my $conf = PVE::QemuConfig->load_config($vmid);
4750
4751         PVE::QemuConfig->check_lock($conf)
4752             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4753
4754         vm_mon_cmd($vmid, "stop");
4755     });
4756 }
4757
4758 sub vm_resume {
4759     my ($vmid, $skiplock, $nocheck) = @_;
4760
4761     PVE::QemuConfig->lock_config($vmid, sub {
4762
4763         if (!$nocheck) {
4764
4765             my $conf = PVE::QemuConfig->load_config($vmid);
4766
4767             PVE::QemuConfig->check_lock($conf)
4768                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4769
4770             vm_mon_cmd($vmid, "cont");
4771
4772         } else {
4773             vm_mon_cmd_nocheck($vmid, "cont");
4774         }
4775     });
4776 }
4777
4778 sub vm_sendkey {
4779     my ($vmid, $skiplock, $key) = @_;
4780
4781     PVE::QemuConfig->lock_config($vmid, sub {
4782
4783         my $conf = PVE::QemuConfig->load_config($vmid);
4784
4785         # there is no qmp command, so we use the human monitor command
4786         vm_human_monitor_command($vmid, "sendkey $key");
4787     });
4788 }
4789
4790 sub vm_destroy {
4791     my ($storecfg, $vmid, $skiplock) = @_;
4792
4793     PVE::QemuConfig->lock_config($vmid, sub {
4794
4795         my $conf = PVE::QemuConfig->load_config($vmid);
4796
4797         if (!check_running($vmid)) {
4798             destroy_vm($storecfg, $vmid, undef, $skiplock);
4799         } else {
4800             die "VM $vmid is running - destroy failed\n";
4801         }
4802     });
4803 }
4804
4805 # pci helpers
4806
4807 sub file_write {
4808     my ($filename, $buf) = @_;
4809
4810     my $fh = IO::File->new($filename, "w");
4811     return undef if !$fh;
4812
4813     my $res = print $fh $buf;
4814
4815     $fh->close();
4816
4817     return $res;
4818 }
4819
4820 sub pci_device_info {
4821     my ($name) = @_;
4822
4823     my $res;
4824
4825     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4826     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4827
4828     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4829     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4830
4831     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4832     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4833
4834     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4835     return undef if !defined($product) || $product !~ s/^0x//;
4836
4837     $res = {
4838         name => $name,
4839         vendor => $vendor,
4840         product => $product,
4841         domain => $domain,
4842         bus => $bus,
4843         slot => $slot,
4844         func => $func,
4845         irq => $irq,
4846         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4847     };
4848
4849     return $res;
4850 }
4851
4852 sub pci_dev_reset {
4853     my ($dev) = @_;
4854
4855     my $name = $dev->{name};
4856
4857     my $fn = "$pcisysfs/devices/$name/reset";
4858
4859     return file_write($fn, "1");
4860 }
4861
4862 sub pci_dev_bind_to_vfio {
4863     my ($dev) = @_;
4864
4865     my $name = $dev->{name};
4866
4867     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4868
4869     if (!-d $vfio_basedir) {
4870         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4871     }
4872     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4873
4874     my $testdir = "$vfio_basedir/$name";
4875     return 1 if -d $testdir;
4876
4877     my $data = "$dev->{vendor} $dev->{product}";
4878     return undef if !file_write("$vfio_basedir/new_id", $data);
4879
4880     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4881     if (!file_write($fn, $name)) {
4882         return undef if -f $fn;
4883     }
4884
4885     $fn = "$vfio_basedir/bind";
4886     if (! -d $testdir) {
4887         return undef if !file_write($fn, $name);
4888     }
4889
4890     return -d $testdir;
4891 }
4892
4893 sub pci_dev_group_bind_to_vfio {
4894     my ($pciid) = @_;
4895
4896     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4897
4898     if (!-d $vfio_basedir) {
4899         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4900     }
4901     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4902
4903     # get IOMMU group devices
4904     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4905       my @devs = grep /^0000:/, readdir($D);
4906     closedir($D);
4907
4908     foreach my $pciid (@devs) {
4909         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4910
4911         # pci bridges, switches or root ports are not supported
4912         # they have a pci_bus subdirectory so skip them
4913         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4914
4915         my $info = pci_device_info($1);
4916         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4917     }
4918
4919     return 1;
4920 }
4921
4922 sub print_pci_addr {
4923     my ($id, $bridges) = @_;
4924
4925     my $res = '';
4926     my $devices = {
4927         piix3 => { bus => 0, addr => 1 },
4928         #addr2 : first videocard
4929         balloon0 => { bus => 0, addr => 3 },
4930         watchdog => { bus => 0, addr => 4 },
4931         scsihw0 => { bus => 0, addr => 5 },
4932         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4933         scsihw1 => { bus => 0, addr => 6 },
4934         ahci0 => { bus => 0, addr => 7 },
4935         qga0 => { bus => 0, addr => 8 },
4936         spice => { bus => 0, addr => 9 },
4937         virtio0 => { bus => 0, addr => 10 },
4938         virtio1 => { bus => 0, addr => 11 },
4939         virtio2 => { bus => 0, addr => 12 },
4940         virtio3 => { bus => 0, addr => 13 },
4941         virtio4 => { bus => 0, addr => 14 },
4942         virtio5 => { bus => 0, addr => 15 },
4943         hostpci0 => { bus => 0, addr => 16 },
4944         hostpci1 => { bus => 0, addr => 17 },
4945         net0 => { bus => 0, addr => 18 },
4946         net1 => { bus => 0, addr => 19 },
4947         net2 => { bus => 0, addr => 20 },
4948         net3 => { bus => 0, addr => 21 },
4949         net4 => { bus => 0, addr => 22 },
4950         net5 => { bus => 0, addr => 23 },
4951         vga1 => { bus => 0, addr => 24 },
4952         vga2 => { bus => 0, addr => 25 },
4953         vga3 => { bus => 0, addr => 26 },
4954         hostpci2 => { bus => 0, addr => 27 },
4955         hostpci3 => { bus => 0, addr => 28 },
4956         #addr29 : usb-host (pve-usb.cfg)
4957         'pci.1' => { bus => 0, addr => 30 },
4958         'pci.2' => { bus => 0, addr => 31 },
4959         'net6' => { bus => 1, addr => 1 },
4960         'net7' => { bus => 1, addr => 2 },
4961         'net8' => { bus => 1, addr => 3 },
4962         'net9' => { bus => 1, addr => 4 },
4963         'net10' => { bus => 1, addr => 5 },
4964         'net11' => { bus => 1, addr => 6 },
4965         'net12' => { bus => 1, addr => 7 },
4966         'net13' => { bus => 1, addr => 8 },
4967         'net14' => { bus => 1, addr => 9 },
4968         'net15' => { bus => 1, addr => 10 },
4969         'net16' => { bus => 1, addr => 11 },
4970         'net17' => { bus => 1, addr => 12 },
4971         'net18' => { bus => 1, addr => 13 },
4972         'net19' => { bus => 1, addr => 14 },
4973         'net20' => { bus => 1, addr => 15 },
4974         'net21' => { bus => 1, addr => 16 },
4975         'net22' => { bus => 1, addr => 17 },
4976         'net23' => { bus => 1, addr => 18 },
4977         'net24' => { bus => 1, addr => 19 },
4978         'net25' => { bus => 1, addr => 20 },
4979         'net26' => { bus => 1, addr => 21 },
4980         'net27' => { bus => 1, addr => 22 },
4981         'net28' => { bus => 1, addr => 23 },
4982         'net29' => { bus => 1, addr => 24 },
4983         'net30' => { bus => 1, addr => 25 },
4984         'net31' => { bus => 1, addr => 26 },
4985         'xhci' => { bus => 1, addr => 27 },
4986         'virtio6' => { bus => 2, addr => 1 },
4987         'virtio7' => { bus => 2, addr => 2 },
4988         'virtio8' => { bus => 2, addr => 3 },
4989         'virtio9' => { bus => 2, addr => 4 },
4990         'virtio10' => { bus => 2, addr => 5 },
4991         'virtio11' => { bus => 2, addr => 6 },
4992         'virtio12' => { bus => 2, addr => 7 },
4993         'virtio13' => { bus => 2, addr => 8 },
4994         'virtio14' => { bus => 2, addr => 9 },
4995         'virtio15' => { bus => 2, addr => 10 },
4996         'virtioscsi0' => { bus => 3, addr => 1 },
4997         'virtioscsi1' => { bus => 3, addr => 2 },
4998         'virtioscsi2' => { bus => 3, addr => 3 },
4999         'virtioscsi3' => { bus => 3, addr => 4 },
5000         'virtioscsi4' => { bus => 3, addr => 5 },
5001         'virtioscsi5' => { bus => 3, addr => 6 },
5002         'virtioscsi6' => { bus => 3, addr => 7 },
5003         'virtioscsi7' => { bus => 3, addr => 8 },
5004         'virtioscsi8' => { bus => 3, addr => 9 },
5005         'virtioscsi9' => { bus => 3, addr => 10 },
5006         'virtioscsi10' => { bus => 3, addr => 11 },
5007         'virtioscsi11' => { bus => 3, addr => 12 },
5008         'virtioscsi12' => { bus => 3, addr => 13 },
5009         'virtioscsi13' => { bus => 3, addr => 14 },
5010         'virtioscsi14' => { bus => 3, addr => 15 },
5011         'virtioscsi15' => { bus => 3, addr => 16 },
5012         'virtioscsi16' => { bus => 3, addr => 17 },
5013         'virtioscsi17' => { bus => 3, addr => 18 },
5014         'virtioscsi18' => { bus => 3, addr => 19 },
5015         'virtioscsi19' => { bus => 3, addr => 20 },
5016         'virtioscsi20' => { bus => 3, addr => 21 },
5017         'virtioscsi21' => { bus => 3, addr => 22 },
5018         'virtioscsi22' => { bus => 3, addr => 23 },
5019         'virtioscsi23' => { bus => 3, addr => 24 },
5020         'virtioscsi24' => { bus => 3, addr => 25 },
5021         'virtioscsi25' => { bus => 3, addr => 26 },
5022         'virtioscsi26' => { bus => 3, addr => 27 },
5023         'virtioscsi27' => { bus => 3, addr => 28 },
5024         'virtioscsi28' => { bus => 3, addr => 29 },
5025         'virtioscsi29' => { bus => 3, addr => 30 },
5026         'virtioscsi30' => { bus => 3, addr => 31 },
5027
5028     };
5029
5030     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5031            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5032            my $bus = $devices->{$id}->{bus};
5033            $res = ",bus=pci.$bus,addr=$addr";
5034            $bridges->{$bus} = 1 if $bridges;
5035     }
5036     return $res;
5037
5038 }
5039
5040 sub print_pcie_addr {
5041     my ($id) = @_;
5042
5043     my $res = '';
5044     my $devices = {
5045         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5046         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5047         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5048         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5049     };
5050
5051     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5052            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5053            my $bus = $devices->{$id}->{bus};
5054            $res = ",bus=$bus,addr=$addr";
5055     }
5056     return $res;
5057
5058 }
5059
5060 # vzdump restore implementaion
5061
5062 sub tar_archive_read_firstfile {
5063     my $archive = shift;
5064
5065     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5066
5067     # try to detect archive type first
5068     my $pid = open (TMP, "tar tf '$archive'|") ||
5069         die "unable to open file '$archive'\n";
5070     my $firstfile = <TMP>;
5071     kill 15, $pid;
5072     close TMP;
5073
5074     die "ERROR: archive contaions no data\n" if !$firstfile;
5075     chomp $firstfile;
5076
5077     return $firstfile;
5078 }
5079
5080 sub tar_restore_cleanup {
5081     my ($storecfg, $statfile) = @_;
5082
5083     print STDERR "starting cleanup\n";
5084
5085     if (my $fd = IO::File->new($statfile, "r")) {
5086         while (defined(my $line = <$fd>)) {
5087             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5088                 my $volid = $2;
5089                 eval {
5090                     if ($volid =~ m|^/|) {
5091                         unlink $volid || die 'unlink failed\n';
5092                     } else {
5093                         PVE::Storage::vdisk_free($storecfg, $volid);
5094                     }
5095                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5096                 };
5097                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5098             } else {
5099                 print STDERR "unable to parse line in statfile - $line";
5100             }
5101         }
5102         $fd->close();
5103     }
5104 }
5105
5106 sub restore_archive {
5107     my ($archive, $vmid, $user, $opts) = @_;
5108
5109     my $format = $opts->{format};
5110     my $comp;
5111
5112     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5113         $format = 'tar' if !$format;
5114         $comp = 'gzip';
5115     } elsif ($archive =~ m/\.tar$/) {
5116         $format = 'tar' if !$format;
5117     } elsif ($archive =~ m/.tar.lzo$/) {
5118         $format = 'tar' if !$format;
5119         $comp = 'lzop';
5120     } elsif ($archive =~ m/\.vma$/) {
5121         $format = 'vma' if !$format;
5122     } elsif ($archive =~ m/\.vma\.gz$/) {
5123         $format = 'vma' if !$format;
5124         $comp = 'gzip';
5125     } elsif ($archive =~ m/\.vma\.lzo$/) {
5126         $format = 'vma' if !$format;
5127         $comp = 'lzop';
5128     } else {
5129         $format = 'vma' if !$format; # default
5130     }
5131
5132     # try to detect archive format
5133     if ($format eq 'tar') {
5134         return restore_tar_archive($archive, $vmid, $user, $opts);
5135     } else {
5136         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5137     }
5138 }
5139
5140 sub restore_update_config_line {
5141     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5142
5143     return if $line =~ m/^\#qmdump\#/;
5144     return if $line =~ m/^\#vzdump\#/;
5145     return if $line =~ m/^lock:/;
5146     return if $line =~ m/^unused\d+:/;
5147     return if $line =~ m/^parent:/;
5148     return if $line =~ m/^template:/; # restored VM is never a template
5149
5150     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5151         # try to convert old 1.X settings
5152         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5153         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5154             my ($model, $macaddr) = split(/\=/, $devconfig);
5155             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5156             my $net = {
5157                 model => $model,
5158                 bridge => "vmbr$ind",
5159                 macaddr => $macaddr,
5160             };
5161             my $netstr = print_net($net);
5162
5163             print $outfd "net$cookie->{netcount}: $netstr\n";
5164             $cookie->{netcount}++;
5165         }
5166     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5167         my ($id, $netstr) = ($1, $2);
5168         my $net = parse_net($netstr);
5169         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5170         $netstr = print_net($net);
5171         print $outfd "$id: $netstr\n";
5172     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5173         my $virtdev = $1;
5174         my $value = $3;
5175         my $di = parse_drive($virtdev, $value);
5176         if (defined($di->{backup}) && !$di->{backup}) {
5177             print $outfd "#$line";
5178         } elsif ($map->{$virtdev}) {
5179             delete $di->{format}; # format can change on restore
5180             $di->{file} = $map->{$virtdev};
5181             $value = print_drive($vmid, $di);
5182             print $outfd "$virtdev: $value\n";
5183         } else {
5184             print $outfd $line;
5185         }
5186     } else {
5187         print $outfd $line;
5188     }
5189 }
5190
5191 sub scan_volids {
5192     my ($cfg, $vmid) = @_;
5193
5194     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5195
5196     my $volid_hash = {};
5197     foreach my $storeid (keys %$info) {
5198         foreach my $item (@{$info->{$storeid}}) {
5199             next if !($item->{volid} && $item->{size});
5200             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5201             $volid_hash->{$item->{volid}} = $item;
5202         }
5203     }
5204
5205     return $volid_hash;
5206 }
5207
5208 sub is_volume_in_use {
5209     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5210
5211     my $path = PVE::Storage::path($storecfg, $volid);
5212
5213     my $scan_config = sub {
5214         my ($cref, $snapname) = @_;
5215
5216         foreach my $key (keys %$cref) {
5217             my $value = $cref->{$key};
5218             if (is_valid_drivename($key)) {
5219                 next if $skip_drive && $key eq $skip_drive;
5220                 my $drive = parse_drive($key, $value);
5221                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5222                 return 1 if $volid eq $drive->{file};
5223                 if ($drive->{file} =~ m!^/!) {
5224                     return 1 if $drive->{file} eq $path;
5225                 } else {
5226                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5227                     next if !$storeid;
5228                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5229                     next if !$scfg;
5230                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5231                 }
5232             }
5233         }
5234
5235         return 0;
5236     };
5237
5238     return 1 if &$scan_config($conf);
5239
5240     undef $skip_drive;
5241
5242     foreach my $snapname (keys %{$conf->{snapshots}}) {
5243         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5244     }
5245
5246     return 0;
5247 }
5248
5249 sub update_disksize {
5250     my ($vmid, $conf, $volid_hash) = @_;
5251
5252     my $changes;
5253
5254     my $used = {};
5255
5256     # Note: it is allowed to define multiple storages with same path (alias), so
5257     # we need to check both 'volid' and real 'path' (two different volid can point
5258     # to the same path).
5259
5260     my $usedpath = {};
5261
5262     # update size info
5263     foreach my $opt (keys %$conf) {
5264         if (is_valid_drivename($opt)) {
5265             my $drive = parse_drive($opt, $conf->{$opt});
5266             my $volid = $drive->{file};
5267             next if !$volid;
5268
5269             $used->{$volid} = 1;
5270             if ($volid_hash->{$volid} &&
5271                 (my $path = $volid_hash->{$volid}->{path})) {
5272                 $usedpath->{$path} = 1;
5273             }
5274
5275             next if drive_is_cdrom($drive);
5276             next if !$volid_hash->{$volid};
5277
5278             $drive->{size} = $volid_hash->{$volid}->{size};
5279             my $new = print_drive($vmid, $drive);
5280             if ($new ne $conf->{$opt}) {
5281                 $changes = 1;
5282                 $conf->{$opt} = $new;
5283             }
5284         }
5285     }
5286
5287     # remove 'unusedX' entry if volume is used
5288     foreach my $opt (keys %$conf) {
5289         next if $opt !~ m/^unused\d+$/;
5290         my $volid = $conf->{$opt};
5291         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5292         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5293             $changes = 1;
5294             delete $conf->{$opt};
5295         }
5296     }
5297
5298     foreach my $volid (sort keys %$volid_hash) {
5299         next if $volid =~ m/vm-$vmid-state-/;
5300         next if $used->{$volid};
5301         my $path = $volid_hash->{$volid}->{path};
5302         next if !$path; # just to be sure
5303         next if $usedpath->{$path};
5304         $changes = 1;
5305         PVE::QemuConfig->add_unused_volume($conf, $volid);
5306         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5307     }
5308
5309     return $changes;
5310 }
5311
5312 sub rescan {
5313     my ($vmid, $nolock) = @_;
5314
5315     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5316
5317     my $volid_hash = scan_volids($cfg, $vmid);
5318
5319     my $updatefn =  sub {
5320         my ($vmid) = @_;
5321
5322         my $conf = PVE::QemuConfig->load_config($vmid);
5323
5324         PVE::QemuConfig->check_lock($conf);
5325
5326         my $vm_volids = {};
5327         foreach my $volid (keys %$volid_hash) {
5328             my $info = $volid_hash->{$volid};
5329             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5330         }
5331
5332         my $changes = update_disksize($vmid, $conf, $vm_volids);
5333
5334         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5335     };
5336
5337     if (defined($vmid)) {
5338         if ($nolock) {
5339             &$updatefn($vmid);
5340         } else {
5341             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5342         }
5343     } else {
5344         my $vmlist = config_list();
5345         foreach my $vmid (keys %$vmlist) {
5346             if ($nolock) {
5347                 &$updatefn($vmid);
5348             } else {
5349                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5350             }
5351         }
5352     }
5353 }
5354
5355 sub restore_vma_archive {
5356     my ($archive, $vmid, $user, $opts, $comp) = @_;
5357
5358     my $input = $archive eq '-' ? "<&STDIN" : undef;
5359     my $readfrom = $archive;
5360
5361     my $uncomp = '';
5362     if ($comp) {
5363         $readfrom = '-';
5364         my $qarchive = PVE::Tools::shellquote($archive);
5365         if ($comp eq 'gzip') {
5366             $uncomp = "zcat $qarchive|";
5367         } elsif ($comp eq 'lzop') {
5368             $uncomp = "lzop -d -c $qarchive|";
5369         } else {
5370             die "unknown compression method '$comp'\n";
5371         }
5372
5373     }
5374
5375     my $tmpdir = "/var/tmp/vzdumptmp$$";
5376     rmtree $tmpdir;
5377
5378     # disable interrupts (always do cleanups)
5379     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5380         warn "got interrupt - ignored\n";
5381     };
5382
5383     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5384     POSIX::mkfifo($mapfifo, 0600);
5385     my $fifofh;
5386
5387     my $openfifo = sub {
5388         open($fifofh, '>', $mapfifo) || die $!;
5389     };
5390
5391     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5392
5393     my $oldtimeout;
5394     my $timeout = 5;
5395
5396     my $devinfo = {};
5397
5398     my $rpcenv = PVE::RPCEnvironment::get();
5399
5400     my $conffile = PVE::QemuConfig->config_file($vmid);
5401     my $tmpfn = "$conffile.$$.tmp";
5402
5403     # Note: $oldconf is undef if VM does not exists
5404     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5405     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5406
5407     my $print_devmap = sub {
5408         my $virtdev_hash = {};
5409
5410         my $cfgfn = "$tmpdir/qemu-server.conf";
5411
5412         # we can read the config - that is already extracted
5413         my $fh = IO::File->new($cfgfn, "r") ||
5414             "unable to read qemu-server.conf - $!\n";
5415
5416         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5417         if (-f $fwcfgfn) {
5418             my $pve_firewall_dir = '/etc/pve/firewall';
5419             mkdir $pve_firewall_dir; # make sure the dir exists
5420             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5421         }
5422
5423         while (defined(my $line = <$fh>)) {
5424             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5425                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5426                 die "archive does not contain data for drive '$virtdev'\n"
5427                     if !$devinfo->{$devname};
5428                 if (defined($opts->{storage})) {
5429                     $storeid = $opts->{storage} || 'local';
5430                 } elsif (!$storeid) {
5431                     $storeid = 'local';
5432                 }
5433                 $format = 'raw' if !$format;
5434                 $devinfo->{$devname}->{devname} = $devname;
5435                 $devinfo->{$devname}->{virtdev} = $virtdev;
5436                 $devinfo->{$devname}->{format} = $format;
5437                 $devinfo->{$devname}->{storeid} = $storeid;
5438
5439                 # check permission on storage
5440                 my $pool = $opts->{pool}; # todo: do we need that?
5441                 if ($user ne 'root@pam') {
5442                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5443                 }
5444
5445                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5446             }
5447         }
5448
5449         foreach my $devname (keys %$devinfo) {
5450             die "found no device mapping information for device '$devname'\n"
5451                 if !$devinfo->{$devname}->{virtdev};
5452         }
5453
5454         my $cfg = cfs_read_file('storage.cfg');
5455
5456         # create empty/temp config
5457         if ($oldconf) {
5458             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5459             foreach_drive($oldconf, sub {
5460                 my ($ds, $drive) = @_;
5461
5462                 return if drive_is_cdrom($drive);
5463
5464                 my $volid = $drive->{file};
5465
5466                 return if !$volid || $volid =~ m|^/|;
5467
5468                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5469                 return if !$path || !$owner || ($owner != $vmid);
5470
5471                 # Note: only delete disk we want to restore
5472                 # other volumes will become unused
5473                 if ($virtdev_hash->{$ds}) {
5474                     PVE::Storage::vdisk_free($cfg, $volid);
5475                 }
5476             });
5477
5478             # delete vmstate files
5479             # since after the restore we have no snapshots anymore
5480             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5481                 my $snap = $oldconf->{snapshots}->{$snapname};
5482                 if ($snap->{vmstate}) {
5483                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5484                     if (my $err = $@) {
5485                         warn $err;
5486                     }
5487                 }
5488             }
5489         }
5490
5491         my $map = {};
5492         foreach my $virtdev (sort keys %$virtdev_hash) {
5493             my $d = $virtdev_hash->{$virtdev};
5494             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5495             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5496
5497             # test if requested format is supported
5498             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5499             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5500             $d->{format} = $defFormat if !$supported;
5501
5502             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5503                                                   $d->{format}, undef, $alloc_size);
5504             print STDERR "new volume ID is '$volid'\n";
5505             $d->{volid} = $volid;
5506             my $path = PVE::Storage::path($cfg, $volid);
5507
5508             PVE::Storage::activate_volumes($cfg,[$volid]);
5509
5510             my $write_zeros = 1;
5511             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5512                 $write_zeros = 0;
5513             }
5514
5515             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5516
5517             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5518             $map->{$virtdev} = $volid;
5519         }
5520
5521         $fh->seek(0, 0) || die "seek failed - $!\n";
5522
5523         my $outfd = new IO::File ($tmpfn, "w") ||
5524             die "unable to write config for VM $vmid\n";
5525
5526         my $cookie = { netcount => 0 };
5527         while (defined(my $line = <$fh>)) {
5528             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5529         }
5530
5531         $fh->close();
5532         $outfd->close();
5533     };
5534
5535     eval {
5536         # enable interrupts
5537         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5538             die "interrupted by signal\n";
5539         };
5540         local $SIG{ALRM} = sub { die "got timeout\n"; };
5541
5542         $oldtimeout = alarm($timeout);
5543
5544         my $parser = sub {
5545             my $line = shift;
5546
5547             print "$line\n";
5548
5549             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5550                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5551                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5552             } elsif ($line =~ m/^CTIME: /) {
5553                 # we correctly received the vma config, so we can disable
5554                 # the timeout now for disk allocation (set to 10 minutes, so
5555                 # that we always timeout if something goes wrong)
5556                 alarm(600);
5557                 &$print_devmap();
5558                 print $fifofh "done\n";
5559                 my $tmp = $oldtimeout || 0;
5560                 $oldtimeout = undef;
5561                 alarm($tmp);
5562                 close($fifofh);
5563             }
5564         };
5565
5566         print "restore vma archive: $cmd\n";
5567         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5568     };
5569     my $err = $@;
5570
5571     alarm($oldtimeout) if $oldtimeout;
5572
5573     my $vollist = [];
5574     foreach my $devname (keys %$devinfo) {
5575         my $volid = $devinfo->{$devname}->{volid};
5576         push @$vollist, $volid if $volid;
5577     }
5578
5579     my $cfg = cfs_read_file('storage.cfg');
5580     PVE::Storage::deactivate_volumes($cfg, $vollist);
5581
5582     unlink $mapfifo;
5583
5584     if ($err) {
5585         rmtree $tmpdir;
5586         unlink $tmpfn;
5587
5588         foreach my $devname (keys %$devinfo) {
5589             my $volid = $devinfo->{$devname}->{volid};
5590             next if !$volid;
5591             eval {
5592                 if ($volid =~ m|^/|) {
5593                     unlink $volid || die 'unlink failed\n';
5594                 } else {
5595                     PVE::Storage::vdisk_free($cfg, $volid);
5596                 }
5597                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5598             };
5599             print STDERR "unable to cleanup '$volid' - $@" if $@;
5600         }
5601         die $err;
5602     }
5603
5604     rmtree $tmpdir;
5605
5606     rename($tmpfn, $conffile) ||
5607         die "unable to commit configuration file '$conffile'\n";
5608
5609     PVE::Cluster::cfs_update(); # make sure we read new file
5610
5611     eval { rescan($vmid, 1); };
5612     warn $@ if $@;
5613 }
5614
5615 sub restore_tar_archive {
5616     my ($archive, $vmid, $user, $opts) = @_;
5617
5618     if ($archive ne '-') {
5619         my $firstfile = tar_archive_read_firstfile($archive);
5620         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5621             if $firstfile ne 'qemu-server.conf';
5622     }
5623
5624     my $storecfg = cfs_read_file('storage.cfg');
5625
5626     # destroy existing data - keep empty config
5627     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5628     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5629
5630     my $tocmd = "/usr/lib/qemu-server/qmextract";
5631
5632     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5633     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5634     $tocmd .= ' --prealloc' if $opts->{prealloc};
5635     $tocmd .= ' --info' if $opts->{info};
5636
5637     # tar option "xf" does not autodetect compression when read from STDIN,
5638     # so we pipe to zcat
5639     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5640         PVE::Tools::shellquote("--to-command=$tocmd");
5641
5642     my $tmpdir = "/var/tmp/vzdumptmp$$";
5643     mkpath $tmpdir;
5644
5645     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5646     local $ENV{VZDUMP_VMID} = $vmid;
5647     local $ENV{VZDUMP_USER} = $user;
5648
5649     my $conffile = PVE::QemuConfig->config_file($vmid);
5650     my $tmpfn = "$conffile.$$.tmp";
5651
5652     # disable interrupts (always do cleanups)
5653     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5654         print STDERR "got interrupt - ignored\n";
5655     };
5656
5657     eval {
5658         # enable interrupts
5659         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5660             die "interrupted by signal\n";
5661         };
5662
5663         if ($archive eq '-') {
5664             print "extracting archive from STDIN\n";
5665             run_command($cmd, input => "<&STDIN");
5666         } else {
5667             print "extracting archive '$archive'\n";
5668             run_command($cmd);
5669         }
5670
5671         return if $opts->{info};
5672
5673         # read new mapping
5674         my $map = {};
5675         my $statfile = "$tmpdir/qmrestore.stat";
5676         if (my $fd = IO::File->new($statfile, "r")) {
5677             while (defined (my $line = <$fd>)) {
5678                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5679                     $map->{$1} = $2 if $1;
5680                 } else {
5681                     print STDERR "unable to parse line in statfile - $line\n";
5682                 }
5683             }
5684             $fd->close();
5685         }
5686
5687         my $confsrc = "$tmpdir/qemu-server.conf";
5688
5689         my $srcfd = new IO::File($confsrc, "r") ||
5690             die "unable to open file '$confsrc'\n";
5691
5692         my $outfd = new IO::File ($tmpfn, "w") ||
5693             die "unable to write config for VM $vmid\n";
5694
5695         my $cookie = { netcount => 0 };
5696         while (defined (my $line = <$srcfd>)) {
5697             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5698         }
5699
5700         $srcfd->close();
5701         $outfd->close();
5702     };
5703     my $err = $@;
5704
5705     if ($err) {
5706
5707         unlink $tmpfn;
5708
5709         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5710
5711         die $err;
5712     }
5713
5714     rmtree $tmpdir;
5715
5716     rename $tmpfn, $conffile ||
5717         die "unable to commit configuration file '$conffile'\n";
5718
5719     PVE::Cluster::cfs_update(); # make sure we read new file
5720
5721     eval { rescan($vmid, 1); };
5722     warn $@ if $@;
5723 };
5724
5725 sub foreach_writable_storage {
5726     my ($conf, $func) = @_;
5727
5728     my $sidhash = {};
5729
5730     foreach my $ds (keys %$conf) {
5731         next if !is_valid_drivename($ds);
5732
5733         my $drive = parse_drive($ds, $conf->{$ds});
5734         next if !$drive;
5735         next if drive_is_cdrom($drive);
5736
5737         my $volid = $drive->{file};
5738
5739         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5740         $sidhash->{$sid} = $sid if $sid;
5741     }
5742
5743     foreach my $sid (sort keys %$sidhash) {
5744         &$func($sid);
5745     }
5746 }
5747
5748 sub do_snapshots_with_qemu {
5749     my ($storecfg, $volid) = @_;
5750
5751     my $storage_name = PVE::Storage::parse_volume_id($volid);
5752
5753     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5754         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5755         return 1;
5756     }
5757
5758     if ($volid =~ m/\.(qcow2|qed)$/){
5759         return 1;
5760     }
5761
5762     return undef;
5763 }
5764
5765 sub qga_check_running {
5766     my ($vmid) = @_;
5767
5768     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5769     if ($@) {
5770         warn "Qemu Guest Agent are not running - $@";
5771         return 0;
5772     }
5773     return 1;
5774 }
5775
5776 sub template_create {
5777     my ($vmid, $conf, $disk) = @_;
5778
5779     my $storecfg = PVE::Storage::config();
5780
5781     foreach_drive($conf, sub {
5782         my ($ds, $drive) = @_;
5783
5784         return if drive_is_cdrom($drive);
5785         return if $disk && $ds ne $disk;
5786
5787         my $volid = $drive->{file};
5788         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5789
5790         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5791         $drive->{file} = $voliddst;
5792         $conf->{$ds} = print_drive($vmid, $drive);
5793         PVE::QemuConfig->write_config($vmid, $conf);
5794     });
5795 }
5796
5797 sub qemu_img_convert {
5798     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5799
5800     my $storecfg = PVE::Storage::config();
5801     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5802     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5803
5804     if ($src_storeid && $dst_storeid) {
5805
5806         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5807
5808         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5809         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5810
5811         my $src_format = qemu_img_format($src_scfg, $src_volname);
5812         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5813
5814         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5815         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5816
5817         my $cmd = [];
5818         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5819         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5820         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5821
5822         my $parser = sub {
5823             my $line = shift;
5824             if($line =~ m/\((\S+)\/100\%\)/){
5825                 my $percent = $1;
5826                 my $transferred = int($size * $percent / 100);
5827                 my $remaining = $size - $transferred;
5828
5829                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5830             }
5831
5832         };
5833
5834         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5835         my $err = $@;
5836         die "copy failed: $err" if $err;
5837     }
5838 }
5839
5840 sub qemu_img_format {
5841     my ($scfg, $volname) = @_;
5842
5843     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5844         return $1;
5845     } else {
5846         return "raw";
5847     }
5848 }
5849
5850 sub qemu_drive_mirror {
5851     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
5852
5853     my $storecfg = PVE::Storage::config();
5854     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5855
5856     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5857
5858     my $format = qemu_img_format($dst_scfg, $dst_volname);
5859
5860     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5861
5862     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
5863     $opts->{format} = $format if $format;
5864
5865     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5866
5867     eval {
5868     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5869         while (1) {
5870             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5871             my $stat = @$stats[0];
5872             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5873             die "error job is not mirroring" if $stat->{type} ne "mirror";
5874
5875             my $busy = $stat->{busy};
5876             my $ready = $stat->{ready};
5877
5878             if (my $total = $stat->{len}) {
5879                 my $transferred = $stat->{offset} || 0;
5880                 my $remaining = $total - $transferred;
5881                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5882
5883                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5884             }
5885
5886
5887             if ($stat->{ready} eq 'true') {
5888
5889                 last if $vmiddst != $vmid;
5890
5891                 # try to switch the disk if source and destination are on the same guest
5892                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5893                 last if !$@;
5894                 die $@ if $@ !~ m/cannot be completed/;
5895             }
5896             sleep 1;
5897         }
5898
5899
5900     };
5901     my $err = $@;
5902
5903     my $cancel_job = sub {
5904         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5905         while (1) {
5906             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5907             my $stat = @$stats[0];
5908             last if !$stat;
5909             sleep 1;
5910         }
5911     };
5912
5913     if ($err) {
5914         eval { &$cancel_job(); };
5915         die "mirroring error: $err";
5916     }
5917
5918     if ($vmiddst != $vmid) {
5919         # if we clone a disk for a new target vm, we don't switch the disk
5920         &$cancel_job(); # so we call block-job-cancel
5921     }
5922 }
5923
5924 sub clone_disk {
5925     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5926         $newvmid, $storage, $format, $full, $newvollist) = @_;
5927
5928     my $newvolid;
5929
5930     if (!$full) {
5931         print "create linked clone of drive $drivename ($drive->{file})\n";
5932         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5933         push @$newvollist, $newvolid;
5934     } else {
5935         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5936         $storeid = $storage if $storage;
5937
5938         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5939         if (!$format) {
5940             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5941             $format = qemu_img_format($scfg, $volname);
5942         }
5943
5944         # test if requested format is supported - else use default
5945         my $supported = grep { $_ eq $format } @$validFormats;
5946         $format = $defFormat if !$supported;
5947
5948         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5949
5950         print "create full clone of drive $drivename ($drive->{file})\n";
5951         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5952         push @$newvollist, $newvolid;
5953
5954         PVE::Storage::activate_volumes($storecfg, $newvollist);
5955
5956         if (!$running || $snapname) {
5957             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5958         } else {
5959             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5960         }
5961     }
5962
5963     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5964
5965     my $disk = $drive;
5966     $disk->{format} = undef;
5967     $disk->{file} = $newvolid;
5968     $disk->{size} = $size;
5969
5970     return $disk;
5971 }
5972
5973 # this only works if VM is running
5974 sub get_current_qemu_machine {
5975     my ($vmid) = @_;
5976
5977     my $cmd = { execute => 'query-machines', arguments => {} };
5978     my $res = vm_qmp_command($vmid, $cmd);
5979
5980     my ($current, $default);
5981     foreach my $e (@$res) {
5982         $default = $e->{name} if $e->{'is-default'};
5983         $current = $e->{name} if $e->{'is-current'};
5984     }
5985
5986     # fallback to the default machine if current is not supported by qemu
5987     return $current || $default || 'pc';
5988 }
5989
5990 sub qemu_machine_feature_enabled {
5991     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5992
5993     my $current_major;
5994     my $current_minor;
5995
5996     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5997
5998         $current_major = $3;
5999         $current_minor = $4;
6000
6001     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6002
6003         $current_major = $1;
6004         $current_minor = $2;
6005     }
6006
6007     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6008
6009
6010 }
6011
6012 sub qemu_machine_pxe {
6013     my ($vmid, $conf, $machine) = @_;
6014
6015     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6016
6017     foreach my $opt (keys %$conf) {
6018         next if $opt !~ m/^net(\d+)$/;
6019         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6020         next if !$net;
6021         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6022         return $machine.".pxe" if $romfile =~ m/pxe/;
6023         last;
6024     }
6025
6026     return $machine;
6027 }
6028
6029 sub qemu_use_old_bios_files {
6030     my ($machine_type) = @_;
6031
6032     return if !$machine_type;
6033
6034     my $use_old_bios_files = undef;
6035
6036     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6037         $machine_type = $1;
6038         $use_old_bios_files = 1;
6039     } else {
6040         my $kvmver = kvm_user_version();
6041         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6042         # load new efi bios files on migration. So this hack is required to allow
6043         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6044         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6045         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6046     }
6047
6048     return ($use_old_bios_files, $machine_type);
6049 }
6050
6051 sub lspci {
6052
6053     my $devices = {};
6054
6055     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6056             my (undef, $id, $function) = @_;
6057             my $res = { id => $id, function => $function};
6058             push @{$devices->{$id}}, $res;
6059     });
6060
6061     return $devices;
6062 }
6063
6064 sub vm_iothreads_list {
6065     my ($vmid) = @_;
6066
6067     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6068
6069     my $iothreads = {};
6070     foreach my $iothread (@$res) {
6071         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6072     }
6073
6074     return $iothreads;
6075 }
6076
6077 sub scsihw_infos {
6078     my ($conf, $drive) = @_;
6079
6080     my $maxdev = 0;
6081
6082     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6083         $maxdev = 7;
6084     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6085         $maxdev = 1;
6086     } else {
6087         $maxdev = 256;
6088     }
6089
6090     my $controller = int($drive->{index} / $maxdev);
6091     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6092
6093     return ($maxdev, $controller, $controller_prefix);
6094 }
6095
6096 # bash completion helper
6097
6098 sub complete_backup_archives {
6099     my ($cmdname, $pname, $cvalue) = @_;
6100
6101     my $cfg = PVE::Storage::config();
6102
6103     my $storeid;
6104
6105     if ($cvalue =~ m/^([^:]+):/) {
6106         $storeid = $1;
6107     }
6108
6109     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6110
6111     my $res = [];
6112     foreach my $id (keys %$data) {
6113         foreach my $item (@{$data->{$id}}) {
6114             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6115             push @$res, $item->{volid} if defined($item->{volid});
6116         }
6117     }
6118
6119     return $res;
6120 }
6121
6122 my $complete_vmid_full = sub {
6123     my ($running) = @_;
6124
6125     my $idlist = vmstatus();
6126
6127     my $res = [];
6128
6129     foreach my $id (keys %$idlist) {
6130         my $d = $idlist->{$id};
6131         if (defined($running)) {
6132             next if $d->{template};
6133             next if $running && $d->{status} ne 'running';
6134             next if !$running && $d->{status} eq 'running';
6135         }
6136         push @$res, $id;
6137
6138     }
6139     return $res;
6140 };
6141
6142 sub complete_vmid {
6143     return &$complete_vmid_full();
6144 }
6145
6146 sub complete_vmid_stopped {
6147     return &$complete_vmid_full(0);
6148 }
6149
6150 sub complete_vmid_running {
6151     return &$complete_vmid_full(1);
6152 }
6153
6154 sub complete_storage {
6155
6156     my $cfg = PVE::Storage::config();
6157     my $ids = $cfg->{ids};
6158
6159     my $res = [];
6160     foreach my $sid (keys %$ids) {
6161         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6162         next if !$ids->{$sid}->{content}->{images};
6163         push @$res, $sid;
6164     }
6165
6166     return $res;
6167 }
6168
6169 1;