PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35
  36 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  37
  38 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  39
  40 # Note about locking: we use flock on the config file protect
  41 # against concurent actions.
  42 # Aditionaly, we have a 'lock' setting in the config file. This
  43 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  44 # allowed when such lock is set. But you can ignore this kind of
  45 # lock with the --skiplock flag.
  46
  47 cfs_register_file('/qemu-server/',
  48                   \&parse_vm_config,
  49                   \&write_vm_config);
  50
  51 PVE::JSONSchema::register_standard_option('skiplock', {
  52     description => "Ignore locks - only root is allowed to use this option.",
  53     type => 'boolean',
  54     optional => 1,
  55 });
  56
  57 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  58     description => "Some command save/restore state from this location.",
  59     type => 'string',
  60     maxLength => 128,
  61     optional => 1,
  62 });
  63
  64 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  65     description => "The name of the snapshot.",
  66     type => 'string', format => 'pve-configid',
  67     maxLength => 40,
  68 });
  69
  70 #no warnings 'redefine';
  71
  72 sub cgroups_write {
  73    my ($controller, $vmid, $option, $value) = @_;
  74
  75    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  76    PVE::ProcFSTools::write_proc_entry($path, $value);
  77
  78 }
  79
  80 my $nodename = PVE::INotify::nodename();
  81
  82 mkdir "/etc/pve/nodes/$nodename";
  83 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  84 mkdir $confdir;
  85
  86 my $var_run_tmpdir = "/var/run/qemu-server";
  87 mkdir $var_run_tmpdir;
  88
  89 my $lock_dir = "/var/lock/qemu-server";
  90 mkdir $lock_dir;
  91
  92 my $pcisysfs = "/sys/bus/pci";
  93
  94 my $cpudesc = {
  95     cputype => {
  96         description => "Emulated CPU type.",
  97         type => 'string',
  98         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  99         default => 'kvm64',
 100         default_key => 1,
 101     },
 102     hidden => {
 103         description => "Do not identify as a KVM virtual machine.",
 104         type => 'boolean',
 105         optional => 1,
 106         default => 0
 107     },
 108 };
 109
 110 my $confdesc = {
 111     onboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Specifies whether a VM will be started during system bootup.",
 115         default => 0,
 116     },
 117     autostart => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Automatic restart after crash (currently ignored).",
 121         default => 0,
 122     },
 123     hotplug => {
 124         optional => 1,
 125         type => 'string', format => 'pve-hotplug-features',
 126         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 127         default => 'network,disk,usb',
 128     },
 129     reboot => {
 130         optional => 1,
 131         type => 'boolean',
 132         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 133         default => 1,
 134     },
 135     lock => {
 136         optional => 1,
 137         type => 'string',
 138         description => "Lock/unlock the VM.",
 139         enum => [qw(migrate backup snapshot rollback)],
 140     },
 141     cpulimit => {
 142         optional => 1,
 143         type => 'number',
 144         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 145         minimum => 0,
 146         maximum => 128,
 147         default => 0,
 148     },
 149     cpuunits => {
 150         optional => 1,
 151         type => 'integer',
 152         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 153         minimum => 0,
 154         maximum => 500000,
 155         default => 1000,
 156     },
 157     memory => {
 158         optional => 1,
 159         type => 'integer',
 160         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 161         minimum => 16,
 162         default => 512,
 163     },
 164     balloon => {
 165         optional => 1,
 166         type => 'integer',
 167         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 168         minimum => 0,
 169     },
 170     shares => {
 171         optional => 1,
 172         type => 'integer',
 173         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 174         minimum => 0,
 175         maximum => 50000,
 176         default => 1000,
 177     },
 178     keyboard => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 182         enum => PVE::Tools::kvmkeymaplist(),
 183         default => 'en-us',
 184     },
 185     name => {
 186         optional => 1,
 187         type => 'string', format => 'dns-name',
 188         description => "Set a name for the VM. Only used on the configuration web interface.",
 189     },
 190     scsihw => {
 191         optional => 1,
 192         type => 'string',
 193         description => "scsi controller model",
 194         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 195         default => 'lsi',
 196     },
 197     description => {
 198         optional => 1,
 199         type => 'string',
 200         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 201     },
 202     ostype => {
 203         optional => 1,
 204         type => 'string',
 205         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 206         description => <<EODESC,
 207 Used to enable special optimization/features for specific
 208 operating systems:
 209
 210 other  => unspecified OS
 211 wxp    => Microsoft Windows XP
 212 w2k    => Microsoft Windows 2000
 213 w2k3   => Microsoft Windows 2003
 214 w2k8   => Microsoft Windows 2008
 215 wvista => Microsoft Windows Vista
 216 win7   => Microsoft Windows 7
 217 win8   => Microsoft Windows 8/2012
 218 l24    => Linux 2.4 Kernel
 219 l26    => Linux 2.6/3.X Kernel
 220 solaris => solaris/opensolaris/openindiania kernel
 221
 222 other|l24|l26|solaris                       ... no special behaviour
 223 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 224 EODESC
 225     },
 226     boot => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 230         pattern => '[acdn]{1,4}',
 231         default => 'cdn',
 232     },
 233     bootdisk => {
 234         optional => 1,
 235         type => 'string', format => 'pve-qm-bootdisk',
 236         description => "Enable booting from specified disk.",
 237         pattern => '(ide|sata|scsi|virtio)\d+',
 238     },
 239     smp => {
 240         optional => 1,
 241         type => 'integer',
 242         description => "The number of CPUs. Please use option -sockets instead.",
 243         minimum => 1,
 244         default => 1,
 245     },
 246     sockets => {
 247         optional => 1,
 248         type => 'integer',
 249         description => "The number of CPU sockets.",
 250         minimum => 1,
 251         default => 1,
 252     },
 253     cores => {
 254         optional => 1,
 255         type => 'integer',
 256         description => "The number of cores per socket.",
 257         minimum => 1,
 258         default => 1,
 259     },
 260     numa => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Enable/disable NUMA.",
 264         default => 0,
 265     },
 266     vcpus => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "Number of hotplugged vcpus.",
 270         minimum => 1,
 271         default => 0,
 272     },
 273     acpi => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable ACPI.",
 277         default => 1,
 278     },
 279     agent => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Enable/disable Qemu GuestAgent.",
 283         default => 0,
 284     },
 285     kvm => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Enable/disable KVM hardware virtualization.",
 289         default => 1,
 290     },
 291     tdf => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Enable/disable time drift fix.",
 295         default => 0,
 296     },
 297     localtime => {
 298         optional => 1,
 299         type => 'boolean',
 300         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 301     },
 302     freeze => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 306     },
 307     vga => {
 308         optional => 1,
 309         type => 'string',
 310         description => "Select the VGA type. If you want to use high resolution" .
 311             " modes (>= 1280x1024x16) then you should use the options " .
 312             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 313             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 314             "display sever. For win* OS you can select how many independent " .
 315             "displays you want, Linux guests can add displays them self. " .
 316             "You can also run without any graphic card, using a serial device" .
 317             " as terminal.",
 318         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 319     },
 320     watchdog => {
 321         optional => 1,
 322         type => 'string', format => 'pve-qm-watchdog',
 323         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 324         description => "Create a virtual hardware watchdog device. Once enabled" .
 325             " (by a guest action), the watchdog must be periodically polled " .
 326             "by an agent inside the guest or else the watchdog will reset " .
 327             "the guest (or execute the respective action specified)",
 328     },
 329     startdate => {
 330         optional => 1,
 331         type => 'string',
 332         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 333         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 334         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 335         default => 'now',
 336     },
 337     startup =>  get_standard_option('pve-startup-order'),
 338     template => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Enable/disable Template.",
 342         default => 0,
 343     },
 344     args => {
 345         optional => 1,
 346         type => 'string',
 347         description => <<EODESCR,
 348 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 349
 350 args: -no-reboot -no-hpet
 351 EODESCR
 352     },
 353     tablet => {
 354         optional => 1,
 355         type => 'boolean',
 356         default => 1,
 357         description => "Enable/disable the USB tablet device. This device is " .
 358             "usually needed to allow absolute mouse positioning with VNC. " .
 359             "Else the mouse runs out of sync with normal VNC clients. " .
 360             "If you're running lots of console-only guests on one host, " .
 361             "you may consider disabling this to save some context switches. " .
 362             "This is turned off by default if you use spice (-vga=qxl).",
 363     },
 364     migrate_speed => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 368         minimum => 0,
 369         default => 0,
 370     },
 371     migrate_downtime => {
 372         optional => 1,
 373         type => 'number',
 374         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 375         minimum => 0,
 376         default => 0.1,
 377     },
 378     cdrom => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-drive',
 381         typetext => 'volume',
 382         description => "This is an alias for option -ide2",
 383     },
 384     cpu => {
 385         optional => 1,
 386         description => "Emulated CPU type.",
 387         type => 'string',
 388         format => $cpudesc,
 389     },
 390     parent => get_standard_option('pve-snapshot-name', {
 391         optional => 1,
 392         description => "Parent snapshot name. This is used internally, and should not be modified.",
 393     }),
 394     snaptime => {
 395         optional => 1,
 396         description => "Timestamp for snapshots.",
 397         type => 'integer',
 398         minimum => 0,
 399     },
 400     vmstate => {
 401         optional => 1,
 402         type => 'string', format => 'pve-volume-id',
 403         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 404     },
 405     machine => {
 406         description => "Specific the Qemu machine type.",
 407         type => 'string',
 408         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 409         maxLength => 40,
 410         optional => 1,
 411     },
 412     smbios1 => {
 413         description => "Specify SMBIOS type 1 fields.",
 414         type => 'string', format => 'pve-qm-smbios1',
 415         maxLength => 256,
 416         optional => 1,
 417     },
 418     protection => {
 419         optional => 1,
 420         type => 'boolean',
 421         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 422         default => 0,
 423     },
 424     bios => {
 425         optional => 1,
 426         type => 'string',
 427         enum => [ qw(seabios ovmf) ],
 428         description => "Select BIOS implementation.",
 429         default => 'seabios',
 430     },
 431 };
 432
 433 # what about other qemu settings ?
 434 #cpu => 'string',
 435 #machine => 'string',
 436 #fda => 'file',
 437 #fdb => 'file',
 438 #mtdblock => 'file',
 439 #sd => 'file',
 440 #pflash => 'file',
 441 #snapshot => 'bool',
 442 #bootp => 'file',
 443 ##tftp => 'dir',
 444 ##smb => 'dir',
 445 #kernel => 'file',
 446 #append => 'string',
 447 #initrd => 'file',
 448 ##soundhw => 'string',
 449
 450 while (my ($k, $v) = each %$confdesc) {
 451     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 452 }
 453
 454 my $MAX_IDE_DISKS = 4;
 455 my $MAX_SCSI_DISKS = 14;
 456 my $MAX_VIRTIO_DISKS = 16;
 457 my $MAX_SATA_DISKS = 6;
 458 my $MAX_USB_DEVICES = 5;
 459 my $MAX_NETS = 32;
 460 my $MAX_UNUSED_DISKS = 8;
 461 my $MAX_HOSTPCI_DEVICES = 4;
 462 my $MAX_SERIAL_PORTS = 4;
 463 my $MAX_PARALLEL_PORTS = 3;
 464 my $MAX_NUMA = 8;
 465 my $MAX_MEM = 4194304;
 466 my $STATICMEM = 1024;
 467
 468 my $numadesc = {
 469     optional => 1,
 470     type => 'string', format => 'pve-qm-numanode',
 471     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 472     description => "numa topology",
 473 };
 474 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 475
 476 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 477     $confdesc->{"numa$i"} = $numadesc;
 478 }
 479
 480 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 481                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 482                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 483 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 484
 485 my $netdesc = {
 486     optional => 1,
 487     type => 'string', format => 'pve-qm-net',
 488     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 489     description => <<EODESCR,
 490 Specify network devices.
 491
 492 MODEL is one of: $nic_model_list_txt
 493
 494 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 495 automatically generated if not specified.
 496
 497 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 498
 499 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 500
 501 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 502
 503 10.0.2.2   Gateway
 504 10.0.2.3   DNS Server
 505 10.0.2.4   SMB Server
 506
 507 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 508
 509 EODESCR
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 512
 513 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 514     $confdesc->{"net$i"} = $netdesc;
 515 }
 516
 517 my $drivename_hash;
 518
 519 my %drivedesc_base = (
 520     volume => { alias => 'file' },
 521     file => {
 522         type => 'pve-volume-id',
 523         default_key => 1,
 524         format_description => 'volume',
 525         description => "The drive's backing volume.",
 526     },
 527     media => {
 528         type => 'string',
 529         format_description => 'cdrom|disk',
 530         enum => [qw(cdrom disk)],
 531         description => "The drive's media type.",
 532         default => 'disk',
 533         optional => 1
 534     },
 535     cyls => {
 536         type => 'integer',
 537         format_description => 'count',
 538         description => "Force the drive's physical geometry to have a specific cylinder count.",
 539         optional => 1
 540     },
 541     heads => {
 542         type => 'integer',
 543         format_description => 'count',
 544         description => "Force the drive's physical geometry to have a specific head count.",
 545         optional => 1
 546     },
 547     secs => {
 548         type => 'integer',
 549         format_description => 'count',
 550         description => "Force the drive's physical geometry to have a specific sector count.",
 551         optional => 1
 552     },
 553     trans => {
 554         type => 'string',
 555         format_description => 'none|lba|auto',
 556         enum => [qw(none lba auto)],
 557         description => "Force disk geometry bios translation mode.",
 558         optional => 1,
 559     },
 560     snapshot => {
 561         type => 'boolean',
 562         format_description => 'on|off',
 563         description => "Whether the drive should be included when making snapshots.",
 564         optional => 1,
 565     },
 566     cache => {
 567         type => 'string',
 568         format_description => 'none|writethrough|writeback|unsafe|directsync',
 569         enum => [qw(none writethrough writeback unsafe directsync)],
 570         description => "The drive's cache mode",
 571         optional => 1,
 572     },
 573     format => {
 574         type => 'string',
 575         format_description => 'drive format',
 576         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 577         description => "The drive's backing file's data format.",
 578         optional => 1,
 579     },
 580     size => {
 581         type => 'string',
 582         format => 'disk-size',
 583         description => "Disk size. This is purely informational and has no effect.",
 584         optional => 1,
 585     },
 586     backup => {
 587         type => 'boolean',
 588         format_description => 'on|off',
 589         description => "Whether the drive should be included when making backups.",
 590         optional => 1,
 591     },
 592     werror => {
 593         type => 'string',
 594         format_description => 'enospc|ignore|report|stop',
 595         enum => [qw(enospc ignore report stop)],
 596         description => 'Write error action.',
 597         optional => 1,
 598     },
 599     aio => {
 600         type => 'string',
 601         format_description => 'native|threads',
 602         enum => [qw(native threads)],
 603         description => 'AIO type to use.',
 604         optional => 1,
 605     },
 606     discard => {
 607         type => 'string',
 608         format_description => 'ignore|on',
 609         enum => [qw(ignore on)],
 610         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 611         optional => 1,
 612     },
 613     detect_zeroes => {
 614         type => 'boolean',
 615         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 616         optional => 1,
 617     },
 618     serial => {
 619         type => 'string',
 620         format_description => 'serial',
 621         description => "The drive's reported serial number.",
 622         optional => 1,
 623     }
 624 );
 625
 626 my %rerror_fmt = (
 627     rerror => {
 628         type => 'string',
 629         format_description => 'ignore|report|stop',
 630         enum => [qw(ignore report stop)],
 631         description => 'Read error action.',
 632         optional => 1,
 633     },
 634 );
 635
 636 my %iothread_fmt = ( iothread => {
 637         type => 'boolean',
 638         format_description => 'off|on',
 639         description => "Whether to use iothreads for this drive",
 640         optional => 1,
 641 });
 642
 643 my %model_fmt = (
 644     model => {
 645         type => 'string',
 646         format_description => 'model',
 647         description => "The drive's reported model name.",
 648         optional => 1,
 649     },
 650 );
 651
 652 my %queues_fmt = (
 653     queues => {
 654         type => 'integer',
 655         format_description => 'nbqueues',
 656         description => "Number of queues.",
 657         minimum => 2,
 658         optional => 1
 659     }
 660 );
 661
 662 my $add_throttle_desc = sub {
 663     my ($key, $type, $what, $size, $longsize) = @_;
 664     $drivedesc_base{$key} = {
 665         type => $type,
 666         format_description => $size,
 667         description => "Maximum $what speed in $longsize per second.",
 668         optional => 1,
 669     };
 670 };
 671 # throughput: (leaky bucket)
 672 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 673 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 674 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 675 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 676 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 677 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 678 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 679 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 680 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 681
 682 # pools: (pool of IO before throttling starts taking effect)
 683 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 684 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 685 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 686 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 687 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 688 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 689
 690 my $ide_fmt = {
 691     %drivedesc_base,
 692     %rerror_fmt,
 693     %model_fmt,
 694 };
 695
 696 my $idedesc = {
 697     optional => 1,
 698     type => 'string', format => $ide_fmt,
 699     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 700 };
 701 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 702
 703 my $scsi_fmt = {
 704     %drivedesc_base,
 705     %iothread_fmt,
 706     %queues_fmt,
 707 };
 708 my $scsidesc = {
 709     optional => 1,
 710     type => 'string', format => $scsi_fmt,
 711     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 712 };
 713 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 714
 715 my $sata_fmt = {
 716     %drivedesc_base,
 717     %rerror_fmt,
 718 };
 719 my $satadesc = {
 720     optional => 1,
 721     type => 'string', format => $sata_fmt,
 722     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 723 };
 724 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 725
 726 my $virtio_fmt = {
 727     %drivedesc_base,
 728     %iothread_fmt,
 729     %rerror_fmt,
 730 };
 731 my $virtiodesc = {
 732     optional => 1,
 733     type => 'string', format => $virtio_fmt,
 734     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 735 };
 736 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 737
 738 my $alldrive_fmt = {
 739     %drivedesc_base,
 740     %rerror_fmt,
 741     %iothread_fmt,
 742     %model_fmt,
 743     %queues_fmt,
 744 };
 745
 746 my $usbformat = {
 747     host => {
 748         default_key => 1,
 749         type => 'string', format => 'pve-qm-usb-device',
 750         format_description => 'HOSTUSBDEVICE|spice',
 751         description => 'The Host USB device or port or the value spice',
 752     },
 753     usb3 => {
 754         optional => 1,
 755         type => 'boolean',
 756         format_description => 'yes|no',
 757         description => 'Specifies whether if given host option is a USB3 device or port',
 758     },
 759 };
 760
 761 my $usbdesc = {
 762     optional => 1,
 763     type => 'string', format => $usbformat,
 764     description => <<EODESCR,
 765 Configure an USB device (n is 0 to 4). This can be used to
 766 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 767
 768 'bus-port(.port)*' (decimal numbers) or
 769 'vendor_id:product_id' (hexadeciaml numbers) or
 770 'spice'
 771
 772 You can use the 'lsusb -t' command to list existing usb devices.
 773
 774 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 775
 776 The value 'spice' can be used to add a usb redirection devices for spice.
 777
 778 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 779
 780 EODESCR
 781 };
 782 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 783
 784 my $hostpcidesc = {
 785         optional => 1,
 786         type => 'string', format => 'pve-qm-hostpci',
 787         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 788         description => <<EODESCR,
 789 Map host pci devices. HOSTPCIDEVICE syntax is:
 790
 791 'bus:dev.func' (hexadecimal numbers)
 792
 793 You can us the 'lspci' command to list existing pci devices.
 794
 795 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 796
 797 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 798
 799 Experimental: user reported problems with this option.
 800 EODESCR
 801 };
 802 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 803
 804 my $serialdesc = {
 805         optional => 1,
 806         type => 'string',
 807         pattern => '(/dev/.+|socket)',
 808         description =>  <<EODESCR,
 809 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 810
 811 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 812
 813 Experimental: user reported problems with this option.
 814 EODESCR
 815 };
 816
 817 my $paralleldesc= {
 818         optional => 1,
 819         type => 'string',
 820         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 821         description =>  <<EODESCR,
 822 Map host parallel devices (n is 0 to 2).
 823
 824 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 825
 826 Experimental: user reported problems with this option.
 827 EODESCR
 828 };
 829
 830 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 831     $confdesc->{"parallel$i"} = $paralleldesc;
 832 }
 833
 834 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 835     $confdesc->{"serial$i"} = $serialdesc;
 836 }
 837
 838 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 839     $confdesc->{"hostpci$i"} = $hostpcidesc;
 840 }
 841
 842 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 843     $drivename_hash->{"ide$i"} = 1;
 844     $confdesc->{"ide$i"} = $idedesc;
 845 }
 846
 847 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 848     $drivename_hash->{"sata$i"} = 1;
 849     $confdesc->{"sata$i"} = $satadesc;
 850 }
 851
 852 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 853     $drivename_hash->{"scsi$i"} = 1;
 854     $confdesc->{"scsi$i"} = $scsidesc ;
 855 }
 856
 857 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 858     $drivename_hash->{"virtio$i"} = 1;
 859     $confdesc->{"virtio$i"} = $virtiodesc;
 860 }
 861
 862 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 863     $confdesc->{"usb$i"} = $usbdesc;
 864 }
 865
 866 my $unuseddesc = {
 867     optional => 1,
 868     type => 'string', format => 'pve-volume-id',
 869     description => "Reference to unused volumes.",
 870 };
 871
 872 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 873     $confdesc->{"unused$i"} = $unuseddesc;
 874 }
 875
 876 my $kvm_api_version = 0;
 877
 878 sub kvm_version {
 879
 880     return $kvm_api_version if $kvm_api_version;
 881
 882     my $fh = IO::File->new("</dev/kvm") ||
 883         return 0;
 884
 885     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 886         $kvm_api_version = $v;
 887     }
 888
 889     $fh->close();
 890
 891     return  $kvm_api_version;
 892 }
 893
 894 my $kvm_user_version;
 895
 896 sub kvm_user_version {
 897
 898     return $kvm_user_version if $kvm_user_version;
 899
 900     $kvm_user_version = 'unknown';
 901
 902     my $code = sub {
 903         my $line = shift;
 904         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 905             $kvm_user_version = $2;
 906         }
 907     };
 908
 909     eval { run_command("kvm -version", outfunc => $code); };
 910     warn $@ if $@;
 911
 912     return $kvm_user_version;
 913
 914 }
 915
 916 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 917
 918 sub valid_drive_names {
 919     # order is important - used to autoselect boot disk
 920     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 921             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 922             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 923             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 924 }
 925
 926 sub is_valid_drivename {
 927     my $dev = shift;
 928
 929     return defined($drivename_hash->{$dev});
 930 }
 931
 932 sub option_exists {
 933     my $key = shift;
 934     return defined($confdesc->{$key});
 935 }
 936
 937 sub nic_models {
 938     return $nic_model_list;
 939 }
 940
 941 sub os_list_description {
 942
 943     return {
 944         other => 'Other',
 945         wxp => 'Windows XP',
 946         w2k => 'Windows 2000',
 947         w2k3 =>, 'Windows 2003',
 948         w2k8 => 'Windows 2008',
 949         wvista => 'Windows Vista',
 950         win7 => 'Windows 7',
 951         win8 => 'Windows 8/2012',
 952         l24 => 'Linux 2.4',
 953         l26 => 'Linux 2.6',
 954     };
 955 }
 956
 957 my $cdrom_path;
 958
 959 sub get_cdrom_path {
 960
 961     return  $cdrom_path if $cdrom_path;
 962
 963     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 964     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 965     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 966 }
 967
 968 sub get_iso_path {
 969     my ($storecfg, $vmid, $cdrom) = @_;
 970
 971     if ($cdrom eq 'cdrom') {
 972         return get_cdrom_path();
 973     } elsif ($cdrom eq 'none') {
 974         return '';
 975     } elsif ($cdrom =~ m|^/|) {
 976         return $cdrom;
 977     } else {
 978         return PVE::Storage::path($storecfg, $cdrom);
 979     }
 980 }
 981
 982 # try to convert old style file names to volume IDs
 983 sub filename_to_volume_id {
 984     my ($vmid, $file, $media) = @_;
 985
 986     if (!($file eq 'none' || $file eq 'cdrom' ||
 987           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 988
 989         return undef if $file =~ m|/|;
 990
 991         if ($media && $media eq 'cdrom') {
 992             $file = "local:iso/$file";
 993         } else {
 994             $file = "local:$vmid/$file";
 995         }
 996     }
 997
 998     return $file;
 999 }
1000
1001 sub verify_media_type {
1002     my ($opt, $vtype, $media) = @_;
1003
1004     return if !$media;
1005
1006     my $etype;
1007     if ($media eq 'disk') {
1008         $etype = 'images';
1009     } elsif ($media eq 'cdrom') {
1010         $etype = 'iso';
1011     } else {
1012         die "internal error";
1013     }
1014
1015     return if ($vtype eq $etype);
1016
1017     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1018 }
1019
1020 sub cleanup_drive_path {
1021     my ($opt, $storecfg, $drive) = @_;
1022
1023     # try to convert filesystem paths to volume IDs
1024
1025     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1026         ($drive->{file} !~ m|^/dev/.+|) &&
1027         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1028         ($drive->{file} !~ m/^\d+$/)) {
1029         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1030         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1031         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1032         verify_media_type($opt, $vtype, $drive->{media});
1033         $drive->{file} = $volid;
1034     }
1035
1036     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1037 }
1038
1039 sub parse_hotplug_features {
1040     my ($data) = @_;
1041
1042     my $res = {};
1043
1044     return $res if $data eq '0';
1045
1046     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1047
1048     foreach my $feature (PVE::Tools::split_list($data)) {
1049         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1050             $res->{$1} = 1;
1051         } else {
1052             warn "ignoring unknown hotplug feature '$feature'\n";
1053         }
1054     }
1055     return $res;
1056 }
1057
1058 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1059 sub pve_verify_hotplug_features {
1060     my ($value, $noerr) = @_;
1061
1062     return $value if parse_hotplug_features($value);
1063
1064     return undef if $noerr;
1065
1066     die "unable to parse hotplug option\n";
1067 }
1068
1069 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1070 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1071 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1072 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1073 #        [,iothread=on][,serial=serial][,model=model]
1074
1075 sub parse_drive {
1076     my ($key, $data) = @_;
1077
1078     my ($interface, $index);
1079
1080     if ($key =~ m/^([^\d]+)(\d+)$/) {
1081         $interface = $1;
1082         $index = $2;
1083     } else {
1084         return undef;
1085     }
1086
1087     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1088                                      : $confdesc->{$key}->{format};
1089     if (!$desc) {
1090         warn "invalid drive key: $key\n";
1091         return undef;
1092     }
1093     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1094     return undef if !$res;
1095     $res->{interface} = $interface;
1096     $res->{index} = $index;
1097
1098     my $error = 0;
1099     foreach my $opt (qw(bps bps_rd bps_wr)) {
1100         if (my $bps = defined(delete $res->{$opt})) {
1101             if (defined($res->{"m$opt"})) {
1102                 warn "both $opt and m$opt specified\n";
1103                 ++$error;
1104                 next;
1105             }
1106             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1107         }
1108     }
1109     return undef if $error;
1110
1111     return undef if $res->{mbps_rd} && $res->{mbps};
1112     return undef if $res->{mbps_wr} && $res->{mbps};
1113     return undef if $res->{iops_rd} && $res->{iops};
1114     return undef if $res->{iops_wr} && $res->{iops};
1115
1116     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1117         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1118         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1119         return undef if $res->{interface} eq 'virtio';
1120     }
1121
1122     if (my $size = $res->{size}) {
1123         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1124     }
1125
1126     return $res;
1127 }
1128
1129 sub print_drive {
1130     my ($vmid, $drive) = @_;
1131     my $data = { %$drive };
1132     delete $data->{$_} for qw(index interface);
1133     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1134 }
1135
1136 sub scsi_inquiry {
1137     my($fh, $noerr) = @_;
1138
1139     my $SG_IO = 0x2285;
1140     my $SG_GET_VERSION_NUM = 0x2282;
1141
1142     my $versionbuf = "\x00" x 8;
1143     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1144     if (!$ret) {
1145         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1146         return undef;
1147     }
1148     my $version = unpack("I", $versionbuf);
1149     if ($version < 30000) {
1150         die "scsi generic interface too old\n"  if !$noerr;
1151         return undef;
1152     }
1153
1154     my $buf = "\x00" x 36;
1155     my $sensebuf = "\x00" x 8;
1156     my $cmd = pack("C x3 C x1", 0x12, 36);
1157
1158     # see /usr/include/scsi/sg.h
1159     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1160
1161     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1162                       length($sensebuf), 0, length($buf), $buf,
1163                       $cmd, $sensebuf, 6000);
1164
1165     $ret = ioctl($fh, $SG_IO, $packet);
1166     if (!$ret) {
1167         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1168         return undef;
1169     }
1170
1171     my @res = unpack($sg_io_hdr_t, $packet);
1172     if ($res[17] || $res[18]) {
1173         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1174         return undef;
1175     }
1176
1177     my $res = {};
1178     (my $byte0, my $byte1, $res->{vendor},
1179      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1180
1181     $res->{removable} = $byte1 & 128 ? 1 : 0;
1182     $res->{type} = $byte0 & 31;
1183
1184     return $res;
1185 }
1186
1187 sub path_is_scsi {
1188     my ($path) = @_;
1189
1190     my $fh = IO::File->new("+<$path") || return undef;
1191     my $res = scsi_inquiry($fh, 1);
1192     close($fh);
1193
1194     return $res;
1195 }
1196
1197 sub machine_type_is_q35 {
1198     my ($conf) = @_;
1199
1200     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1201 }
1202
1203 sub print_tabletdevice_full {
1204     my ($conf) = @_;
1205
1206     my $q35 = machine_type_is_q35($conf);
1207
1208     # we use uhci for old VMs because tablet driver was buggy in older qemu
1209     my $usbbus = $q35 ? "ehci" : "uhci";
1210
1211     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1212 }
1213
1214 sub print_drivedevice_full {
1215     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1216
1217     my $device = '';
1218     my $maxdev = 0;
1219
1220     if ($drive->{interface} eq 'virtio') {
1221         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1222         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1223         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1224     } elsif ($drive->{interface} eq 'scsi') {
1225
1226         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1227         my $unit = $drive->{index} % $maxdev;
1228         my $devicetype = 'hd';
1229         my $path = '';
1230         if (drive_is_cdrom($drive)) {
1231             $devicetype = 'cd';
1232         } else {
1233             if ($drive->{file} =~ m|^/|) {
1234                 $path = $drive->{file};
1235                 if (my $info = path_is_scsi($path)) {
1236                     if ($info->{type} == 0) {
1237                         $devicetype = 'block';
1238                     } elsif ($info->{type} == 1) { # tape
1239                         $devicetype = 'generic';
1240                     }
1241                 }
1242             } else {
1243                  $path = PVE::Storage::path($storecfg, $drive->{file});
1244             }
1245
1246             if($path =~ m/^iscsi\:\/\//){
1247                 $devicetype = 'generic';
1248             }
1249         }
1250
1251         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1252             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1253         } else {
1254             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1255         }
1256
1257     } elsif ($drive->{interface} eq 'ide'){
1258         $maxdev = 2;
1259         my $controller = int($drive->{index} / $maxdev);
1260         my $unit = $drive->{index} % $maxdev;
1261         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1262
1263         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1264         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1265             $device .= ",model=$model";
1266         }
1267     } elsif ($drive->{interface} eq 'sata'){
1268         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1269         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1270         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1271     } elsif ($drive->{interface} eq 'usb') {
1272         die "implement me";
1273         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1274     } else {
1275         die "unsupported interface type";
1276     }
1277
1278     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1279
1280     return $device;
1281 }
1282
1283 sub get_initiator_name {
1284     my $initiator;
1285
1286     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1287     while (defined(my $line = <$fh>)) {
1288         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1289         $initiator = $1;
1290         last;
1291     }
1292     $fh->close();
1293
1294     return $initiator;
1295 }
1296
1297 sub print_drive_full {
1298     my ($storecfg, $vmid, $drive) = @_;
1299
1300     my $path;
1301     my $volid = $drive->{file};
1302     my $format;
1303
1304     if (drive_is_cdrom($drive)) {
1305         $path = get_iso_path($storecfg, $vmid, $volid);
1306     } else {
1307         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1308         if ($storeid) {
1309             $path = PVE::Storage::path($storecfg, $volid);
1310             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1311             $format = qemu_img_format($scfg, $volname);
1312         } else {
1313             $path = $volid;
1314             $format = "raw";
1315         }
1316    }
1317
1318     my $opts = '';
1319     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1320     foreach my $o (@qemu_drive_options) {
1321         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1322     }
1323
1324     $opts .= ",format=$format" if $format && !$drive->{format};
1325
1326     foreach my $o (qw(bps bps_rd bps_wr)) {
1327         my $v = $drive->{"m$o"};
1328         $opts .= ",$o=" . int($v*1024*1024) if $v;
1329     }
1330
1331     my $cache_direct = 0;
1332
1333     if (my $cache = $drive->{cache}) {
1334         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1335     } elsif (!drive_is_cdrom($drive)) {
1336         $opts .= ",cache=none";
1337         $cache_direct = 1;
1338     }
1339
1340     # aio native works only with O_DIRECT
1341     if (!$drive->{aio}) {
1342         if($cache_direct) {
1343             $opts .= ",aio=native";
1344         } else {
1345             $opts .= ",aio=threads";
1346         }
1347     }
1348
1349     if (!drive_is_cdrom($drive)) {
1350         my $detectzeroes;
1351         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1352             $detectzeroes = 'off';
1353         } elsif ($drive->{discard}) {
1354             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1355         } else {
1356             # This used to be our default with discard not being specified:
1357             $detectzeroes = 'on';
1358         }
1359         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1360     }
1361
1362     my $pathinfo = $path ? "file=$path," : '';
1363
1364     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1365 }
1366
1367 sub print_netdevice_full {
1368     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1369
1370     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1371
1372     my $device = $net->{model};
1373     if ($net->{model} eq 'virtio') {
1374          $device = 'virtio-net-pci';
1375      };
1376
1377     my $pciaddr = print_pci_addr("$netid", $bridges);
1378     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1379     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1380         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1381         my $vectors = $net->{queues} * 2 + 2;
1382         $tmpstr .= ",vectors=$vectors,mq=on";
1383     }
1384     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1385
1386     if ($use_old_bios_files) {
1387         my $romfile;
1388         if ($device eq 'virtio-net-pci') {
1389             $romfile = 'pxe-virtio.rom';
1390         } elsif ($device eq 'e1000') {
1391             $romfile = 'pxe-e1000.rom';
1392         } elsif ($device eq 'ne2k') {
1393             $romfile = 'pxe-ne2k_pci.rom';
1394         } elsif ($device eq 'pcnet') {
1395             $romfile = 'pxe-pcnet.rom';
1396         } elsif ($device eq 'rtl8139') {
1397             $romfile = 'pxe-rtl8139.rom';
1398         }
1399         $tmpstr .= ",romfile=$romfile" if $romfile;
1400     }
1401
1402     return $tmpstr;
1403 }
1404
1405 sub print_netdev_full {
1406     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1407
1408     my $i = '';
1409     if ($netid =~ m/^net(\d+)$/) {
1410         $i = int($1);
1411     }
1412
1413     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1414
1415     my $ifname = "tap${vmid}i$i";
1416
1417     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1418     die "interface name '$ifname' is too long (max 15 character)\n"
1419         if length($ifname) >= 16;
1420
1421     my $vhostparam = '';
1422     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1423
1424     my $vmname = $conf->{name} || "vm$vmid";
1425
1426     my $netdev = "";
1427     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1428
1429     if ($net->{bridge}) {
1430         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1431     } else {
1432         $netdev = "type=user,id=$netid,hostname=$vmname";
1433     }
1434
1435     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1436
1437     return $netdev;
1438 }
1439
1440 sub drive_is_cdrom {
1441     my ($drive) = @_;
1442
1443     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1444
1445 }
1446
1447 sub parse_numa {
1448     my ($data) = @_;
1449
1450     my $res = {};
1451
1452     foreach my $kvp (split(/,/, $data)) {
1453
1454         if ($kvp =~ m/^memory=(\S+)$/) {
1455             $res->{memory} = $1;
1456         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1457             $res->{policy} = $1;
1458         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1459             $res->{cpus}->{start} = $1;
1460             $res->{cpus}->{end} = $3;
1461         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1462             $res->{hostnodes}->{start} = $1;
1463             $res->{hostnodes}->{end} = $3;
1464         } else {
1465             return undef;
1466         }
1467     }
1468
1469     return $res;
1470 }
1471
1472 sub parse_hostpci {
1473     my ($value) = @_;
1474
1475     return undef if !$value;
1476
1477
1478     my @list = split(/,/, $value);
1479     my $found;
1480
1481     my $res = {};
1482     foreach my $kv (@list) {
1483
1484         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1485             $found = 1;
1486             if(defined($4)){
1487                 push @{$res->{pciid}}, { id => $2 , function => $4};
1488
1489             }else{
1490                 my $pcidevices = lspci($2);
1491                 $res->{pciid} = $pcidevices->{$2};
1492             }
1493         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1494             $res->{rombar} = $1;
1495         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1496             $res->{'x-vga'} = $1;
1497         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1498             $res->{pcie} = 1 if $1 == 1;
1499         } else {
1500             warn "unknown hostpci setting '$kv'\n";
1501         }
1502     }
1503
1504     return undef if !$found;
1505
1506     return $res;
1507 }
1508
1509 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1510 sub parse_net {
1511     my ($data) = @_;
1512
1513     my $res = {};
1514
1515     foreach my $kvp (split(/,/, $data)) {
1516
1517         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1518             my $model = lc($1);
1519             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1520             $res->{model} = $model;
1521             $res->{macaddr} = $mac;
1522         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1523             $res->{bridge} = $1;
1524         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1525             $res->{queues} = $1;
1526         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1527             $res->{rate} = $1;
1528         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1529             $res->{tag} = $1;
1530         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1531             $res->{trunks} = $1;
1532         } elsif ($kvp =~ m/^firewall=([01])$/) {
1533             $res->{firewall} = $1;
1534         } elsif ($kvp =~ m/^link_down=([01])$/) {
1535             $res->{link_down} = $1;
1536         } else {
1537             return undef;
1538         }
1539
1540     }
1541
1542     return undef if !$res->{model};
1543
1544     return $res;
1545 }
1546
1547 sub print_net {
1548     my $net = shift;
1549
1550     my $res = "$net->{model}";
1551     $res .= "=$net->{macaddr}" if $net->{macaddr};
1552     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1553     $res .= ",rate=$net->{rate}" if $net->{rate};
1554     $res .= ",tag=$net->{tag}" if $net->{tag};
1555     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1556     $res .= ",firewall=1" if $net->{firewall};
1557     $res .= ",link_down=1" if $net->{link_down};
1558     $res .= ",queues=$net->{queues}" if $net->{queues};
1559
1560     return $res;
1561 }
1562
1563 sub add_random_macs {
1564     my ($settings) = @_;
1565
1566     foreach my $opt (keys %$settings) {
1567         next if $opt !~ m/^net(\d+)$/;
1568         my $net = parse_net($settings->{$opt});
1569         next if !$net;
1570         $settings->{$opt} = print_net($net);
1571     }
1572 }
1573
1574 sub vm_is_volid_owner {
1575     my ($storecfg, $vmid, $volid) = @_;
1576
1577     if ($volid !~  m|^/|) {
1578         my ($path, $owner);
1579         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1580         if ($owner && ($owner == $vmid)) {
1581             return 1;
1582         }
1583     }
1584
1585     return undef;
1586 }
1587
1588 sub split_flagged_list {
1589     my $text = shift || '';
1590     $text =~ s/[,;]/ /g;
1591     $text =~ s/^\s+//;
1592     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1593 }
1594
1595 sub join_flagged_list {
1596     my ($how, $lst) = @_;
1597     join $how, map { $lst->{$_} . $_ } keys %$lst;
1598 }
1599
1600 sub vmconfig_delete_pending_option {
1601     my ($conf, $key, $force) = @_;
1602
1603     delete $conf->{pending}->{$key};
1604     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1605     $pending_delete_hash->{$key} = $force ? '!' : '';
1606     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1607 }
1608
1609 sub vmconfig_undelete_pending_option {
1610     my ($conf, $key) = @_;
1611
1612     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1613     delete $pending_delete_hash->{$key};
1614
1615     if (%$pending_delete_hash) {
1616         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1617     } else {
1618         delete $conf->{pending}->{delete};
1619     }
1620 }
1621
1622 sub vmconfig_register_unused_drive {
1623     my ($storecfg, $vmid, $conf, $drive) = @_;
1624
1625     if (!drive_is_cdrom($drive)) {
1626         my $volid = $drive->{file};
1627         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1628             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1629         }
1630     }
1631 }
1632
1633 sub vmconfig_cleanup_pending {
1634     my ($conf) = @_;
1635
1636     # remove pending changes when nothing changed
1637     my $changes;
1638     foreach my $opt (keys %{$conf->{pending}}) {
1639         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1640             $changes = 1;
1641             delete $conf->{pending}->{$opt};
1642         }
1643     }
1644
1645     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1646     my $pending_delete_hash = {};
1647     while (my ($opt, $force) = each %$current_delete_hash) {
1648         if (defined($conf->{$opt})) {
1649             $pending_delete_hash->{$opt} = $force;
1650         } else {
1651             $changes = 1;
1652         }
1653     }
1654
1655     if (%$pending_delete_hash) {
1656         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1657     } else {
1658         delete $conf->{pending}->{delete};
1659     }
1660
1661     return $changes;
1662 }
1663
1664 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1665 my $smbios1_desc = {
1666     uuid => {
1667         type => 'string',
1668         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1669         format_description => 'UUID',
1670         optional => 1,
1671     },
1672     version => {
1673         type => 'string',
1674         pattern => '\S+',
1675         format_description => 'str',
1676         optional => 1,
1677     },
1678     serial => {
1679         type => 'string',
1680         pattern => '\S+',
1681         format_description => 'str',
1682         optional => 1,
1683     },
1684     manufacturer => {
1685         type => 'string',
1686         pattern => '\S+',
1687         format_description => 'name',
1688         optional => 1,
1689     },
1690     product => {
1691         type => 'string',
1692         pattern => '\S+',
1693         format_description => 'name',
1694         optional => 1,
1695     },
1696     sku => {
1697         type => 'string',
1698         pattern => '\S+',
1699         format_description => 'str',
1700         optional => 1,
1701     },
1702     family => {
1703         type => 'string',
1704         pattern => '\S+',
1705         format_description => 'str',
1706         optional => 1,
1707     },
1708 };
1709
1710 sub parse_smbios1 {
1711     my ($data) = @_;
1712
1713     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1714     warn $@ if $@;
1715     return $res;
1716 }
1717
1718 sub print_smbios1 {
1719     my ($smbios1) = @_;
1720     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1721 }
1722
1723 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1724
1725 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1726 sub verify_bootdisk {
1727     my ($value, $noerr) = @_;
1728
1729     return $value if is_valid_drivename($value);
1730
1731     return undef if $noerr;
1732
1733     die "invalid boot disk '$value'\n";
1734 }
1735
1736 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1737 sub verify_numa {
1738     my ($value, $noerr) = @_;
1739
1740     return $value if parse_numa($value);
1741
1742     return undef if $noerr;
1743
1744     die "unable to parse numa options\n";
1745 }
1746
1747 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1748 sub verify_net {
1749     my ($value, $noerr) = @_;
1750
1751     return $value if parse_net($value);
1752
1753     return undef if $noerr;
1754
1755     die "unable to parse network options\n";
1756 }
1757
1758 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1759 sub verify_hostpci {
1760     my ($value, $noerr) = @_;
1761
1762     return $value if parse_hostpci($value);
1763
1764     return undef if $noerr;
1765
1766     die "unable to parse pci id\n";
1767 }
1768
1769 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1770 sub verify_watchdog {
1771     my ($value, $noerr) = @_;
1772
1773     return $value if parse_watchdog($value);
1774
1775     return undef if $noerr;
1776
1777     die "unable to parse watchdog options\n";
1778 }
1779
1780 sub parse_watchdog {
1781     my ($value) = @_;
1782
1783     return undef if !$value;
1784
1785     my $res = {};
1786
1787     foreach my $p (split(/,/, $value)) {
1788         next if $p =~ m/^\s*$/;
1789
1790         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1791             $res->{model} = $2;
1792         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1793             $res->{action} = $2;
1794         } else {
1795             return undef;
1796         }
1797     }
1798
1799     return $res;
1800 }
1801
1802 sub parse_usb_device {
1803     my ($value) = @_;
1804
1805     return undef if !$value;
1806
1807     my $res = {};
1808     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1809         $res->{vendorid} = $2;
1810         $res->{productid} = $4;
1811     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1812         $res->{hostbus} = $1;
1813         $res->{hostport} = $2;
1814     } elsif ($value =~ m/^spice$/i) {
1815         $res->{spice} = 1;
1816     } else {
1817         return undef;
1818     }
1819
1820     return $res;
1821 }
1822
1823 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1824 sub verify_usb_device {
1825     my ($value, $noerr) = @_;
1826
1827     return $value if parse_usb_device($value);
1828
1829     return undef if $noerr;
1830
1831     die "unable to parse usb device\n";
1832 }
1833
1834 # add JSON properties for create and set function
1835 sub json_config_properties {
1836     my $prop = shift;
1837
1838     foreach my $opt (keys %$confdesc) {
1839         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1840         $prop->{$opt} = $confdesc->{$opt};
1841     }
1842
1843     return $prop;
1844 }
1845
1846 sub check_type {
1847     my ($key, $value) = @_;
1848
1849     die "unknown setting '$key'\n" if !$confdesc->{$key};
1850
1851     my $type = $confdesc->{$key}->{type};
1852
1853     if (!defined($value)) {
1854         die "got undefined value\n";
1855     }
1856
1857     if ($value =~ m/[\n\r]/) {
1858         die "property contains a line feed\n";
1859     }
1860
1861     if ($type eq 'boolean') {
1862         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1863         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1864         die "type check ('boolean') failed - got '$value'\n";
1865     } elsif ($type eq 'integer') {
1866         return int($1) if $value =~ m/^(\d+)$/;
1867         die "type check ('integer') failed - got '$value'\n";
1868     } elsif ($type eq 'number') {
1869         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1870         die "type check ('number') failed - got '$value'\n";
1871     } elsif ($type eq 'string') {
1872         if (my $fmt = $confdesc->{$key}->{format}) {
1873             if ($fmt eq 'pve-qm-drive') {
1874                 # special case - we need to pass $key to parse_drive()
1875                 my $drive = parse_drive($key, $value);
1876                 return $value if $drive;
1877                 die "unable to parse drive options\n";
1878             }
1879             PVE::JSONSchema::check_format($fmt, $value);
1880             return $value;
1881         }
1882         $value =~ s/^\"(.*)\"$/$1/;
1883         return $value;
1884     } else {
1885         die "internal error"
1886     }
1887 }
1888
1889 sub check_iommu_support{
1890     #fixme : need to check IOMMU support
1891     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1892
1893     my $iommu=1;
1894     return $iommu;
1895
1896 }
1897
1898 sub touch_config {
1899     my ($vmid) = @_;
1900
1901     my $conf = PVE::QemuConfig->config_file($vmid);
1902     utime undef, undef, $conf;
1903 }
1904
1905 sub destroy_vm {
1906     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1907
1908     my $conffile = PVE::QemuConfig->config_file($vmid);
1909
1910     my $conf = PVE::QemuConfig->load_config($vmid);
1911
1912     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1913
1914     # only remove disks owned by this VM
1915     foreach_drive($conf, sub {
1916         my ($ds, $drive) = @_;
1917
1918         return if drive_is_cdrom($drive);
1919
1920         my $volid = $drive->{file};
1921
1922         return if !$volid || $volid =~ m|^/|;
1923
1924         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1925         return if !$path || !$owner || ($owner != $vmid);
1926
1927         PVE::Storage::vdisk_free($storecfg, $volid);
1928     });
1929
1930     if ($keep_empty_config) {
1931         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1932     } else {
1933         unlink $conffile;
1934     }
1935
1936     # also remove unused disk
1937     eval {
1938         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1939
1940         eval {
1941             PVE::Storage::foreach_volid($dl, sub {
1942                 my ($volid, $sid, $volname, $d) = @_;
1943                 PVE::Storage::vdisk_free($storecfg, $volid);
1944             });
1945         };
1946         warn $@ if $@;
1947
1948     };
1949     warn $@ if $@;
1950 }
1951
1952 sub parse_vm_config {
1953     my ($filename, $raw) = @_;
1954
1955     return undef if !defined($raw);
1956
1957     my $res = {
1958         digest => Digest::SHA::sha1_hex($raw),
1959         snapshots => {},
1960         pending => {},
1961     };
1962
1963     $filename =~ m|/qemu-server/(\d+)\.conf$|
1964         || die "got strange filename '$filename'";
1965
1966     my $vmid = $1;
1967
1968     my $conf = $res;
1969     my $descr;
1970     my $section = '';
1971
1972     my @lines = split(/\n/, $raw);
1973     foreach my $line (@lines) {
1974         next if $line =~ m/^\s*$/;
1975
1976         if ($line =~ m/^\[PENDING\]\s*$/i) {
1977             $section = 'pending';
1978             if (defined($descr)) {
1979                 $descr =~ s/\s+$//;
1980                 $conf->{description} = $descr;
1981             }
1982             $descr = undef;
1983             $conf = $res->{$section} = {};
1984             next;
1985
1986         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1987             $section = $1;
1988             if (defined($descr)) {
1989                 $descr =~ s/\s+$//;
1990                 $conf->{description} = $descr;
1991             }
1992             $descr = undef;
1993             $conf = $res->{snapshots}->{$section} = {};
1994             next;
1995         }
1996
1997         if ($line =~ m/^\#(.*)\s*$/) {
1998             $descr = '' if !defined($descr);
1999             $descr .= PVE::Tools::decode_text($1) . "\n";
2000             next;
2001         }
2002
2003         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2004             $descr = '' if !defined($descr);
2005             $descr .= PVE::Tools::decode_text($2);
2006         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2007             $conf->{snapstate} = $1;
2008         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2009             my $key = $1;
2010             my $value = $2;
2011             $conf->{$key} = $value;
2012         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2013             my $value = $1;
2014             if ($section eq 'pending') {
2015                 $conf->{delete} = $value; # we parse this later
2016             } else {
2017                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2018             }
2019         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2020             my $key = $1;
2021             my $value = $2;
2022             eval { $value = check_type($key, $value); };
2023             if ($@) {
2024                 warn "vm $vmid - unable to parse value of '$key' - $@";
2025             } else {
2026                 my $fmt = $confdesc->{$key}->{format};
2027                 if ($fmt && $fmt eq 'pve-qm-drive') {
2028                     my $v = parse_drive($key, $value);
2029                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2030                         $v->{file} = $volid;
2031                         $value = print_drive($vmid, $v);
2032                     } else {
2033                         warn "vm $vmid - unable to parse value of '$key'\n";
2034                         next;
2035                     }
2036                 }
2037
2038                 if ($key eq 'cdrom') {
2039                     $conf->{ide2} = $value;
2040                 } else {
2041                     $conf->{$key} = $value;
2042                 }
2043             }
2044         }
2045     }
2046
2047     if (defined($descr)) {
2048         $descr =~ s/\s+$//;
2049         $conf->{description} = $descr;
2050     }
2051     delete $res->{snapstate}; # just to be sure
2052
2053     return $res;
2054 }
2055
2056 sub write_vm_config {
2057     my ($filename, $conf) = @_;
2058
2059     delete $conf->{snapstate}; # just to be sure
2060
2061     if ($conf->{cdrom}) {
2062         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2063         $conf->{ide2} = $conf->{cdrom};
2064         delete $conf->{cdrom};
2065     }
2066
2067     # we do not use 'smp' any longer
2068     if ($conf->{sockets}) {
2069         delete $conf->{smp};
2070     } elsif ($conf->{smp}) {
2071         $conf->{sockets} = $conf->{smp};
2072         delete $conf->{cores};
2073         delete $conf->{smp};
2074     }
2075
2076     my $used_volids = {};
2077
2078     my $cleanup_config = sub {
2079         my ($cref, $pending, $snapname) = @_;
2080
2081         foreach my $key (keys %$cref) {
2082             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2083                 $key eq 'snapstate' || $key eq 'pending';
2084             my $value = $cref->{$key};
2085             if ($key eq 'delete') {
2086                 die "propertry 'delete' is only allowed in [PENDING]\n"
2087                     if !$pending;
2088                 # fixme: check syntax?
2089                 next;
2090             }
2091             eval { $value = check_type($key, $value); };
2092             die "unable to parse value of '$key' - $@" if $@;
2093
2094             $cref->{$key} = $value;
2095
2096             if (!$snapname && is_valid_drivename($key)) {
2097                 my $drive = parse_drive($key, $value);
2098                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2099             }
2100         }
2101     };
2102
2103     &$cleanup_config($conf);
2104
2105     &$cleanup_config($conf->{pending}, 1);
2106
2107     foreach my $snapname (keys %{$conf->{snapshots}}) {
2108         die "internal error" if $snapname eq 'pending';
2109         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2110     }
2111
2112     # remove 'unusedX' settings if we re-add a volume
2113     foreach my $key (keys %$conf) {
2114         my $value = $conf->{$key};
2115         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2116             delete $conf->{$key};
2117         }
2118     }
2119
2120     my $generate_raw_config = sub {
2121         my ($conf, $pending) = @_;
2122
2123         my $raw = '';
2124
2125         # add description as comment to top of file
2126         if (defined(my $descr = $conf->{description})) {
2127             if ($descr) {
2128                 foreach my $cl (split(/\n/, $descr)) {
2129                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2130                 }
2131             } else {
2132                 $raw .= "#\n" if $pending;
2133             }
2134         }
2135
2136         foreach my $key (sort keys %$conf) {
2137             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2138             $raw .= "$key: $conf->{$key}\n";
2139         }
2140         return $raw;
2141     };
2142
2143     my $raw = &$generate_raw_config($conf);
2144
2145     if (scalar(keys %{$conf->{pending}})){
2146         $raw .= "\n[PENDING]\n";
2147         $raw .= &$generate_raw_config($conf->{pending}, 1);
2148     }
2149
2150     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2151         $raw .= "\n[$snapname]\n";
2152         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2153     }
2154
2155     return $raw;
2156 }
2157
2158 sub load_defaults {
2159
2160     my $res = {};
2161
2162     # we use static defaults from our JSON schema configuration
2163     foreach my $key (keys %$confdesc) {
2164         if (defined(my $default = $confdesc->{$key}->{default})) {
2165             $res->{$key} = $default;
2166         }
2167     }
2168
2169     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2170     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2171
2172     return $res;
2173 }
2174
2175 sub config_list {
2176     my $vmlist = PVE::Cluster::get_vmlist();
2177     my $res = {};
2178     return $res if !$vmlist || !$vmlist->{ids};
2179     my $ids = $vmlist->{ids};
2180
2181     foreach my $vmid (keys %$ids) {
2182         my $d = $ids->{$vmid};
2183         next if !$d->{node} || $d->{node} ne $nodename;
2184         next if !$d->{type} || $d->{type} ne 'qemu';
2185         $res->{$vmid}->{exists} = 1;
2186     }
2187     return $res;
2188 }
2189
2190 # test if VM uses local resources (to prevent migration)
2191 sub check_local_resources {
2192     my ($conf, $noerr) = @_;
2193
2194     my $loc_res = 0;
2195
2196     $loc_res = 1 if $conf->{hostusb}; # old syntax
2197     $loc_res = 1 if $conf->{hostpci}; # old syntax
2198
2199     foreach my $k (keys %$conf) {
2200         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2201         # sockets are safe: they will recreated be on the target side post-migrate
2202         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2203         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2204     }
2205
2206     die "VM uses local resources\n" if $loc_res && !$noerr;
2207
2208     return $loc_res;
2209 }
2210
2211 # check if used storages are available on all nodes (use by migrate)
2212 sub check_storage_availability {
2213     my ($storecfg, $conf, $node) = @_;
2214
2215     foreach_drive($conf, sub {
2216         my ($ds, $drive) = @_;
2217
2218         my $volid = $drive->{file};
2219         return if !$volid;
2220
2221         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2222         return if !$sid;
2223
2224         # check if storage is available on both nodes
2225         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2226         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2227    });
2228 }
2229
2230 # list nodes where all VM images are available (used by has_feature API)
2231 sub shared_nodes {
2232     my ($conf, $storecfg) = @_;
2233
2234     my $nodelist = PVE::Cluster::get_nodelist();
2235     my $nodehash = { map { $_ => 1 } @$nodelist };
2236     my $nodename = PVE::INotify::nodename();
2237
2238     foreach_drive($conf, sub {
2239         my ($ds, $drive) = @_;
2240
2241         my $volid = $drive->{file};
2242         return if !$volid;
2243
2244         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2245         if ($storeid) {
2246             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2247             if ($scfg->{disable}) {
2248                 $nodehash = {};
2249             } elsif (my $avail = $scfg->{nodes}) {
2250                 foreach my $node (keys %$nodehash) {
2251                     delete $nodehash->{$node} if !$avail->{$node};
2252                 }
2253             } elsif (!$scfg->{shared}) {
2254                 foreach my $node (keys %$nodehash) {
2255                     delete $nodehash->{$node} if $node ne $nodename
2256                 }
2257             }
2258         }
2259     });
2260
2261     return $nodehash
2262 }
2263
2264 sub check_cmdline {
2265     my ($pidfile, $pid) = @_;
2266
2267     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2268     if (defined($fh)) {
2269         my $line = <$fh>;
2270         $fh->close;
2271         return undef if !$line;
2272         my @param = split(/\0/, $line);
2273
2274         my $cmd = $param[0];
2275         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2276
2277         for (my $i = 0; $i < scalar (@param); $i++) {
2278             my $p = $param[$i];
2279             next if !$p;
2280             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2281                 my $p = $param[$i+1];
2282                 return 1 if $p && ($p eq $pidfile);
2283                 return undef;
2284             }
2285         }
2286     }
2287     return undef;
2288 }
2289
2290 sub check_running {
2291     my ($vmid, $nocheck, $node) = @_;
2292
2293     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2294
2295     die "unable to find configuration file for VM $vmid - no such machine\n"
2296         if !$nocheck && ! -f $filename;
2297
2298     my $pidfile = pidfile_name($vmid);
2299
2300     if (my $fd = IO::File->new("<$pidfile")) {
2301         my $st = stat($fd);
2302         my $line = <$fd>;
2303         close($fd);
2304
2305         my $mtime = $st->mtime;
2306         if ($mtime > time()) {
2307             warn "file '$filename' modified in future\n";
2308         }
2309
2310         if ($line =~ m/^(\d+)$/) {
2311             my $pid = $1;
2312             if (check_cmdline($pidfile, $pid)) {
2313                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2314                     return $pid;
2315                 }
2316             }
2317         }
2318     }
2319
2320     return undef;
2321 }
2322
2323 sub vzlist {
2324
2325     my $vzlist = config_list();
2326
2327     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2328
2329     while (defined(my $de = $fd->read)) {
2330         next if $de !~ m/^(\d+)\.pid$/;
2331         my $vmid = $1;
2332         next if !defined($vzlist->{$vmid});
2333         if (my $pid = check_running($vmid)) {
2334             $vzlist->{$vmid}->{pid} = $pid;
2335         }
2336     }
2337
2338     return $vzlist;
2339 }
2340
2341 sub disksize {
2342     my ($storecfg, $conf) = @_;
2343
2344     my $bootdisk = $conf->{bootdisk};
2345     return undef if !$bootdisk;
2346     return undef if !is_valid_drivename($bootdisk);
2347
2348     return undef if !$conf->{$bootdisk};
2349
2350     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2351     return undef if !defined($drive);
2352
2353     return undef if drive_is_cdrom($drive);
2354
2355     my $volid = $drive->{file};
2356     return undef if !$volid;
2357
2358     return $drive->{size};
2359 }
2360
2361 my $last_proc_pid_stat;
2362
2363 # get VM status information
2364 # This must be fast and should not block ($full == false)
2365 # We only query KVM using QMP if $full == true (this can be slow)
2366 sub vmstatus {
2367     my ($opt_vmid, $full) = @_;
2368
2369     my $res = {};
2370
2371     my $storecfg = PVE::Storage::config();
2372
2373     my $list = vzlist();
2374     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2375
2376     my $cpucount = $cpuinfo->{cpus} || 1;
2377
2378     foreach my $vmid (keys %$list) {
2379         next if $opt_vmid && ($vmid ne $opt_vmid);
2380
2381         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2382         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2383
2384         my $d = {};
2385         $d->{pid} = $list->{$vmid}->{pid};
2386
2387         # fixme: better status?
2388         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2389
2390         my $size = disksize($storecfg, $conf);
2391         if (defined($size)) {
2392             $d->{disk} = 0; # no info available
2393             $d->{maxdisk} = $size;
2394         } else {
2395             $d->{disk} = 0;
2396             $d->{maxdisk} = 0;
2397         }
2398
2399         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2400         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2401         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2402
2403         $d->{name} = $conf->{name} || "VM $vmid";
2404         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2405
2406         if ($conf->{balloon}) {
2407             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2408             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2409         }
2410
2411         $d->{uptime} = 0;
2412         $d->{cpu} = 0;
2413         $d->{mem} = 0;
2414
2415         $d->{netout} = 0;
2416         $d->{netin} = 0;
2417
2418         $d->{diskread} = 0;
2419         $d->{diskwrite} = 0;
2420
2421         $d->{template} = PVE::QemuConfig->is_template($conf);
2422
2423         $res->{$vmid} = $d;
2424     }
2425
2426     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2427     foreach my $dev (keys %$netdev) {
2428         next if $dev !~ m/^tap([1-9]\d*)i/;
2429         my $vmid = $1;
2430         my $d = $res->{$vmid};
2431         next if !$d;
2432
2433         $d->{netout} += $netdev->{$dev}->{receive};
2434         $d->{netin} += $netdev->{$dev}->{transmit};
2435
2436         if ($full) {
2437             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2438             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2439         }
2440
2441     }
2442
2443     my $ctime = gettimeofday;
2444
2445     foreach my $vmid (keys %$list) {
2446
2447         my $d = $res->{$vmid};
2448         my $pid = $d->{pid};
2449         next if !$pid;
2450
2451         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2452         next if !$pstat; # not running
2453
2454         my $used = $pstat->{utime} + $pstat->{stime};
2455
2456         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2457
2458         if ($pstat->{vsize}) {
2459             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2460         }
2461
2462         my $old = $last_proc_pid_stat->{$pid};
2463         if (!$old) {
2464             $last_proc_pid_stat->{$pid} = {
2465                 time => $ctime,
2466                 used => $used,
2467                 cpu => 0,
2468             };
2469             next;
2470         }
2471
2472         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2473
2474         if ($dtime > 1000) {
2475             my $dutime = $used -  $old->{used};
2476
2477             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2478             $last_proc_pid_stat->{$pid} = {
2479                 time => $ctime,
2480                 used => $used,
2481                 cpu => $d->{cpu},
2482             };
2483         } else {
2484             $d->{cpu} = $old->{cpu};
2485         }
2486     }
2487
2488     return $res if !$full;
2489
2490     my $qmpclient = PVE::QMPClient->new();
2491
2492     my $ballooncb = sub {
2493         my ($vmid, $resp) = @_;
2494
2495         my $info = $resp->{'return'};
2496         return if !$info->{max_mem};
2497
2498         my $d = $res->{$vmid};
2499
2500         # use memory assigned to VM
2501         $d->{maxmem} = $info->{max_mem};
2502         $d->{balloon} = $info->{actual};
2503
2504         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2505             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2506             $d->{freemem} = $info->{free_mem};
2507         }
2508
2509         $d->{ballooninfo} = $info;
2510     };
2511
2512     my $blockstatscb = sub {
2513         my ($vmid, $resp) = @_;
2514         my $data = $resp->{'return'} || [];
2515         my $totalrdbytes = 0;
2516         my $totalwrbytes = 0;
2517
2518         for my $blockstat (@$data) {
2519             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2520             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2521
2522             $blockstat->{device} =~ s/drive-//;
2523             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2524         }
2525         $res->{$vmid}->{diskread} = $totalrdbytes;
2526         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2527     };
2528
2529     my $statuscb = sub {
2530         my ($vmid, $resp) = @_;
2531
2532         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2533         # this fails if ballon driver is not loaded, so this must be
2534         # the last commnand (following command are aborted if this fails).
2535         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2536
2537         my $status = 'unknown';
2538         if (!defined($status = $resp->{'return'}->{status})) {
2539             warn "unable to get VM status\n";
2540             return;
2541         }
2542
2543         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2544     };
2545
2546     foreach my $vmid (keys %$list) {
2547         next if $opt_vmid && ($vmid ne $opt_vmid);
2548         next if !$res->{$vmid}->{pid}; # not running
2549         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2550     }
2551
2552     $qmpclient->queue_execute(undef, 1);
2553
2554     foreach my $vmid (keys %$list) {
2555         next if $opt_vmid && ($vmid ne $opt_vmid);
2556         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2557     }
2558
2559     return $res;
2560 }
2561
2562 sub foreach_dimm {
2563     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2564
2565     my $dimm_id = 0;
2566     my $current_size = 1024;
2567     my $dimm_size = 512;
2568     return if $current_size == $memory;
2569
2570     for (my $j = 0; $j < 8; $j++) {
2571         for (my $i = 0; $i < 32; $i++) {
2572             my $name = "dimm${dimm_id}";
2573             $dimm_id++;
2574             my $numanode = $i % $sockets;
2575             $current_size += $dimm_size;
2576             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2577             return  $current_size if $current_size >= $memory;
2578         }
2579         $dimm_size *= 2;
2580     }
2581 }
2582
2583 sub foreach_reverse_dimm {
2584     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2585
2586     my $dimm_id = 253;
2587     my $current_size = 4177920;
2588     my $dimm_size = 65536;
2589     return if $current_size == $memory;
2590
2591     for (my $j = 0; $j < 8; $j++) {
2592         for (my $i = 0; $i < 32; $i++) {
2593             my $name = "dimm${dimm_id}";
2594             $dimm_id--;
2595             my $numanode = $i % $sockets;
2596             $current_size -= $dimm_size;
2597             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2598             return  $current_size if $current_size <= $memory;
2599         }
2600         $dimm_size /= 2;
2601     }
2602 }
2603
2604 sub foreach_drive {
2605     my ($conf, $func) = @_;
2606
2607     foreach my $ds (valid_drive_names()) {
2608         next if !defined($conf->{$ds});
2609
2610         my $drive = parse_drive($ds, $conf->{$ds});
2611         next if !$drive;
2612
2613         &$func($ds, $drive);
2614     }
2615 }
2616
2617 sub foreach_volid {
2618     my ($conf, $func) = @_;
2619
2620     my $volhash = {};
2621
2622     my $test_volid = sub {
2623         my ($volid, $is_cdrom) = @_;
2624
2625         return if !$volid;
2626
2627         $volhash->{$volid} = $is_cdrom || 0;
2628     };
2629
2630     foreach_drive($conf, sub {
2631         my ($ds, $drive) = @_;
2632         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2633     });
2634
2635     foreach my $snapname (keys %{$conf->{snapshots}}) {
2636         my $snap = $conf->{snapshots}->{$snapname};
2637         &$test_volid($snap->{vmstate}, 0);
2638         foreach_drive($snap, sub {
2639             my ($ds, $drive) = @_;
2640             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2641         });
2642     }
2643
2644     foreach my $volid (keys %$volhash) {
2645         &$func($volid, $volhash->{$volid});
2646     }
2647 }
2648
2649 sub vga_conf_has_spice {
2650     my ($vga) = @_;
2651
2652     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2653
2654     return $1 || 1;
2655 }
2656
2657 sub config_to_command {
2658     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2659
2660     my $cmd = [];
2661     my $globalFlags = [];
2662     my $machineFlags = [];
2663     my $rtcFlags = [];
2664     my $cpuFlags = [];
2665     my $devices = [];
2666     my $pciaddr = '';
2667     my $bridges = {};
2668     my $kvmver = kvm_user_version();
2669     my $vernum = 0; # unknown
2670     my $ostype = $conf->{ostype};
2671     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2672         $vernum = $1*1000000+$2*1000;
2673     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2674         $vernum = $1*1000000+$2*1000+$3;
2675     }
2676
2677     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2678
2679     my $have_ovz = -f '/proc/vz/vestat';
2680
2681     my $q35 = machine_type_is_q35($conf);
2682     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2683     my $machine_type = $forcemachine || $conf->{machine};
2684     my $use_old_bios_files = undef;
2685     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2686
2687     my $cpuunits = defined($conf->{cpuunits}) ?
2688             $conf->{cpuunits} : $defaults->{cpuunits};
2689
2690     push @$cmd, '/usr/bin/systemd-run';
2691     push @$cmd, '--scope';
2692     push @$cmd, '--slice', "qemu";
2693     push @$cmd, '--unit', $vmid;
2694     # set KillMode=none, so that systemd don't kill those scopes
2695     # at shutdown (pve-manager service should stop the VMs instead)
2696     push @$cmd, '-p', "KillMode=none";
2697     push @$cmd, '-p', "CPUShares=$cpuunits";
2698     if ($conf->{cpulimit}) {
2699         my $cpulimit = int($conf->{cpulimit} * 100);
2700         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2701     }
2702
2703     push @$cmd, '/usr/bin/kvm';
2704
2705     push @$cmd, '-id', $vmid;
2706
2707     my $use_virtio = 0;
2708
2709     my $qmpsocket = qmp_socket($vmid);
2710     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2711     push @$cmd, '-mon', "chardev=qmp,mode=control";
2712
2713
2714     push @$cmd, '-pidfile' , pidfile_name($vmid);
2715
2716     push @$cmd, '-daemonize';
2717
2718     if ($conf->{smbios1}) {
2719         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2720     }
2721
2722     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2723         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2724         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2725         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2726         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2727         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2728         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2729     }
2730
2731     if ($q35) {
2732         # the q35 chipset support native usb2, so we enable usb controller
2733         # by default for this machine type
2734         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2735     } else {
2736         $pciaddr = print_pci_addr("piix3", $bridges);
2737         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2738
2739         my $use_usb2 = 0;
2740         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2741             next if !$conf->{"usb$i"};
2742             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2743             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2744             $use_usb2 = 1;
2745         }
2746         # include usb device config
2747         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2748     }
2749
2750     # add usb3 controller if needed
2751
2752     my $use_usb3 = 0;
2753     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2754         next if !$conf->{"usb$i"};
2755         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2756         next if !$d || !$d->{usb3};
2757         $use_usb3 = 1;
2758     }
2759
2760     $pciaddr = print_pci_addr("xhci", $bridges);
2761     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2762
2763     my $vga = $conf->{vga};
2764
2765     my $qxlnum = vga_conf_has_spice($vga);
2766     $vga = 'qxl' if $qxlnum;
2767
2768     if (!$vga) {
2769         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2770                                 $conf->{ostype} eq 'win7' ||
2771                                 $conf->{ostype} eq 'w2k8')) {
2772             $vga = 'std';
2773         } else {
2774             $vga = 'cirrus';
2775         }
2776     }
2777
2778     # enable absolute mouse coordinates (needed by vnc)
2779     my $tablet;
2780     if (defined($conf->{tablet})) {
2781         $tablet = $conf->{tablet};
2782     } else {
2783         $tablet = $defaults->{tablet};
2784         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2785         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2786     }
2787
2788     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2789
2790     my $kvm_off = 0;
2791     # host pci devices
2792     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2793         my $d = parse_hostpci($conf->{"hostpci$i"});
2794         next if !$d;
2795
2796         my $pcie = $d->{pcie};
2797         if($pcie){
2798             die "q35 machine model is not enabled" if !$q35;
2799             $pciaddr = print_pcie_addr("hostpci$i");
2800         }else{
2801             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2802         }
2803
2804         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2805         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2806         if ($xvga && $xvga ne '') {
2807             $kvm_off = 1;
2808             $vga = 'none';
2809             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2810                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2811             }
2812             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2813                 $xvga = "";
2814             }
2815         }
2816         my $pcidevices = $d->{pciid};
2817         my $multifunction = 1 if @$pcidevices > 1;
2818
2819         my $j=0;
2820         foreach my $pcidevice (@$pcidevices) {
2821
2822             my $id = "hostpci$i";
2823             $id .= ".$j" if $multifunction;
2824             my $addr = $pciaddr;
2825             $addr .= ".$j" if $multifunction;
2826             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2827
2828             if($j == 0){
2829                 $devicestr .= "$rombar$xvga";
2830                 $devicestr .= ",multifunction=on" if $multifunction;
2831             }
2832
2833             push @$devices, '-device', $devicestr;
2834             $j++;
2835         }
2836     }
2837
2838     # usb devices
2839     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2840         next if !$conf->{"usb$i"};
2841         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2842         next if !$d;
2843
2844         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2845         my $usbbus = '';
2846         if (defined($d->{usb3}) && $d->{usb3}) {
2847             $usbbus = ',bus=xhci.0';
2848         }
2849
2850         if (defined($d->{host})) {
2851             $d = parse_usb_device($d->{host});
2852             if (defined($d->{vendorid}) && defined($d->{productid})) {
2853                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2854             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2855                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2856             } elsif (defined($d->{spice}) && $d->{spice}) {
2857                 # usb redir support for spice, currently no usb3
2858                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2859                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2860             }
2861         }
2862     }
2863
2864     # serial devices
2865     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2866         if (my $path = $conf->{"serial$i"}) {
2867             if ($path eq 'socket') {
2868                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2869                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2870                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2871             } else {
2872                 die "no such serial device\n" if ! -c $path;
2873                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2874                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2875             }
2876         }
2877     }
2878
2879     # parallel devices
2880     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2881         if (my $path = $conf->{"parallel$i"}) {
2882             die "no such parallel device\n" if ! -c $path;
2883             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2884             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2885             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2886         }
2887     }
2888
2889     my $vmname = $conf->{name} || "vm$vmid";
2890
2891     push @$cmd, '-name', $vmname;
2892
2893     my $sockets = 1;
2894     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2895     $sockets = $conf->{sockets} if  $conf->{sockets};
2896
2897     my $cores = $conf->{cores} || 1;
2898
2899     my $maxcpus = $sockets * $cores;
2900
2901     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2902
2903     my $allowed_vcpus = $cpuinfo->{cpus};
2904
2905     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2906         if ($allowed_vcpus < $maxcpus);
2907
2908     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2909
2910     push @$cmd, '-nodefaults';
2911
2912     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2913
2914     my $bootindex_hash = {};
2915     my $i = 1;
2916     foreach my $o (split(//, $bootorder)) {
2917         $bootindex_hash->{$o} = $i*100;
2918         $i++;
2919     }
2920
2921     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2922
2923     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2924
2925     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2926
2927     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2928
2929     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2930         my $socket = vnc_socket($vmid);
2931         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2932     } else {
2933         push @$cmd, '-nographic';
2934     }
2935
2936     # time drift fix
2937     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2938
2939     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2940     my $useLocaltime = $conf->{localtime};
2941
2942     if ($ostype) {
2943         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2944
2945         if ($ostype =~ m/^w/) { # windows
2946             $useLocaltime = 1 if !defined($conf->{localtime});
2947
2948             # use time drift fix when acpi is enabled
2949             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2950                 $tdf = 1 if !defined($conf->{tdf});
2951             }
2952         }
2953
2954         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2955             $ostype eq 'wvista') {
2956             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2957             push @$cmd, '-no-hpet';
2958             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2959                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2960                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2961                 push @$cpuFlags , 'hv_time' if !$nokvm;
2962
2963             } else {
2964                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2965             }
2966         }
2967
2968         if ($ostype eq 'win7' || $ostype eq 'win8') {
2969             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2970         }
2971     }
2972
2973     push @$rtcFlags, 'driftfix=slew' if $tdf;
2974
2975     if ($nokvm) {
2976         push @$machineFlags, 'accel=tcg';
2977     } else {
2978         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2979     }
2980
2981     if ($machine_type) {
2982         push @$machineFlags, "type=${machine_type}";
2983     }
2984
2985     if ($conf->{startdate}) {
2986         push @$rtcFlags, "base=$conf->{startdate}";
2987     } elsif ($useLocaltime) {
2988         push @$rtcFlags, 'base=localtime';
2989     }
2990
2991     my $cpu = $nokvm ? "qemu64" : "kvm64";
2992     if (my $cputype = $conf->{cpu}) {
2993         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
2994             or die "Cannot parse cpu description: $cputype\n";
2995         $cpu = $cpuconf->{cputype};
2996         $kvm_off = 1 if $cpuconf->{hidden};
2997     }
2998
2999     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3000
3001     push @$cpuFlags , '-x2apic'
3002         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3003
3004     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3005
3006     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3007
3008     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3009
3010         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3011         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3012     }
3013
3014     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3015
3016     push @$cpuFlags, 'kvm=off' if $kvm_off;
3017
3018     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3019
3020     push @$cmd, '-cpu', $cpu;
3021
3022     my $memory = $conf->{memory} || $defaults->{memory};
3023     my $static_memory = 0;
3024     my $dimm_memory = 0;
3025
3026     if ($hotplug_features->{memory}) {
3027         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3028         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3029         $static_memory = $STATICMEM;
3030         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3031         $dimm_memory = $memory - $static_memory;
3032         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3033
3034     } else {
3035
3036         $static_memory = $memory;
3037         push @$cmd, '-m', $static_memory;
3038     }
3039
3040     if ($conf->{numa}) {
3041
3042         my $numa_totalmemory = undef;
3043         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3044             next if !$conf->{"numa$i"};
3045             my $numa = parse_numa($conf->{"numa$i"});
3046             next if !$numa;
3047             # memory
3048             die "missing numa node$i memory value\n" if !$numa->{memory};
3049             my $numa_memory = $numa->{memory};
3050             $numa_totalmemory += $numa_memory;
3051             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3052
3053             # cpus
3054             my $cpus_start = $numa->{cpus}->{start};
3055             die "missing numa node$i cpus\n" if !defined($cpus_start);
3056             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3057             my $cpus = $cpus_start;
3058             if (defined($cpus_end)) {
3059                 $cpus .= "-$cpus_end";
3060                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3061             }
3062
3063             # hostnodes
3064             my $hostnodes_start = $numa->{hostnodes}->{start};
3065             if (defined($hostnodes_start)) {
3066                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3067                 my $hostnodes = $hostnodes_start;
3068                 if (defined($hostnodes_end)) {
3069                     $hostnodes .= "-$hostnodes_end";
3070                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3071                 }
3072
3073                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3074                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3075                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3076                 }
3077
3078                 # policy
3079                 my $policy = $numa->{policy};
3080                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3081                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3082             }
3083
3084             push @$cmd, '-object', $numa_object;
3085             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3086         }
3087
3088         die "total memory for NUMA nodes must be equal to vm static memory\n"
3089             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3090
3091         #if no custom tology, we split memory and cores across numa nodes
3092         if(!$numa_totalmemory) {
3093
3094             my $numa_memory = ($static_memory / $sockets) . "M";
3095
3096             for (my $i = 0; $i < $sockets; $i++)  {
3097
3098                 my $cpustart = ($cores * $i);
3099                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3100                 my $cpus = $cpustart;
3101                 $cpus .= "-$cpuend" if $cpuend;
3102
3103                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3104                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3105             }
3106         }
3107     }
3108
3109     if ($hotplug_features->{memory}) {
3110         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3111             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3112             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3113             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3114
3115             #if dimm_memory is not aligned to dimm map
3116             if($current_size > $memory) {
3117                  $conf->{memory} = $current_size;
3118                  PVE::QemuConfig->write_config($vmid, $conf);
3119             }
3120         });
3121     }
3122
3123     push @$cmd, '-S' if $conf->{freeze};
3124
3125     # set keyboard layout
3126     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3127     push @$cmd, '-k', $kb if $kb;
3128
3129     # enable sound
3130     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3131     #push @$cmd, '-soundhw', 'es1370';
3132     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3133
3134     if($conf->{agent}) {
3135         my $qgasocket = qmp_socket($vmid, 1);
3136         my $pciaddr = print_pci_addr("qga0", $bridges);
3137         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3138         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3139         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3140     }
3141
3142     my $spice_port;
3143
3144     if ($qxlnum) {
3145         if ($qxlnum > 1) {
3146             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3147                 for(my $i = 1; $i < $qxlnum; $i++){
3148                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3149                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3150                 }
3151             } else {
3152                 # assume other OS works like Linux
3153                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3154                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3155             }
3156         }
3157
3158         my $pciaddr = print_pci_addr("spice", $bridges);
3159
3160         my $nodename = PVE::INotify::nodename();
3161         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3162         $spice_port = PVE::Tools::next_spice_port($pfamily);
3163
3164         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3165
3166         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3167         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3168         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3169     }
3170
3171     # enable balloon by default, unless explicitly disabled
3172     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3173         $pciaddr = print_pci_addr("balloon0", $bridges);
3174         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3175     }
3176
3177     if ($conf->{watchdog}) {
3178         my $wdopts = parse_watchdog($conf->{watchdog});
3179         $pciaddr = print_pci_addr("watchdog", $bridges);
3180         my $watchdog = $wdopts->{model} || 'i6300esb';
3181         push @$devices, '-device', "$watchdog$pciaddr";
3182         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3183     }
3184
3185     my $vollist = [];
3186     my $scsicontroller = {};
3187     my $ahcicontroller = {};
3188     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3189
3190     # Add iscsi initiator name if available
3191     if (my $initiator = get_initiator_name()) {
3192         push @$devices, '-iscsi', "initiator-name=$initiator";
3193     }
3194
3195     foreach_drive($conf, sub {
3196         my ($ds, $drive) = @_;
3197
3198         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3199             push @$vollist, $drive->{file};
3200         }
3201
3202         $use_virtio = 1 if $ds =~ m/^virtio/;
3203
3204         if (drive_is_cdrom ($drive)) {
3205             if ($bootindex_hash->{d}) {
3206                 $drive->{bootindex} = $bootindex_hash->{d};
3207                 $bootindex_hash->{d} += 1;
3208             }
3209         } else {
3210             if ($bootindex_hash->{c}) {
3211                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3212                 $bootindex_hash->{c} += 1;
3213             }
3214         }
3215
3216         if($drive->{interface} eq 'virtio'){
3217            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3218         }
3219
3220         if ($drive->{interface} eq 'scsi') {
3221
3222             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3223
3224             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3225             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3226
3227             my $iothread = '';
3228             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3229                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3230                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3231             }
3232
3233             my $queues = '';
3234             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3235                 $queues = ",num_queues=$drive->{queues}";
3236             }
3237
3238             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3239             $scsicontroller->{$controller}=1;
3240         }
3241
3242         if ($drive->{interface} eq 'sata') {
3243            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3244            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3245            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3246            $ahcicontroller->{$controller}=1;
3247         }
3248
3249         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3250         push @$devices, '-drive',$drive_cmd;
3251         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3252     });
3253
3254     for (my $i = 0; $i < $MAX_NETS; $i++) {
3255          next if !$conf->{"net$i"};
3256          my $d = parse_net($conf->{"net$i"});
3257          next if !$d;
3258
3259          $use_virtio = 1 if $d->{model} eq 'virtio';
3260
3261          if ($bootindex_hash->{n}) {
3262             $d->{bootindex} = $bootindex_hash->{n};
3263             $bootindex_hash->{n} += 1;
3264          }
3265
3266          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3267          push @$devices, '-netdev', $netdevfull;
3268
3269          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3270          push @$devices, '-device', $netdevicefull;
3271     }
3272
3273     if (!$q35) {
3274         # add pci bridges
3275         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3276            $bridges->{1} = 1;
3277            $bridges->{2} = 1;
3278         }
3279
3280         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3281
3282         while (my ($k, $v) = each %$bridges) {
3283             $pciaddr = print_pci_addr("pci.$k");
3284             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3285         }
3286     }
3287
3288     # add custom args
3289     if ($conf->{args}) {
3290         my $aa = PVE::Tools::split_args($conf->{args});
3291         push @$cmd, @$aa;
3292     }
3293
3294     push @$cmd, @$devices;
3295     push @$cmd, '-rtc', join(',', @$rtcFlags)
3296         if scalar(@$rtcFlags);
3297     push @$cmd, '-machine', join(',', @$machineFlags)
3298         if scalar(@$machineFlags);
3299     push @$cmd, '-global', join(',', @$globalFlags)
3300         if scalar(@$globalFlags);
3301
3302     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3303 }
3304
3305 sub vnc_socket {
3306     my ($vmid) = @_;
3307     return "${var_run_tmpdir}/$vmid.vnc";
3308 }
3309
3310 sub spice_port {
3311     my ($vmid) = @_;
3312
3313     my $res = vm_mon_cmd($vmid, 'query-spice');
3314
3315     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3316 }
3317
3318 sub qmp_socket {
3319     my ($vmid, $qga) = @_;
3320     my $sockettype = $qga ? 'qga' : 'qmp';
3321     return "${var_run_tmpdir}/$vmid.$sockettype";
3322 }
3323
3324 sub pidfile_name {
3325     my ($vmid) = @_;
3326     return "${var_run_tmpdir}/$vmid.pid";
3327 }
3328
3329 sub vm_devices_list {
3330     my ($vmid) = @_;
3331
3332     my $res = vm_mon_cmd($vmid, 'query-pci');
3333     my $devices = {};
3334     foreach my $pcibus (@$res) {
3335         foreach my $device (@{$pcibus->{devices}}) {
3336             next if !$device->{'qdev_id'};
3337             if ($device->{'pci_bridge'}) {
3338                 $devices->{$device->{'qdev_id'}} = 1;
3339                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3340                     next if !$bridge_device->{'qdev_id'};
3341                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3342                     $devices->{$device->{'qdev_id'}}++;
3343                 }
3344             } else {
3345                 $devices->{$device->{'qdev_id'}} = 1;
3346             }
3347         }
3348     }
3349
3350     my $resblock = vm_mon_cmd($vmid, 'query-block');
3351     foreach my $block (@$resblock) {
3352         if($block->{device} =~ m/^drive-(\S+)/){
3353                 $devices->{$1} = 1;
3354         }
3355     }
3356
3357     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3358     foreach my $mice (@$resmice) {
3359         if ($mice->{name} eq 'QEMU HID Tablet') {
3360             $devices->{tablet} = 1;
3361             last;
3362         }
3363     }
3364
3365     return $devices;
3366 }
3367
3368 sub vm_deviceplug {
3369     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3370
3371     my $q35 = machine_type_is_q35($conf);
3372
3373     my $devices_list = vm_devices_list($vmid);
3374     return 1 if defined($devices_list->{$deviceid});
3375
3376     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3377
3378     if ($deviceid eq 'tablet') {
3379
3380         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3381
3382     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3383
3384         qemu_iothread_add($vmid, $deviceid, $device);
3385
3386         qemu_driveadd($storecfg, $vmid, $device);
3387         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3388
3389         qemu_deviceadd($vmid, $devicefull);
3390         eval { qemu_deviceaddverify($vmid, $deviceid); };
3391         if (my $err = $@) {
3392             eval { qemu_drivedel($vmid, $deviceid); };
3393             warn $@ if $@;
3394             die $err;
3395         }
3396
3397     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3398
3399
3400         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3401         my $pciaddr = print_pci_addr($deviceid);
3402         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3403
3404         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3405
3406         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3407             qemu_iothread_add($vmid, $deviceid, $device);
3408             $devicefull .= ",iothread=iothread-$deviceid";
3409         }
3410
3411         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3412             $devicefull .= ",num_queues=$device->{queues}";
3413         }
3414
3415         qemu_deviceadd($vmid, $devicefull);
3416         qemu_deviceaddverify($vmid, $deviceid);
3417
3418     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3419
3420         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3421         qemu_driveadd($storecfg, $vmid, $device);
3422
3423         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3424         eval { qemu_deviceadd($vmid, $devicefull); };
3425         if (my $err = $@) {
3426             eval { qemu_drivedel($vmid, $deviceid); };
3427             warn $@ if $@;
3428             die $err;
3429         }
3430
3431     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3432
3433         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3434
3435         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3436         my $use_old_bios_files = undef;
3437         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3438
3439         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3440         qemu_deviceadd($vmid, $netdevicefull);
3441         eval { qemu_deviceaddverify($vmid, $deviceid); };
3442         if (my $err = $@) {
3443             eval { qemu_netdevdel($vmid, $deviceid); };
3444             warn $@ if $@;
3445             die $err;
3446         }
3447
3448     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3449
3450         my $bridgeid = $2;
3451         my $pciaddr = print_pci_addr($deviceid);
3452         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3453
3454         qemu_deviceadd($vmid, $devicefull);
3455         qemu_deviceaddverify($vmid, $deviceid);
3456
3457     } else {
3458         die "can't hotplug device '$deviceid'\n";
3459     }
3460
3461     return 1;
3462 }
3463
3464 # fixme: this should raise exceptions on error!
3465 sub vm_deviceunplug {
3466     my ($vmid, $conf, $deviceid) = @_;
3467
3468     my $devices_list = vm_devices_list($vmid);
3469     return 1 if !defined($devices_list->{$deviceid});
3470
3471     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3472
3473     if ($deviceid eq 'tablet') {
3474
3475         qemu_devicedel($vmid, $deviceid);
3476
3477     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3478
3479         qemu_devicedel($vmid, $deviceid);
3480         qemu_devicedelverify($vmid, $deviceid);
3481         qemu_drivedel($vmid, $deviceid);
3482         qemu_iothread_del($conf, $vmid, $deviceid);
3483
3484     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3485
3486         qemu_devicedel($vmid, $deviceid);
3487         qemu_devicedelverify($vmid, $deviceid);
3488         qemu_iothread_del($conf, $vmid, $deviceid);
3489
3490     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3491
3492         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3493         my $device = parse_drive($deviceid, $conf->{$deviceid});
3494         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3495
3496         qemu_devicedel($vmid, $deviceid);
3497         qemu_drivedel($vmid, $deviceid);
3498         qemu_deletescsihw($conf, $vmid, $deviceid);
3499
3500     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3501
3502         qemu_devicedel($vmid, $deviceid);
3503         qemu_devicedelverify($vmid, $deviceid);
3504         qemu_netdevdel($vmid, $deviceid);
3505
3506     } else {
3507         die "can't unplug device '$deviceid'\n";
3508     }
3509
3510     return 1;
3511 }
3512
3513 sub qemu_deviceadd {
3514     my ($vmid, $devicefull) = @_;
3515
3516     $devicefull = "driver=".$devicefull;
3517     my %options =  split(/[=,]/, $devicefull);
3518
3519     vm_mon_cmd($vmid, "device_add" , %options);
3520 }
3521
3522 sub qemu_devicedel {
3523     my ($vmid, $deviceid) = @_;
3524
3525     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3526 }
3527
3528 sub qemu_iothread_add {
3529     my($vmid, $deviceid, $device) = @_;
3530
3531     if ($device->{iothread}) {
3532         my $iothreads = vm_iothreads_list($vmid);
3533         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3534     }
3535 }
3536
3537 sub qemu_iothread_del {
3538     my($conf, $vmid, $deviceid) = @_;
3539
3540     my $device = parse_drive($deviceid, $conf->{$deviceid});
3541     if ($device->{iothread}) {
3542         my $iothreads = vm_iothreads_list($vmid);
3543         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3544     }
3545 }
3546
3547 sub qemu_objectadd {
3548     my($vmid, $objectid, $qomtype) = @_;
3549
3550     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3551
3552     return 1;
3553 }
3554
3555 sub qemu_objectdel {
3556     my($vmid, $objectid) = @_;
3557
3558     vm_mon_cmd($vmid, "object-del", id => $objectid);
3559
3560     return 1;
3561 }
3562
3563 sub qemu_driveadd {
3564     my ($storecfg, $vmid, $device) = @_;
3565
3566     my $drive = print_drive_full($storecfg, $vmid, $device);
3567     $drive =~ s/\\/\\\\/g;
3568     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3569
3570     # If the command succeeds qemu prints: "OK"
3571     return 1 if $ret =~ m/OK/s;
3572
3573     die "adding drive failed: $ret\n";
3574 }
3575
3576 sub qemu_drivedel {
3577     my($vmid, $deviceid) = @_;
3578
3579     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3580     $ret =~ s/^\s+//;
3581
3582     return 1 if $ret eq "";
3583
3584     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3585     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3586
3587     die "deleting drive $deviceid failed : $ret\n";
3588 }
3589
3590 sub qemu_deviceaddverify {
3591     my ($vmid, $deviceid) = @_;
3592
3593     for (my $i = 0; $i <= 5; $i++) {
3594          my $devices_list = vm_devices_list($vmid);
3595          return 1 if defined($devices_list->{$deviceid});
3596          sleep 1;
3597     }
3598
3599     die "error on hotplug device '$deviceid'\n";
3600 }
3601
3602
3603 sub qemu_devicedelverify {
3604     my ($vmid, $deviceid) = @_;
3605
3606     # need to verify that the device is correctly removed as device_del
3607     # is async and empty return is not reliable
3608
3609     for (my $i = 0; $i <= 5; $i++) {
3610          my $devices_list = vm_devices_list($vmid);
3611          return 1 if !defined($devices_list->{$deviceid});
3612          sleep 1;
3613     }
3614
3615     die "error on hot-unplugging device '$deviceid'\n";
3616 }
3617
3618 sub qemu_findorcreatescsihw {
3619     my ($storecfg, $conf, $vmid, $device) = @_;
3620
3621     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3622
3623     my $scsihwid="$controller_prefix$controller";
3624     my $devices_list = vm_devices_list($vmid);
3625
3626     if(!defined($devices_list->{$scsihwid})) {
3627         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3628     }
3629
3630     return 1;
3631 }
3632
3633 sub qemu_deletescsihw {
3634     my ($conf, $vmid, $opt) = @_;
3635
3636     my $device = parse_drive($opt, $conf->{$opt});
3637
3638     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3639         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3640         return 1;
3641     }
3642
3643     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3644
3645     my $devices_list = vm_devices_list($vmid);
3646     foreach my $opt (keys %{$devices_list}) {
3647         if (PVE::QemuServer::is_valid_drivename($opt)) {
3648             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3649             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3650                 return 1;
3651             }
3652         }
3653     }
3654
3655     my $scsihwid="scsihw$controller";
3656
3657     vm_deviceunplug($vmid, $conf, $scsihwid);
3658
3659     return 1;
3660 }
3661
3662 sub qemu_add_pci_bridge {
3663     my ($storecfg, $conf, $vmid, $device) = @_;
3664
3665     my $bridges = {};
3666
3667     my $bridgeid;
3668
3669     print_pci_addr($device, $bridges);
3670
3671     while (my ($k, $v) = each %$bridges) {
3672         $bridgeid = $k;
3673     }
3674     return 1 if !defined($bridgeid) || $bridgeid < 1;
3675
3676     my $bridge = "pci.$bridgeid";
3677     my $devices_list = vm_devices_list($vmid);
3678
3679     if (!defined($devices_list->{$bridge})) {
3680         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3681     }
3682
3683     return 1;
3684 }
3685
3686 sub qemu_set_link_status {
3687     my ($vmid, $device, $up) = @_;
3688
3689     vm_mon_cmd($vmid, "set_link", name => $device,
3690                up => $up ? JSON::true : JSON::false);
3691 }
3692
3693 sub qemu_netdevadd {
3694     my ($vmid, $conf, $device, $deviceid) = @_;
3695
3696     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3697     my %options =  split(/[=,]/, $netdev);
3698
3699     vm_mon_cmd($vmid, "netdev_add",  %options);
3700     return 1;
3701 }
3702
3703 sub qemu_netdevdel {
3704     my ($vmid, $deviceid) = @_;
3705
3706     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3707 }
3708
3709 sub qemu_cpu_hotplug {
3710     my ($vmid, $conf, $vcpus) = @_;
3711
3712     my $sockets = 1;
3713     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3714     $sockets = $conf->{sockets} if  $conf->{sockets};
3715     my $cores = $conf->{cores} || 1;
3716     my $maxcpus = $sockets * $cores;
3717
3718     $vcpus = $maxcpus if !$vcpus;
3719
3720     die "you can't add more vcpus than maxcpus\n"
3721         if $vcpus > $maxcpus;
3722
3723     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3724     die "online cpu unplug is not yet possible\n"
3725         if $vcpus < $currentvcpus;
3726
3727     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3728     die "vcpus in running vm is different than configuration\n"
3729         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3730
3731     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3732         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3733     }
3734 }
3735
3736 sub qemu_memory_hotplug {
3737     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3738
3739     return $value if !check_running($vmid);
3740
3741     my $memory = $conf->{memory} || $defaults->{memory};
3742     $value = $defaults->{memory} if !$value;
3743     return $value if $value == $memory;
3744
3745     my $static_memory = $STATICMEM;
3746     my $dimm_memory = $memory - $static_memory;
3747
3748     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3749     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3750
3751
3752     my $sockets = 1;
3753     $sockets = $conf->{sockets} if $conf->{sockets};
3754
3755     if($value > $memory) {
3756
3757         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3758             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3759
3760                 return if $current_size <= $conf->{memory};
3761
3762                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3763                 if (my $err = $@) {
3764                     eval { qemu_objectdel($vmid, "mem-$name"); };
3765                     die $err;
3766                 }
3767
3768                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3769                 if (my $err = $@) {
3770                     eval { qemu_objectdel($vmid, "mem-$name"); };
3771                     die $err;
3772                 }
3773                 #update conf after each succesful module hotplug
3774                 $conf->{memory} = $current_size;
3775                 PVE::QemuConfig->write_config($vmid, $conf);
3776         });
3777
3778     } else {
3779
3780         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3781             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3782
3783                 return if $current_size >= $conf->{memory};
3784                 print "try to unplug memory dimm $name\n";
3785
3786                 my $retry = 0;
3787                 while (1) {
3788                     eval { qemu_devicedel($vmid, $name) };
3789                     sleep 3;
3790                     my $dimm_list = qemu_dimm_list($vmid);
3791                     last if !$dimm_list->{$name};
3792                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3793                     $retry++;
3794                 }
3795
3796                 #update conf after each succesful module unplug
3797                 $conf->{memory} = $current_size;
3798
3799                 eval { qemu_objectdel($vmid, "mem-$name"); };
3800                 PVE::QemuConfig->write_config($vmid, $conf);
3801         });
3802     }
3803 }
3804
3805 sub qemu_dimm_list {
3806     my ($vmid) = @_;
3807
3808     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3809     my $dimms = {};
3810
3811     foreach my $dimm (@$dimmarray) {
3812
3813         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3814         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3815         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3816         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3817         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3818     }
3819     return $dimms;
3820 }
3821
3822 sub qemu_block_set_io_throttle {
3823     my ($vmid, $deviceid,
3824         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3825         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3826
3827     return if !check_running($vmid) ;
3828
3829     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3830         bps => int($bps),
3831         bps_rd => int($bps_rd),
3832         bps_wr => int($bps_wr),
3833         iops => int($iops),
3834         iops_rd => int($iops_rd),
3835         iops_wr => int($iops_wr),
3836         bps_max => int($bps_max),
3837         bps_rd_max => int($bps_rd_max),
3838         bps_wr_max => int($bps_wr_max),
3839         iops_max => int($iops_max),
3840         iops_rd_max => int($iops_rd_max),
3841         iops_wr_max => int($iops_wr_max)
3842     );
3843
3844 }
3845
3846 # old code, only used to shutdown old VM after update
3847 sub __read_avail {
3848     my ($fh, $timeout) = @_;
3849
3850     my $sel = new IO::Select;
3851     $sel->add($fh);
3852
3853     my $res = '';
3854     my $buf;
3855
3856     my @ready;
3857     while (scalar (@ready = $sel->can_read($timeout))) {
3858         my $count;
3859         if ($count = $fh->sysread($buf, 8192)) {
3860             if ($buf =~ /^(.*)\(qemu\) $/s) {
3861                 $res .= $1;
3862                 last;
3863             } else {
3864                 $res .= $buf;
3865             }
3866         } else {
3867             if (!defined($count)) {
3868                 die "$!\n";
3869             }
3870             last;
3871         }
3872     }
3873
3874     die "monitor read timeout\n" if !scalar(@ready);
3875
3876     return $res;
3877 }
3878
3879 # old code, only used to shutdown old VM after update
3880 sub vm_monitor_command {
3881     my ($vmid, $cmdstr, $nocheck) = @_;
3882
3883     my $res;
3884
3885     eval {
3886         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3887
3888         my $sname = "${var_run_tmpdir}/$vmid.mon";
3889
3890         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3891             die "unable to connect to VM $vmid socket - $!\n";
3892
3893         my $timeout = 3;
3894
3895         # hack: migrate sometime blocks the monitor (when migrate_downtime
3896         # is set)
3897         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3898             $timeout = 60*60; # 1 hour
3899         }
3900
3901         # read banner;
3902         my $data = __read_avail($sock, $timeout);
3903
3904         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3905             die "got unexpected qemu monitor banner\n";
3906         }
3907
3908         my $sel = new IO::Select;
3909         $sel->add($sock);
3910
3911         if (!scalar(my @ready = $sel->can_write($timeout))) {
3912             die "monitor write error - timeout";
3913         }
3914
3915         my $fullcmd = "$cmdstr\r";
3916
3917         # syslog('info', "VM $vmid monitor command: $cmdstr");
3918
3919         my $b;
3920         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3921             die "monitor write error - $!";
3922         }
3923
3924         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3925
3926         $timeout = 20;
3927
3928         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3929             $timeout = 60*60; # 1 hour
3930         } elsif ($cmdstr =~ m/^(eject|change)/) {
3931             $timeout = 60; # note: cdrom mount command is slow
3932         }
3933         if ($res = __read_avail($sock, $timeout)) {
3934
3935             my @lines = split("\r?\n", $res);
3936
3937             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3938
3939             $res = join("\n", @lines);
3940             $res .= "\n";
3941         }
3942     };
3943
3944     my $err = $@;
3945
3946     if ($err) {
3947         syslog("err", "VM $vmid monitor command failed - $err");
3948         die $err;
3949     }
3950
3951     return $res;
3952 }
3953
3954 sub qemu_block_resize {
3955     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3956
3957     my $running = check_running($vmid);
3958
3959     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3960
3961     return if !$running;
3962
3963     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3964
3965 }
3966
3967 sub qemu_volume_snapshot {
3968     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3969
3970     my $running = check_running($vmid);
3971
3972     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3973         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3974     } else {
3975         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3976     }
3977 }
3978
3979 sub qemu_volume_snapshot_delete {
3980     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3981
3982     my $running = check_running($vmid);
3983
3984     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3985
3986     return if !$running;
3987
3988     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3989 }
3990
3991 sub set_migration_caps {
3992     my ($vmid) = @_;
3993
3994     my $cap_ref = [];
3995
3996     my $enabled_cap = {
3997         "auto-converge" => 1,
3998         "xbzrle" => 1,
3999         "x-rdma-pin-all" => 0,
4000         "zero-blocks" => 0,
4001         "compress" => 0
4002     };
4003
4004     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4005
4006     for my $supported_capability (@$supported_capabilities) {
4007         push @$cap_ref, {
4008             capability => $supported_capability->{capability},
4009             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4010         };
4011     }
4012
4013     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4014 }
4015
4016 my $fast_plug_option = {
4017     'lock' => 1,
4018     'name' => 1,
4019     'onboot' => 1,
4020     'shares' => 1,
4021     'startup' => 1,
4022     'description' => 1,
4023 };
4024
4025 # hotplug changes in [PENDING]
4026 # $selection hash can be used to only apply specified options, for
4027 # example: { cores => 1 } (only apply changed 'cores')
4028 # $errors ref is used to return error messages
4029 sub vmconfig_hotplug_pending {
4030     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4031
4032     my $defaults = load_defaults();
4033
4034     # commit values which do not have any impact on running VM first
4035     # Note: those option cannot raise errors, we we do not care about
4036     # $selection and always apply them.
4037
4038     my $add_error = sub {
4039         my ($opt, $msg) = @_;
4040         $errors->{$opt} = "hotplug problem - $msg";
4041     };
4042
4043     my $changes = 0;
4044     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4045         if ($fast_plug_option->{$opt}) {
4046             $conf->{$opt} = $conf->{pending}->{$opt};
4047             delete $conf->{pending}->{$opt};
4048             $changes = 1;
4049         }
4050     }
4051
4052     if ($changes) {
4053         PVE::QemuConfig->write_config($vmid, $conf);
4054         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4055     }
4056
4057     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4058
4059     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4060     while (my ($opt, $force) = each %$pending_delete_hash) {
4061         next if $selection && !$selection->{$opt};
4062         eval {
4063             if ($opt eq 'hotplug') {
4064                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4065             } elsif ($opt eq 'tablet') {
4066                 die "skip\n" if !$hotplug_features->{usb};
4067                 if ($defaults->{tablet}) {
4068                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4069                 } else {
4070                     vm_deviceunplug($vmid, $conf, $opt);
4071                 }
4072             } elsif ($opt eq 'vcpus') {
4073                 die "skip\n" if !$hotplug_features->{cpu};
4074                 qemu_cpu_hotplug($vmid, $conf, undef);
4075             } elsif ($opt eq 'balloon') {
4076                 # enable balloon device is not hotpluggable
4077                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4078             } elsif ($fast_plug_option->{$opt}) {
4079                 # do nothing
4080             } elsif ($opt =~ m/^net(\d+)$/) {
4081                 die "skip\n" if !$hotplug_features->{network};
4082                 vm_deviceunplug($vmid, $conf, $opt);
4083             } elsif (is_valid_drivename($opt)) {
4084                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4085                 vm_deviceunplug($vmid, $conf, $opt);
4086                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4087             } elsif ($opt =~ m/^memory$/) {
4088                 die "skip\n" if !$hotplug_features->{memory};
4089                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4090             } elsif ($opt eq 'cpuunits') {
4091                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4092             } elsif ($opt eq 'cpulimit') {
4093                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4094             } else {
4095                 die "skip\n";
4096             }
4097         };
4098         if (my $err = $@) {
4099             &$add_error($opt, $err) if $err ne "skip\n";
4100         } else {
4101             # save new config if hotplug was successful
4102             delete $conf->{$opt};
4103             vmconfig_undelete_pending_option($conf, $opt);
4104             PVE::QemuConfig->write_config($vmid, $conf);
4105             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4106         }
4107     }
4108
4109     foreach my $opt (keys %{$conf->{pending}}) {
4110         next if $selection && !$selection->{$opt};
4111         my $value = $conf->{pending}->{$opt};
4112         eval {
4113             if ($opt eq 'hotplug') {
4114                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4115             } elsif ($opt eq 'tablet') {
4116                 die "skip\n" if !$hotplug_features->{usb};
4117                 if ($value == 1) {
4118                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4119                 } elsif ($value == 0) {
4120                     vm_deviceunplug($vmid, $conf, $opt);
4121                 }
4122             } elsif ($opt eq 'vcpus') {
4123                 die "skip\n" if !$hotplug_features->{cpu};
4124                 qemu_cpu_hotplug($vmid, $conf, $value);
4125             } elsif ($opt eq 'balloon') {
4126                 # enable/disable balloning device is not hotpluggable
4127                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4128                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4129                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4130
4131                 # allow manual ballooning if shares is set to zero
4132                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4133                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4134                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4135                 }
4136             } elsif ($opt =~ m/^net(\d+)$/) {
4137                 # some changes can be done without hotplug
4138                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4139                                     $vmid, $opt, $value);
4140             } elsif (is_valid_drivename($opt)) {
4141                 # some changes can be done without hotplug
4142                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4143                                      $vmid, $opt, $value, 1);
4144             } elsif ($opt =~ m/^memory$/) { #dimms
4145                 die "skip\n" if !$hotplug_features->{memory};
4146                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4147             } elsif ($opt eq 'cpuunits') {
4148                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4149             } elsif ($opt eq 'cpulimit') {
4150                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4151                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4152             } else {
4153                 die "skip\n";  # skip non-hot-pluggable options
4154             }
4155         };
4156         if (my $err = $@) {
4157             &$add_error($opt, $err) if $err ne "skip\n";
4158         } else {
4159             # save new config if hotplug was successful
4160             $conf->{$opt} = $value;
4161             delete $conf->{pending}->{$opt};
4162             PVE::QemuConfig->write_config($vmid, $conf);
4163             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4164         }
4165     }
4166 }
4167
4168 sub try_deallocate_drive {
4169     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4170
4171     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4172         my $volid = $drive->{file};
4173         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4174             my $sid = PVE::Storage::parse_volume_id($volid);
4175             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4176
4177             # check if the disk is really unused
4178             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4179                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4180             PVE::Storage::vdisk_free($storecfg, $volid);
4181             return 1;
4182         } else {
4183             # If vm is not owner of this disk remove from config
4184             return 1;
4185         }
4186     }
4187
4188     return undef;
4189 }
4190
4191 sub vmconfig_delete_or_detach_drive {
4192     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4193
4194     my $drive = parse_drive($opt, $conf->{$opt});
4195
4196     my $rpcenv = PVE::RPCEnvironment::get();
4197     my $authuser = $rpcenv->get_user();
4198
4199     if ($force) {
4200         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4201         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4202     } else {
4203         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4204     }
4205 }
4206
4207 sub vmconfig_apply_pending {
4208     my ($vmid, $conf, $storecfg) = @_;
4209
4210     # cold plug
4211
4212     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4213     while (my ($opt, $force) = each %$pending_delete_hash) {
4214         die "internal error" if $opt =~ m/^unused/;
4215         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4216         if (!defined($conf->{$opt})) {
4217             vmconfig_undelete_pending_option($conf, $opt);
4218             PVE::QemuConfig->write_config($vmid, $conf);
4219         } elsif (is_valid_drivename($opt)) {
4220             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4221             vmconfig_undelete_pending_option($conf, $opt);
4222             delete $conf->{$opt};
4223             PVE::QemuConfig->write_config($vmid, $conf);
4224         } else {
4225             vmconfig_undelete_pending_option($conf, $opt);
4226             delete $conf->{$opt};
4227             PVE::QemuConfig->write_config($vmid, $conf);
4228         }
4229     }
4230
4231     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4232
4233     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4234         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4235
4236         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4237             # skip if nothing changed
4238         } elsif (is_valid_drivename($opt)) {
4239             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4240                 if defined($conf->{$opt});
4241             $conf->{$opt} = $conf->{pending}->{$opt};
4242         } else {
4243             $conf->{$opt} = $conf->{pending}->{$opt};
4244         }
4245
4246         delete $conf->{pending}->{$opt};
4247         PVE::QemuConfig->write_config($vmid, $conf);
4248     }
4249 }
4250
4251 my $safe_num_ne = sub {
4252     my ($a, $b) = @_;
4253
4254     return 0 if !defined($a) && !defined($b);
4255     return 1 if !defined($a);
4256     return 1 if !defined($b);
4257
4258     return $a != $b;
4259 };
4260
4261 my $safe_string_ne = sub {
4262     my ($a, $b) = @_;
4263
4264     return 0 if !defined($a) && !defined($b);
4265     return 1 if !defined($a);
4266     return 1 if !defined($b);
4267
4268     return $a ne $b;
4269 };
4270
4271 sub vmconfig_update_net {
4272     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4273
4274     my $newnet = parse_net($value);
4275
4276     if ($conf->{$opt}) {
4277         my $oldnet = parse_net($conf->{$opt});
4278
4279         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4280             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4281             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4282             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4283
4284             # for non online change, we try to hot-unplug
4285             die "skip\n" if !$hotplug;
4286             vm_deviceunplug($vmid, $conf, $opt);
4287         } else {
4288
4289             die "internal error" if $opt !~ m/net(\d+)/;
4290             my $iface = "tap${vmid}i$1";
4291
4292             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4293                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4294                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4295                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4296                 PVE::Network::tap_unplug($iface);
4297                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4298             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4299                 # Rate can be applied on its own but any change above needs to
4300                 # include the rate in tap_plug since OVS resets everything.
4301                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4302             }
4303
4304             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4305                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4306             }
4307
4308             return 1;
4309         }
4310     }
4311
4312     if ($hotplug) {
4313         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4314     } else {
4315         die "skip\n";
4316     }
4317 }
4318
4319 sub vmconfig_update_disk {
4320     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4321
4322     # fixme: do we need force?
4323
4324     my $drive = parse_drive($opt, $value);
4325
4326     if ($conf->{$opt}) {
4327
4328         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4329
4330             my $media = $drive->{media} || 'disk';
4331             my $oldmedia = $old_drive->{media} || 'disk';
4332             die "unable to change media type\n" if $media ne $oldmedia;
4333
4334             if (!drive_is_cdrom($old_drive)) {
4335
4336                 if ($drive->{file} ne $old_drive->{file}) {
4337
4338                     die "skip\n" if !$hotplug;
4339
4340                     # unplug and register as unused
4341                     vm_deviceunplug($vmid, $conf, $opt);
4342                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4343
4344                 } else {
4345                     # update existing disk
4346
4347                     # skip non hotpluggable value
4348                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4349                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4350                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4351                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4352                         die "skip\n";
4353                     }
4354
4355                     # apply throttle
4356                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4357                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4358                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4359                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4360                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4361                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4362                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4363                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4364                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4365                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4366                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4367                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4368
4369                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4370                                                    ($drive->{mbps} || 0)*1024*1024,
4371                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4372                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4373                                                    $drive->{iops} || 0,
4374                                                    $drive->{iops_rd} || 0,
4375                                                    $drive->{iops_wr} || 0,
4376                                                    ($drive->{mbps_max} || 0)*1024*1024,
4377                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4378                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4379                                                    $drive->{iops_max} || 0,
4380                                                    $drive->{iops_rd_max} || 0,
4381                                                    $drive->{iops_wr_max} || 0);
4382
4383                     }
4384
4385                     return 1;
4386                 }
4387
4388             } else { # cdrom
4389
4390                 if ($drive->{file} eq 'none') {
4391                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4392                 } else {
4393                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4394                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4395                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4396                 }
4397
4398                 return 1;
4399             }
4400         }
4401     }
4402
4403     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4404     # hotplug new disks
4405     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4406     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4407 }
4408
4409 sub vm_start {
4410     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4411         $forcemachine, $spice_ticket) = @_;
4412
4413     PVE::QemuConfig->lock_config($vmid, sub {
4414         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4415
4416         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4417
4418         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4419
4420         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4421
4422         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4423             vmconfig_apply_pending($vmid, $conf, $storecfg);
4424             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4425         }
4426
4427         my $defaults = load_defaults();
4428
4429         # set environment variable useful inside network script
4430         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4431
4432         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4433
4434         my $migrate_port = 0;
4435         my $migrate_uri;
4436         if ($statefile) {
4437             if ($statefile eq 'tcp') {
4438                 my $localip = "localhost";
4439                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4440                 my $nodename = PVE::INotify::nodename();
4441                 if ($datacenterconf->{migration_unsecure}) {
4442                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4443                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4444                 }
4445                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4446                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4447                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4448                 push @$cmd, '-incoming', $migrate_uri;
4449                 push @$cmd, '-S';
4450             } else {
4451                 push @$cmd, '-loadstate', $statefile;
4452             }
4453         } elsif ($paused) {
4454             push @$cmd, '-S';
4455         }
4456
4457         # host pci devices
4458         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4459           my $d = parse_hostpci($conf->{"hostpci$i"});
4460           next if !$d;
4461           my $pcidevices = $d->{pciid};
4462           foreach my $pcidevice (@$pcidevices) {
4463                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4464
4465                 my $info = pci_device_info("0000:$pciid");
4466                 die "IOMMU not present\n" if !check_iommu_support();
4467                 die "no pci device info for device '$pciid'\n" if !$info;
4468                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4469                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4470           }
4471         }
4472
4473         PVE::Storage::activate_volumes($storecfg, $vollist);
4474
4475         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4476                     umask => 0077); };
4477
4478         if (my $err = $@) {
4479             # deactivate volumes if start fails
4480             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4481             die "start failed: $err";
4482         }
4483
4484         print "migration listens on $migrate_uri\n" if $migrate_uri;
4485
4486         if ($statefile && $statefile ne 'tcp')  {
4487             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4488             warn $@ if $@;
4489         }
4490
4491         if ($migratedfrom) {
4492
4493             eval {
4494                 set_migration_caps($vmid);
4495             };
4496             warn $@ if $@;
4497
4498             if ($spice_port) {
4499                 print "spice listens on port $spice_port\n";
4500                 if ($spice_ticket) {
4501                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4502                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4503                 }
4504             }
4505
4506         } else {
4507
4508             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4509                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4510                     if $conf->{balloon};
4511             }
4512
4513             foreach my $opt (keys %$conf) {
4514                 next if $opt !~  m/^net\d+$/;
4515                 my $nicconf = parse_net($conf->{$opt});
4516                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4517             }
4518         }
4519
4520         vm_mon_cmd_nocheck($vmid, 'qom-set',
4521                     path => "machine/peripheral/balloon0",
4522                     property => "guest-stats-polling-interval",
4523                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4524
4525     });
4526 }
4527
4528 sub vm_mon_cmd {
4529     my ($vmid, $execute, %params) = @_;
4530
4531     my $cmd = { execute => $execute, arguments => \%params };
4532     vm_qmp_command($vmid, $cmd);
4533 }
4534
4535 sub vm_mon_cmd_nocheck {
4536     my ($vmid, $execute, %params) = @_;
4537
4538     my $cmd = { execute => $execute, arguments => \%params };
4539     vm_qmp_command($vmid, $cmd, 1);
4540 }
4541
4542 sub vm_qmp_command {
4543     my ($vmid, $cmd, $nocheck) = @_;
4544
4545     my $res;
4546
4547     my $timeout;
4548     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4549         $timeout = $cmd->{arguments}->{timeout};
4550         delete $cmd->{arguments}->{timeout};
4551     }
4552
4553     eval {
4554         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4555         my $sname = qmp_socket($vmid);
4556         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4557             my $qmpclient = PVE::QMPClient->new();
4558
4559             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4560         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4561             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4562                 if scalar(%{$cmd->{arguments}});
4563             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4564         } else {
4565             die "unable to open monitor socket\n";
4566         }
4567     };
4568     if (my $err = $@) {
4569         syslog("err", "VM $vmid qmp command failed - $err");
4570         die $err;
4571     }
4572
4573     return $res;
4574 }
4575
4576 sub vm_human_monitor_command {
4577     my ($vmid, $cmdline) = @_;
4578
4579     my $res;
4580
4581     my $cmd = {
4582         execute => 'human-monitor-command',
4583         arguments => { 'command-line' => $cmdline},
4584     };
4585
4586     return vm_qmp_command($vmid, $cmd);
4587 }
4588
4589 sub vm_commandline {
4590     my ($storecfg, $vmid) = @_;
4591
4592     my $conf = PVE::QemuConfig->load_config($vmid);
4593
4594     my $defaults = load_defaults();
4595
4596     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4597
4598     return join(' ', @$cmd);
4599 }
4600
4601 sub vm_reset {
4602     my ($vmid, $skiplock) = @_;
4603
4604     PVE::QemuConfig->lock_config($vmid, sub {
4605
4606         my $conf = PVE::QemuConfig->load_config($vmid);
4607
4608         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4609
4610         vm_mon_cmd($vmid, "system_reset");
4611     });
4612 }
4613
4614 sub get_vm_volumes {
4615     my ($conf) = @_;
4616
4617     my $vollist = [];
4618     foreach_volid($conf, sub {
4619         my ($volid, $is_cdrom) = @_;
4620
4621         return if $volid =~ m|^/|;
4622
4623         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4624         return if !$sid;
4625
4626         push @$vollist, $volid;
4627     });
4628
4629     return $vollist;
4630 }
4631
4632 sub vm_stop_cleanup {
4633     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4634
4635     eval {
4636
4637         if (!$keepActive) {
4638             my $vollist = get_vm_volumes($conf);
4639             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4640         }
4641
4642         foreach my $ext (qw(mon qmp pid vnc qga)) {
4643             unlink "/var/run/qemu-server/${vmid}.$ext";
4644         }
4645
4646         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4647     };
4648     warn $@ if $@; # avoid errors - just warn
4649 }
4650
4651 # Note: use $nockeck to skip tests if VM configuration file exists.
4652 # We need that when migration VMs to other nodes (files already moved)
4653 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4654 sub vm_stop {
4655     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4656
4657     $force = 1 if !defined($force) && !$shutdown;
4658
4659     if ($migratedfrom){
4660         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4661         kill 15, $pid if $pid;
4662         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4663         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4664         return;
4665     }
4666
4667     PVE::QemuConfig->lock_config($vmid, sub {
4668
4669         my $pid = check_running($vmid, $nocheck);
4670         return if !$pid;
4671
4672         my $conf;
4673         if (!$nocheck) {
4674             $conf = PVE::QemuConfig->load_config($vmid);
4675             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4676             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4677                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4678                 $timeout = $opts->{down} if $opts->{down};
4679             }
4680         }
4681
4682         $timeout = 60 if !defined($timeout);
4683
4684         eval {
4685             if ($shutdown) {
4686                 if (defined($conf) && $conf->{agent}) {
4687                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4688                 } else {
4689                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4690                 }
4691             } else {
4692                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4693             }
4694         };
4695         my $err = $@;
4696
4697         if (!$err) {
4698             my $count = 0;
4699             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4700                 $count++;
4701                 sleep 1;
4702             }
4703
4704             if ($count >= $timeout) {
4705                 if ($force) {
4706                     warn "VM still running - terminating now with SIGTERM\n";
4707                     kill 15, $pid;
4708                 } else {
4709                     die "VM quit/powerdown failed - got timeout\n";
4710                 }
4711             } else {
4712                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4713                 return;
4714             }
4715         } else {
4716             if ($force) {
4717                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4718                 kill 15, $pid;
4719             } else {
4720                 die "VM quit/powerdown failed\n";
4721             }
4722         }
4723
4724         # wait again
4725         $timeout = 10;
4726
4727         my $count = 0;
4728         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4729             $count++;
4730             sleep 1;
4731         }
4732
4733         if ($count >= $timeout) {
4734             warn "VM still running - terminating now with SIGKILL\n";
4735             kill 9, $pid;
4736             sleep 1;
4737         }
4738
4739         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4740    });
4741 }
4742
4743 sub vm_suspend {
4744     my ($vmid, $skiplock) = @_;
4745
4746     PVE::QemuConfig->lock_config($vmid, sub {
4747
4748         my $conf = PVE::QemuConfig->load_config($vmid);
4749
4750         PVE::QemuConfig->check_lock($conf)
4751             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4752
4753         vm_mon_cmd($vmid, "stop");
4754     });
4755 }
4756
4757 sub vm_resume {
4758     my ($vmid, $skiplock, $nocheck) = @_;
4759
4760     PVE::QemuConfig->lock_config($vmid, sub {
4761
4762         if (!$nocheck) {
4763
4764             my $conf = PVE::QemuConfig->load_config($vmid);
4765
4766             PVE::QemuConfig->check_lock($conf)
4767                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4768
4769             vm_mon_cmd($vmid, "cont");
4770
4771         } else {
4772             vm_mon_cmd_nocheck($vmid, "cont");
4773         }
4774     });
4775 }
4776
4777 sub vm_sendkey {
4778     my ($vmid, $skiplock, $key) = @_;
4779
4780     PVE::QemuConfig->lock_config($vmid, sub {
4781
4782         my $conf = PVE::QemuConfig->load_config($vmid);
4783
4784         # there is no qmp command, so we use the human monitor command
4785         vm_human_monitor_command($vmid, "sendkey $key");
4786     });
4787 }
4788
4789 sub vm_destroy {
4790     my ($storecfg, $vmid, $skiplock) = @_;
4791
4792     PVE::QemuConfig->lock_config($vmid, sub {
4793
4794         my $conf = PVE::QemuConfig->load_config($vmid);
4795
4796         if (!check_running($vmid)) {
4797             destroy_vm($storecfg, $vmid, undef, $skiplock);
4798         } else {
4799             die "VM $vmid is running - destroy failed\n";
4800         }
4801     });
4802 }
4803
4804 # pci helpers
4805
4806 sub file_write {
4807     my ($filename, $buf) = @_;
4808
4809     my $fh = IO::File->new($filename, "w");
4810     return undef if !$fh;
4811
4812     my $res = print $fh $buf;
4813
4814     $fh->close();
4815
4816     return $res;
4817 }
4818
4819 sub pci_device_info {
4820     my ($name) = @_;
4821
4822     my $res;
4823
4824     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4825     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4826
4827     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4828     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4829
4830     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4831     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4832
4833     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4834     return undef if !defined($product) || $product !~ s/^0x//;
4835
4836     $res = {
4837         name => $name,
4838         vendor => $vendor,
4839         product => $product,
4840         domain => $domain,
4841         bus => $bus,
4842         slot => $slot,
4843         func => $func,
4844         irq => $irq,
4845         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4846     };
4847
4848     return $res;
4849 }
4850
4851 sub pci_dev_reset {
4852     my ($dev) = @_;
4853
4854     my $name = $dev->{name};
4855
4856     my $fn = "$pcisysfs/devices/$name/reset";
4857
4858     return file_write($fn, "1");
4859 }
4860
4861 sub pci_dev_bind_to_vfio {
4862     my ($dev) = @_;
4863
4864     my $name = $dev->{name};
4865
4866     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4867
4868     if (!-d $vfio_basedir) {
4869         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4870     }
4871     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4872
4873     my $testdir = "$vfio_basedir/$name";
4874     return 1 if -d $testdir;
4875
4876     my $data = "$dev->{vendor} $dev->{product}";
4877     return undef if !file_write("$vfio_basedir/new_id", $data);
4878
4879     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4880     if (!file_write($fn, $name)) {
4881         return undef if -f $fn;
4882     }
4883
4884     $fn = "$vfio_basedir/bind";
4885     if (! -d $testdir) {
4886         return undef if !file_write($fn, $name);
4887     }
4888
4889     return -d $testdir;
4890 }
4891
4892 sub pci_dev_group_bind_to_vfio {
4893     my ($pciid) = @_;
4894
4895     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4896
4897     if (!-d $vfio_basedir) {
4898         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4899     }
4900     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4901
4902     # get IOMMU group devices
4903     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4904       my @devs = grep /^0000:/, readdir($D);
4905     closedir($D);
4906
4907     foreach my $pciid (@devs) {
4908         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4909
4910         # pci bridges, switches or root ports are not supported
4911         # they have a pci_bus subdirectory so skip them
4912         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4913
4914         my $info = pci_device_info($1);
4915         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4916     }
4917
4918     return 1;
4919 }
4920
4921 sub print_pci_addr {
4922     my ($id, $bridges) = @_;
4923
4924     my $res = '';
4925     my $devices = {
4926         piix3 => { bus => 0, addr => 1 },
4927         #addr2 : first videocard
4928         balloon0 => { bus => 0, addr => 3 },
4929         watchdog => { bus => 0, addr => 4 },
4930         scsihw0 => { bus => 0, addr => 5 },
4931         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4932         scsihw1 => { bus => 0, addr => 6 },
4933         ahci0 => { bus => 0, addr => 7 },
4934         qga0 => { bus => 0, addr => 8 },
4935         spice => { bus => 0, addr => 9 },
4936         virtio0 => { bus => 0, addr => 10 },
4937         virtio1 => { bus => 0, addr => 11 },
4938         virtio2 => { bus => 0, addr => 12 },
4939         virtio3 => { bus => 0, addr => 13 },
4940         virtio4 => { bus => 0, addr => 14 },
4941         virtio5 => { bus => 0, addr => 15 },
4942         hostpci0 => { bus => 0, addr => 16 },
4943         hostpci1 => { bus => 0, addr => 17 },
4944         net0 => { bus => 0, addr => 18 },
4945         net1 => { bus => 0, addr => 19 },
4946         net2 => { bus => 0, addr => 20 },
4947         net3 => { bus => 0, addr => 21 },
4948         net4 => { bus => 0, addr => 22 },
4949         net5 => { bus => 0, addr => 23 },
4950         vga1 => { bus => 0, addr => 24 },
4951         vga2 => { bus => 0, addr => 25 },
4952         vga3 => { bus => 0, addr => 26 },
4953         hostpci2 => { bus => 0, addr => 27 },
4954         hostpci3 => { bus => 0, addr => 28 },
4955         #addr29 : usb-host (pve-usb.cfg)
4956         'pci.1' => { bus => 0, addr => 30 },
4957         'pci.2' => { bus => 0, addr => 31 },
4958         'net6' => { bus => 1, addr => 1 },
4959         'net7' => { bus => 1, addr => 2 },
4960         'net8' => { bus => 1, addr => 3 },
4961         'net9' => { bus => 1, addr => 4 },
4962         'net10' => { bus => 1, addr => 5 },
4963         'net11' => { bus => 1, addr => 6 },
4964         'net12' => { bus => 1, addr => 7 },
4965         'net13' => { bus => 1, addr => 8 },
4966         'net14' => { bus => 1, addr => 9 },
4967         'net15' => { bus => 1, addr => 10 },
4968         'net16' => { bus => 1, addr => 11 },
4969         'net17' => { bus => 1, addr => 12 },
4970         'net18' => { bus => 1, addr => 13 },
4971         'net19' => { bus => 1, addr => 14 },
4972         'net20' => { bus => 1, addr => 15 },
4973         'net21' => { bus => 1, addr => 16 },
4974         'net22' => { bus => 1, addr => 17 },
4975         'net23' => { bus => 1, addr => 18 },
4976         'net24' => { bus => 1, addr => 19 },
4977         'net25' => { bus => 1, addr => 20 },
4978         'net26' => { bus => 1, addr => 21 },
4979         'net27' => { bus => 1, addr => 22 },
4980         'net28' => { bus => 1, addr => 23 },
4981         'net29' => { bus => 1, addr => 24 },
4982         'net30' => { bus => 1, addr => 25 },
4983         'net31' => { bus => 1, addr => 26 },
4984         'xhci' => { bus => 1, addr => 27 },
4985         'virtio6' => { bus => 2, addr => 1 },
4986         'virtio7' => { bus => 2, addr => 2 },
4987         'virtio8' => { bus => 2, addr => 3 },
4988         'virtio9' => { bus => 2, addr => 4 },
4989         'virtio10' => { bus => 2, addr => 5 },
4990         'virtio11' => { bus => 2, addr => 6 },
4991         'virtio12' => { bus => 2, addr => 7 },
4992         'virtio13' => { bus => 2, addr => 8 },
4993         'virtio14' => { bus => 2, addr => 9 },
4994         'virtio15' => { bus => 2, addr => 10 },
4995         'virtioscsi0' => { bus => 3, addr => 1 },
4996         'virtioscsi1' => { bus => 3, addr => 2 },
4997         'virtioscsi2' => { bus => 3, addr => 3 },
4998         'virtioscsi3' => { bus => 3, addr => 4 },
4999         'virtioscsi4' => { bus => 3, addr => 5 },
5000         'virtioscsi5' => { bus => 3, addr => 6 },
5001         'virtioscsi6' => { bus => 3, addr => 7 },
5002         'virtioscsi7' => { bus => 3, addr => 8 },
5003         'virtioscsi8' => { bus => 3, addr => 9 },
5004         'virtioscsi9' => { bus => 3, addr => 10 },
5005         'virtioscsi10' => { bus => 3, addr => 11 },
5006         'virtioscsi11' => { bus => 3, addr => 12 },
5007         'virtioscsi12' => { bus => 3, addr => 13 },
5008         'virtioscsi13' => { bus => 3, addr => 14 },
5009         'virtioscsi14' => { bus => 3, addr => 15 },
5010         'virtioscsi15' => { bus => 3, addr => 16 },
5011         'virtioscsi16' => { bus => 3, addr => 17 },
5012         'virtioscsi17' => { bus => 3, addr => 18 },
5013         'virtioscsi18' => { bus => 3, addr => 19 },
5014         'virtioscsi19' => { bus => 3, addr => 20 },
5015         'virtioscsi20' => { bus => 3, addr => 21 },
5016         'virtioscsi21' => { bus => 3, addr => 22 },
5017         'virtioscsi22' => { bus => 3, addr => 23 },
5018         'virtioscsi23' => { bus => 3, addr => 24 },
5019         'virtioscsi24' => { bus => 3, addr => 25 },
5020         'virtioscsi25' => { bus => 3, addr => 26 },
5021         'virtioscsi26' => { bus => 3, addr => 27 },
5022         'virtioscsi27' => { bus => 3, addr => 28 },
5023         'virtioscsi28' => { bus => 3, addr => 29 },
5024         'virtioscsi29' => { bus => 3, addr => 30 },
5025         'virtioscsi30' => { bus => 3, addr => 31 },
5026
5027     };
5028
5029     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5030            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5031            my $bus = $devices->{$id}->{bus};
5032            $res = ",bus=pci.$bus,addr=$addr";
5033            $bridges->{$bus} = 1 if $bridges;
5034     }
5035     return $res;
5036
5037 }
5038
5039 sub print_pcie_addr {
5040     my ($id) = @_;
5041
5042     my $res = '';
5043     my $devices = {
5044         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5045         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5046         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5047         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5048     };
5049
5050     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5051            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5052            my $bus = $devices->{$id}->{bus};
5053            $res = ",bus=$bus,addr=$addr";
5054     }
5055     return $res;
5056
5057 }
5058
5059 # vzdump restore implementaion
5060
5061 sub tar_archive_read_firstfile {
5062     my $archive = shift;
5063
5064     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5065
5066     # try to detect archive type first
5067     my $pid = open (TMP, "tar tf '$archive'|") ||
5068         die "unable to open file '$archive'\n";
5069     my $firstfile = <TMP>;
5070     kill 15, $pid;
5071     close TMP;
5072
5073     die "ERROR: archive contaions no data\n" if !$firstfile;
5074     chomp $firstfile;
5075
5076     return $firstfile;
5077 }
5078
5079 sub tar_restore_cleanup {
5080     my ($storecfg, $statfile) = @_;
5081
5082     print STDERR "starting cleanup\n";
5083
5084     if (my $fd = IO::File->new($statfile, "r")) {
5085         while (defined(my $line = <$fd>)) {
5086             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5087                 my $volid = $2;
5088                 eval {
5089                     if ($volid =~ m|^/|) {
5090                         unlink $volid || die 'unlink failed\n';
5091                     } else {
5092                         PVE::Storage::vdisk_free($storecfg, $volid);
5093                     }
5094                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5095                 };
5096                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5097             } else {
5098                 print STDERR "unable to parse line in statfile - $line";
5099             }
5100         }
5101         $fd->close();
5102     }
5103 }
5104
5105 sub restore_archive {
5106     my ($archive, $vmid, $user, $opts) = @_;
5107
5108     my $format = $opts->{format};
5109     my $comp;
5110
5111     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5112         $format = 'tar' if !$format;
5113         $comp = 'gzip';
5114     } elsif ($archive =~ m/\.tar$/) {
5115         $format = 'tar' if !$format;
5116     } elsif ($archive =~ m/.tar.lzo$/) {
5117         $format = 'tar' if !$format;
5118         $comp = 'lzop';
5119     } elsif ($archive =~ m/\.vma$/) {
5120         $format = 'vma' if !$format;
5121     } elsif ($archive =~ m/\.vma\.gz$/) {
5122         $format = 'vma' if !$format;
5123         $comp = 'gzip';
5124     } elsif ($archive =~ m/\.vma\.lzo$/) {
5125         $format = 'vma' if !$format;
5126         $comp = 'lzop';
5127     } else {
5128         $format = 'vma' if !$format; # default
5129     }
5130
5131     # try to detect archive format
5132     if ($format eq 'tar') {
5133         return restore_tar_archive($archive, $vmid, $user, $opts);
5134     } else {
5135         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5136     }
5137 }
5138
5139 sub restore_update_config_line {
5140     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5141
5142     return if $line =~ m/^\#qmdump\#/;
5143     return if $line =~ m/^\#vzdump\#/;
5144     return if $line =~ m/^lock:/;
5145     return if $line =~ m/^unused\d+:/;
5146     return if $line =~ m/^parent:/;
5147     return if $line =~ m/^template:/; # restored VM is never a template
5148
5149     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5150         # try to convert old 1.X settings
5151         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5152         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5153             my ($model, $macaddr) = split(/\=/, $devconfig);
5154             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5155             my $net = {
5156                 model => $model,
5157                 bridge => "vmbr$ind",
5158                 macaddr => $macaddr,
5159             };
5160             my $netstr = print_net($net);
5161
5162             print $outfd "net$cookie->{netcount}: $netstr\n";
5163             $cookie->{netcount}++;
5164         }
5165     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5166         my ($id, $netstr) = ($1, $2);
5167         my $net = parse_net($netstr);
5168         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5169         $netstr = print_net($net);
5170         print $outfd "$id: $netstr\n";
5171     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5172         my $virtdev = $1;
5173         my $value = $3;
5174         my $di = parse_drive($virtdev, $value);
5175         if (defined($di->{backup}) && !$di->{backup}) {
5176             print $outfd "#$line";
5177         } elsif ($map->{$virtdev}) {
5178             delete $di->{format}; # format can change on restore
5179             $di->{file} = $map->{$virtdev};
5180             $value = print_drive($vmid, $di);
5181             print $outfd "$virtdev: $value\n";
5182         } else {
5183             print $outfd $line;
5184         }
5185     } else {
5186         print $outfd $line;
5187     }
5188 }
5189
5190 sub scan_volids {
5191     my ($cfg, $vmid) = @_;
5192
5193     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5194
5195     my $volid_hash = {};
5196     foreach my $storeid (keys %$info) {
5197         foreach my $item (@{$info->{$storeid}}) {
5198             next if !($item->{volid} && $item->{size});
5199             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5200             $volid_hash->{$item->{volid}} = $item;
5201         }
5202     }
5203
5204     return $volid_hash;
5205 }
5206
5207 sub is_volume_in_use {
5208     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5209
5210     my $path = PVE::Storage::path($storecfg, $volid);
5211
5212     my $scan_config = sub {
5213         my ($cref, $snapname) = @_;
5214
5215         foreach my $key (keys %$cref) {
5216             my $value = $cref->{$key};
5217             if (is_valid_drivename($key)) {
5218                 next if $skip_drive && $key eq $skip_drive;
5219                 my $drive = parse_drive($key, $value);
5220                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5221                 return 1 if $volid eq $drive->{file};
5222                 if ($drive->{file} =~ m!^/!) {
5223                     return 1 if $drive->{file} eq $path;
5224                 } else {
5225                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5226                     next if !$storeid;
5227                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5228                     next if !$scfg;
5229                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5230                 }
5231             }
5232         }
5233
5234         return 0;
5235     };
5236
5237     return 1 if &$scan_config($conf);
5238
5239     undef $skip_drive;
5240
5241     foreach my $snapname (keys %{$conf->{snapshots}}) {
5242         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5243     }
5244
5245     return 0;
5246 }
5247
5248 sub update_disksize {
5249     my ($vmid, $conf, $volid_hash) = @_;
5250
5251     my $changes;
5252
5253     my $used = {};
5254
5255     # Note: it is allowed to define multiple storages with same path (alias), so
5256     # we need to check both 'volid' and real 'path' (two different volid can point
5257     # to the same path).
5258
5259     my $usedpath = {};
5260
5261     # update size info
5262     foreach my $opt (keys %$conf) {
5263         if (is_valid_drivename($opt)) {
5264             my $drive = parse_drive($opt, $conf->{$opt});
5265             my $volid = $drive->{file};
5266             next if !$volid;
5267
5268             $used->{$volid} = 1;
5269             if ($volid_hash->{$volid} &&
5270                 (my $path = $volid_hash->{$volid}->{path})) {
5271                 $usedpath->{$path} = 1;
5272             }
5273
5274             next if drive_is_cdrom($drive);
5275             next if !$volid_hash->{$volid};
5276
5277             $drive->{size} = $volid_hash->{$volid}->{size};
5278             my $new = print_drive($vmid, $drive);
5279             if ($new ne $conf->{$opt}) {
5280                 $changes = 1;
5281                 $conf->{$opt} = $new;
5282             }
5283         }
5284     }
5285
5286     # remove 'unusedX' entry if volume is used
5287     foreach my $opt (keys %$conf) {
5288         next if $opt !~ m/^unused\d+$/;
5289         my $volid = $conf->{$opt};
5290         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5291         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5292             $changes = 1;
5293             delete $conf->{$opt};
5294         }
5295     }
5296
5297     foreach my $volid (sort keys %$volid_hash) {
5298         next if $volid =~ m/vm-$vmid-state-/;
5299         next if $used->{$volid};
5300         my $path = $volid_hash->{$volid}->{path};
5301         next if !$path; # just to be sure
5302         next if $usedpath->{$path};
5303         $changes = 1;
5304         PVE::QemuConfig->add_unused_volume($conf, $volid);
5305         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5306     }
5307
5308     return $changes;
5309 }
5310
5311 sub rescan {
5312     my ($vmid, $nolock) = @_;
5313
5314     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5315
5316     my $volid_hash = scan_volids($cfg, $vmid);
5317
5318     my $updatefn =  sub {
5319         my ($vmid) = @_;
5320
5321         my $conf = PVE::QemuConfig->load_config($vmid);
5322
5323         PVE::QemuConfig->check_lock($conf);
5324
5325         my $vm_volids = {};
5326         foreach my $volid (keys %$volid_hash) {
5327             my $info = $volid_hash->{$volid};
5328             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5329         }
5330
5331         my $changes = update_disksize($vmid, $conf, $vm_volids);
5332
5333         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5334     };
5335
5336     if (defined($vmid)) {
5337         if ($nolock) {
5338             &$updatefn($vmid);
5339         } else {
5340             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5341         }
5342     } else {
5343         my $vmlist = config_list();
5344         foreach my $vmid (keys %$vmlist) {
5345             if ($nolock) {
5346                 &$updatefn($vmid);
5347             } else {
5348                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5349             }
5350         }
5351     }
5352 }
5353
5354 sub restore_vma_archive {
5355     my ($archive, $vmid, $user, $opts, $comp) = @_;
5356
5357     my $input = $archive eq '-' ? "<&STDIN" : undef;
5358     my $readfrom = $archive;
5359
5360     my $uncomp = '';
5361     if ($comp) {
5362         $readfrom = '-';
5363         my $qarchive = PVE::Tools::shellquote($archive);
5364         if ($comp eq 'gzip') {
5365             $uncomp = "zcat $qarchive|";
5366         } elsif ($comp eq 'lzop') {
5367             $uncomp = "lzop -d -c $qarchive|";
5368         } else {
5369             die "unknown compression method '$comp'\n";
5370         }
5371
5372     }
5373
5374     my $tmpdir = "/var/tmp/vzdumptmp$$";
5375     rmtree $tmpdir;
5376
5377     # disable interrupts (always do cleanups)
5378     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5379         warn "got interrupt - ignored\n";
5380     };
5381
5382     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5383     POSIX::mkfifo($mapfifo, 0600);
5384     my $fifofh;
5385
5386     my $openfifo = sub {
5387         open($fifofh, '>', $mapfifo) || die $!;
5388     };
5389
5390     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5391
5392     my $oldtimeout;
5393     my $timeout = 5;
5394
5395     my $devinfo = {};
5396
5397     my $rpcenv = PVE::RPCEnvironment::get();
5398
5399     my $conffile = PVE::QemuConfig->config_file($vmid);
5400     my $tmpfn = "$conffile.$$.tmp";
5401
5402     # Note: $oldconf is undef if VM does not exists
5403     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5404     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5405
5406     my $print_devmap = sub {
5407         my $virtdev_hash = {};
5408
5409         my $cfgfn = "$tmpdir/qemu-server.conf";
5410
5411         # we can read the config - that is already extracted
5412         my $fh = IO::File->new($cfgfn, "r") ||
5413             "unable to read qemu-server.conf - $!\n";
5414
5415         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5416         if (-f $fwcfgfn) {
5417             my $pve_firewall_dir = '/etc/pve/firewall';
5418             mkdir $pve_firewall_dir; # make sure the dir exists
5419             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5420         }
5421
5422         while (defined(my $line = <$fh>)) {
5423             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5424                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5425                 die "archive does not contain data for drive '$virtdev'\n"
5426                     if !$devinfo->{$devname};
5427                 if (defined($opts->{storage})) {
5428                     $storeid = $opts->{storage} || 'local';
5429                 } elsif (!$storeid) {
5430                     $storeid = 'local';
5431                 }
5432                 $format = 'raw' if !$format;
5433                 $devinfo->{$devname}->{devname} = $devname;
5434                 $devinfo->{$devname}->{virtdev} = $virtdev;
5435                 $devinfo->{$devname}->{format} = $format;
5436                 $devinfo->{$devname}->{storeid} = $storeid;
5437
5438                 # check permission on storage
5439                 my $pool = $opts->{pool}; # todo: do we need that?
5440                 if ($user ne 'root@pam') {
5441                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5442                 }
5443
5444                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5445             }
5446         }
5447
5448         foreach my $devname (keys %$devinfo) {
5449             die "found no device mapping information for device '$devname'\n"
5450                 if !$devinfo->{$devname}->{virtdev};
5451         }
5452
5453         my $cfg = cfs_read_file('storage.cfg');
5454
5455         # create empty/temp config
5456         if ($oldconf) {
5457             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5458             foreach_drive($oldconf, sub {
5459                 my ($ds, $drive) = @_;
5460
5461                 return if drive_is_cdrom($drive);
5462
5463                 my $volid = $drive->{file};
5464
5465                 return if !$volid || $volid =~ m|^/|;
5466
5467                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5468                 return if !$path || !$owner || ($owner != $vmid);
5469
5470                 # Note: only delete disk we want to restore
5471                 # other volumes will become unused
5472                 if ($virtdev_hash->{$ds}) {
5473                     PVE::Storage::vdisk_free($cfg, $volid);
5474                 }
5475             });
5476
5477             # delete vmstate files
5478             # since after the restore we have no snapshots anymore
5479             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5480                 my $snap = $oldconf->{snapshots}->{$snapname};
5481                 if ($snap->{vmstate}) {
5482                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5483                     if (my $err = $@) {
5484                         warn $err;
5485                     }
5486                 }
5487             }
5488         }
5489
5490         my $map = {};
5491         foreach my $virtdev (sort keys %$virtdev_hash) {
5492             my $d = $virtdev_hash->{$virtdev};
5493             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5494             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5495
5496             # test if requested format is supported
5497             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5498             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5499             $d->{format} = $defFormat if !$supported;
5500
5501             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5502                                                   $d->{format}, undef, $alloc_size);
5503             print STDERR "new volume ID is '$volid'\n";
5504             $d->{volid} = $volid;
5505             my $path = PVE::Storage::path($cfg, $volid);
5506
5507             PVE::Storage::activate_volumes($cfg,[$volid]);
5508
5509             my $write_zeros = 1;
5510             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5511                 $write_zeros = 0;
5512             }
5513
5514             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5515
5516             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5517             $map->{$virtdev} = $volid;
5518         }
5519
5520         $fh->seek(0, 0) || die "seek failed - $!\n";
5521
5522         my $outfd = new IO::File ($tmpfn, "w") ||
5523             die "unable to write config for VM $vmid\n";
5524
5525         my $cookie = { netcount => 0 };
5526         while (defined(my $line = <$fh>)) {
5527             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5528         }
5529
5530         $fh->close();
5531         $outfd->close();
5532     };
5533
5534     eval {
5535         # enable interrupts
5536         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5537             die "interrupted by signal\n";
5538         };
5539         local $SIG{ALRM} = sub { die "got timeout\n"; };
5540
5541         $oldtimeout = alarm($timeout);
5542
5543         my $parser = sub {
5544             my $line = shift;
5545
5546             print "$line\n";
5547
5548             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5549                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5550                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5551             } elsif ($line =~ m/^CTIME: /) {
5552                 # we correctly received the vma config, so we can disable
5553                 # the timeout now for disk allocation (set to 10 minutes, so
5554                 # that we always timeout if something goes wrong)
5555                 alarm(600);
5556                 &$print_devmap();
5557                 print $fifofh "done\n";
5558                 my $tmp = $oldtimeout || 0;
5559                 $oldtimeout = undef;
5560                 alarm($tmp);
5561                 close($fifofh);
5562             }
5563         };
5564
5565         print "restore vma archive: $cmd\n";
5566         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5567     };
5568     my $err = $@;
5569
5570     alarm($oldtimeout) if $oldtimeout;
5571
5572     my $vollist = [];
5573     foreach my $devname (keys %$devinfo) {
5574         my $volid = $devinfo->{$devname}->{volid};
5575         push @$vollist, $volid if $volid;
5576     }
5577
5578     my $cfg = cfs_read_file('storage.cfg');
5579     PVE::Storage::deactivate_volumes($cfg, $vollist);
5580
5581     unlink $mapfifo;
5582
5583     if ($err) {
5584         rmtree $tmpdir;
5585         unlink $tmpfn;
5586
5587         foreach my $devname (keys %$devinfo) {
5588             my $volid = $devinfo->{$devname}->{volid};
5589             next if !$volid;
5590             eval {
5591                 if ($volid =~ m|^/|) {
5592                     unlink $volid || die 'unlink failed\n';
5593                 } else {
5594                     PVE::Storage::vdisk_free($cfg, $volid);
5595                 }
5596                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5597             };
5598             print STDERR "unable to cleanup '$volid' - $@" if $@;
5599         }
5600         die $err;
5601     }
5602
5603     rmtree $tmpdir;
5604
5605     rename($tmpfn, $conffile) ||
5606         die "unable to commit configuration file '$conffile'\n";
5607
5608     PVE::Cluster::cfs_update(); # make sure we read new file
5609
5610     eval { rescan($vmid, 1); };
5611     warn $@ if $@;
5612 }
5613
5614 sub restore_tar_archive {
5615     my ($archive, $vmid, $user, $opts) = @_;
5616
5617     if ($archive ne '-') {
5618         my $firstfile = tar_archive_read_firstfile($archive);
5619         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5620             if $firstfile ne 'qemu-server.conf';
5621     }
5622
5623     my $storecfg = cfs_read_file('storage.cfg');
5624
5625     # destroy existing data - keep empty config
5626     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5627     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5628
5629     my $tocmd = "/usr/lib/qemu-server/qmextract";
5630
5631     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5632     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5633     $tocmd .= ' --prealloc' if $opts->{prealloc};
5634     $tocmd .= ' --info' if $opts->{info};
5635
5636     # tar option "xf" does not autodetect compression when read from STDIN,
5637     # so we pipe to zcat
5638     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5639         PVE::Tools::shellquote("--to-command=$tocmd");
5640
5641     my $tmpdir = "/var/tmp/vzdumptmp$$";
5642     mkpath $tmpdir;
5643
5644     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5645     local $ENV{VZDUMP_VMID} = $vmid;
5646     local $ENV{VZDUMP_USER} = $user;
5647
5648     my $conffile = PVE::QemuConfig->config_file($vmid);
5649     my $tmpfn = "$conffile.$$.tmp";
5650
5651     # disable interrupts (always do cleanups)
5652     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5653         print STDERR "got interrupt - ignored\n";
5654     };
5655
5656     eval {
5657         # enable interrupts
5658         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5659             die "interrupted by signal\n";
5660         };
5661
5662         if ($archive eq '-') {
5663             print "extracting archive from STDIN\n";
5664             run_command($cmd, input => "<&STDIN");
5665         } else {
5666             print "extracting archive '$archive'\n";
5667             run_command($cmd);
5668         }
5669
5670         return if $opts->{info};
5671
5672         # read new mapping
5673         my $map = {};
5674         my $statfile = "$tmpdir/qmrestore.stat";
5675         if (my $fd = IO::File->new($statfile, "r")) {
5676             while (defined (my $line = <$fd>)) {
5677                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5678                     $map->{$1} = $2 if $1;
5679                 } else {
5680                     print STDERR "unable to parse line in statfile - $line\n";
5681                 }
5682             }
5683             $fd->close();
5684         }
5685
5686         my $confsrc = "$tmpdir/qemu-server.conf";
5687
5688         my $srcfd = new IO::File($confsrc, "r") ||
5689             die "unable to open file '$confsrc'\n";
5690
5691         my $outfd = new IO::File ($tmpfn, "w") ||
5692             die "unable to write config for VM $vmid\n";
5693
5694         my $cookie = { netcount => 0 };
5695         while (defined (my $line = <$srcfd>)) {
5696             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5697         }
5698
5699         $srcfd->close();
5700         $outfd->close();
5701     };
5702     my $err = $@;
5703
5704     if ($err) {
5705
5706         unlink $tmpfn;
5707
5708         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5709
5710         die $err;
5711     }
5712
5713     rmtree $tmpdir;
5714
5715     rename $tmpfn, $conffile ||
5716         die "unable to commit configuration file '$conffile'\n";
5717
5718     PVE::Cluster::cfs_update(); # make sure we read new file
5719
5720     eval { rescan($vmid, 1); };
5721     warn $@ if $@;
5722 };
5723
5724 sub foreach_writable_storage {
5725     my ($conf, $func) = @_;
5726
5727     my $sidhash = {};
5728
5729     foreach my $ds (keys %$conf) {
5730         next if !is_valid_drivename($ds);
5731
5732         my $drive = parse_drive($ds, $conf->{$ds});
5733         next if !$drive;
5734         next if drive_is_cdrom($drive);
5735
5736         my $volid = $drive->{file};
5737
5738         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5739         $sidhash->{$sid} = $sid if $sid;
5740     }
5741
5742     foreach my $sid (sort keys %$sidhash) {
5743         &$func($sid);
5744     }
5745 }
5746
5747 sub do_snapshots_with_qemu {
5748     my ($storecfg, $volid) = @_;
5749
5750     my $storage_name = PVE::Storage::parse_volume_id($volid);
5751
5752     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5753         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5754         return 1;
5755     }
5756
5757     if ($volid =~ m/\.(qcow2|qed)$/){
5758         return 1;
5759     }
5760
5761     return undef;
5762 }
5763
5764 sub qga_check_running {
5765     my ($vmid) = @_;
5766
5767     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5768     if ($@) {
5769         warn "Qemu Guest Agent are not running - $@";
5770         return 0;
5771     }
5772     return 1;
5773 }
5774
5775 sub template_create {
5776     my ($vmid, $conf, $disk) = @_;
5777
5778     my $storecfg = PVE::Storage::config();
5779
5780     foreach_drive($conf, sub {
5781         my ($ds, $drive) = @_;
5782
5783         return if drive_is_cdrom($drive);
5784         return if $disk && $ds ne $disk;
5785
5786         my $volid = $drive->{file};
5787         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5788
5789         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5790         $drive->{file} = $voliddst;
5791         $conf->{$ds} = print_drive($vmid, $drive);
5792         PVE::QemuConfig->write_config($vmid, $conf);
5793     });
5794 }
5795
5796 sub qemu_img_convert {
5797     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5798
5799     my $storecfg = PVE::Storage::config();
5800     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5801     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5802
5803     if ($src_storeid && $dst_storeid) {
5804
5805         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5806
5807         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5808         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5809
5810         my $src_format = qemu_img_format($src_scfg, $src_volname);
5811         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5812
5813         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5814         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5815
5816         my $cmd = [];
5817         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5818         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5819         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5820
5821         my $parser = sub {
5822             my $line = shift;
5823             if($line =~ m/\((\S+)\/100\%\)/){
5824                 my $percent = $1;
5825                 my $transferred = int($size * $percent / 100);
5826                 my $remaining = $size - $transferred;
5827
5828                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5829             }
5830
5831         };
5832
5833         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5834         my $err = $@;
5835         die "copy failed: $err" if $err;
5836     }
5837 }
5838
5839 sub qemu_img_format {
5840     my ($scfg, $volname) = @_;
5841
5842     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5843         return $1;
5844     } else {
5845         return "raw";
5846     }
5847 }
5848
5849 sub qemu_drive_mirror {
5850     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
5851
5852     my $storecfg = PVE::Storage::config();
5853     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5854
5855     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5856
5857     my $format = qemu_img_format($dst_scfg, $dst_volname);
5858
5859     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5860
5861     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
5862     $opts->{format} = $format if $format;
5863
5864     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5865
5866     eval {
5867     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5868         while (1) {
5869             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5870             my $stat = @$stats[0];
5871             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5872             die "error job is not mirroring" if $stat->{type} ne "mirror";
5873
5874             my $busy = $stat->{busy};
5875             my $ready = $stat->{ready};
5876
5877             if (my $total = $stat->{len}) {
5878                 my $transferred = $stat->{offset} || 0;
5879                 my $remaining = $total - $transferred;
5880                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5881
5882                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5883             }
5884
5885
5886             if ($stat->{ready} eq 'true') {
5887
5888                 last if $vmiddst != $vmid;
5889
5890                 # try to switch the disk if source and destination are on the same guest
5891                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5892                 last if !$@;
5893                 die $@ if $@ !~ m/cannot be completed/;
5894             }
5895             sleep 1;
5896         }
5897
5898
5899     };
5900     my $err = $@;
5901
5902     my $cancel_job = sub {
5903         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5904         while (1) {
5905             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5906             my $stat = @$stats[0];
5907             last if !$stat;
5908             sleep 1;
5909         }
5910     };
5911
5912     if ($err) {
5913         eval { &$cancel_job(); };
5914         die "mirroring error: $err";
5915     }
5916
5917     if ($vmiddst != $vmid) {
5918         # if we clone a disk for a new target vm, we don't switch the disk
5919         &$cancel_job(); # so we call block-job-cancel
5920     }
5921 }
5922
5923 sub clone_disk {
5924     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5925         $newvmid, $storage, $format, $full, $newvollist) = @_;
5926
5927     my $newvolid;
5928
5929     if (!$full) {
5930         print "create linked clone of drive $drivename ($drive->{file})\n";
5931         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5932         push @$newvollist, $newvolid;
5933     } else {
5934         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5935         $storeid = $storage if $storage;
5936
5937         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5938         if (!$format) {
5939             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5940             $format = qemu_img_format($scfg, $volname);
5941         }
5942
5943         # test if requested format is supported - else use default
5944         my $supported = grep { $_ eq $format } @$validFormats;
5945         $format = $defFormat if !$supported;
5946
5947         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5948
5949         print "create full clone of drive $drivename ($drive->{file})\n";
5950         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5951         push @$newvollist, $newvolid;
5952
5953         PVE::Storage::activate_volumes($storecfg, $newvollist);
5954
5955         if (!$running || $snapname) {
5956             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5957         } else {
5958             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5959         }
5960     }
5961
5962     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5963
5964     my $disk = $drive;
5965     $disk->{format} = undef;
5966     $disk->{file} = $newvolid;
5967     $disk->{size} = $size;
5968
5969     return $disk;
5970 }
5971
5972 # this only works if VM is running
5973 sub get_current_qemu_machine {
5974     my ($vmid) = @_;
5975
5976     my $cmd = { execute => 'query-machines', arguments => {} };
5977     my $res = vm_qmp_command($vmid, $cmd);
5978
5979     my ($current, $default);
5980     foreach my $e (@$res) {
5981         $default = $e->{name} if $e->{'is-default'};
5982         $current = $e->{name} if $e->{'is-current'};
5983     }
5984
5985     # fallback to the default machine if current is not supported by qemu
5986     return $current || $default || 'pc';
5987 }
5988
5989 sub qemu_machine_feature_enabled {
5990     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5991
5992     my $current_major;
5993     my $current_minor;
5994
5995     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5996
5997         $current_major = $3;
5998         $current_minor = $4;
5999
6000     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6001
6002         $current_major = $1;
6003         $current_minor = $2;
6004     }
6005
6006     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6007
6008
6009 }
6010
6011 sub qemu_machine_pxe {
6012     my ($vmid, $conf, $machine) = @_;
6013
6014     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6015
6016     foreach my $opt (keys %$conf) {
6017         next if $opt !~ m/^net(\d+)$/;
6018         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6019         next if !$net;
6020         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6021         return $machine.".pxe" if $romfile =~ m/pxe/;
6022         last;
6023     }
6024
6025     return $machine;
6026 }
6027
6028 sub qemu_use_old_bios_files {
6029     my ($machine_type) = @_;
6030
6031     return if !$machine_type;
6032
6033     my $use_old_bios_files = undef;
6034
6035     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6036         $machine_type = $1;
6037         $use_old_bios_files = 1;
6038     } else {
6039         my $kvmver = kvm_user_version();
6040         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6041         # load new efi bios files on migration. So this hack is required to allow
6042         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6043         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6044         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6045     }
6046
6047     return ($use_old_bios_files, $machine_type);
6048 }
6049
6050 sub lspci {
6051
6052     my $devices = {};
6053
6054     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6055             my (undef, $id, $function) = @_;
6056             my $res = { id => $id, function => $function};
6057             push @{$devices->{$id}}, $res;
6058     });
6059
6060     return $devices;
6061 }
6062
6063 sub vm_iothreads_list {
6064     my ($vmid) = @_;
6065
6066     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6067
6068     my $iothreads = {};
6069     foreach my $iothread (@$res) {
6070         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6071     }
6072
6073     return $iothreads;
6074 }
6075
6076 sub scsihw_infos {
6077     my ($conf, $drive) = @_;
6078
6079     my $maxdev = 0;
6080
6081     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6082         $maxdev = 7;
6083     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6084         $maxdev = 1;
6085     } else {
6086         $maxdev = 256;
6087     }
6088
6089     my $controller = int($drive->{index} / $maxdev);
6090     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6091
6092     return ($maxdev, $controller, $controller_prefix);
6093 }
6094
6095 # bash completion helper
6096
6097 sub complete_backup_archives {
6098     my ($cmdname, $pname, $cvalue) = @_;
6099
6100     my $cfg = PVE::Storage::config();
6101
6102     my $storeid;
6103
6104     if ($cvalue =~ m/^([^:]+):/) {
6105         $storeid = $1;
6106     }
6107
6108     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6109
6110     my $res = [];
6111     foreach my $id (keys %$data) {
6112         foreach my $item (@{$data->{$id}}) {
6113             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6114             push @$res, $item->{volid} if defined($item->{volid});
6115         }
6116     }
6117
6118     return $res;
6119 }
6120
6121 my $complete_vmid_full = sub {
6122     my ($running) = @_;
6123
6124     my $idlist = vmstatus();
6125
6126     my $res = [];
6127
6128     foreach my $id (keys %$idlist) {
6129         my $d = $idlist->{$id};
6130         if (defined($running)) {
6131             next if $d->{template};
6132             next if $running && $d->{status} ne 'running';
6133             next if !$running && $d->{status} eq 'running';
6134         }
6135         push @$res, $id;
6136
6137     }
6138     return $res;
6139 };
6140
6141 sub complete_vmid {
6142     return &$complete_vmid_full();
6143 }
6144
6145 sub complete_vmid_stopped {
6146     return &$complete_vmid_full(0);
6147 }
6148
6149 sub complete_vmid_running {
6150     return &$complete_vmid_full(1);
6151 }
6152
6153 sub complete_storage {
6154
6155     my $cfg = PVE::Storage::config();
6156     my $ids = $cfg->{ids};
6157
6158     my $res = [];
6159     foreach my $sid (keys %$ids) {
6160         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6161         next if !$ids->{$sid}->{content}->{images};
6162         push @$res, $sid;
6163     }
6164
6165     return $res;
6166 }
6167
6168 1;