PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35
  36 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  37
  38 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  39
  40 # Note about locking: we use flock on the config file protect
  41 # against concurent actions.
  42 # Aditionaly, we have a 'lock' setting in the config file. This
  43 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  44 # allowed when such lock is set. But you can ignore this kind of
  45 # lock with the --skiplock flag.
  46
  47 cfs_register_file('/qemu-server/',
  48                   \&parse_vm_config,
  49                   \&write_vm_config);
  50
  51 PVE::JSONSchema::register_standard_option('skiplock', {
  52     description => "Ignore locks - only root is allowed to use this option.",
  53     type => 'boolean',
  54     optional => 1,
  55 });
  56
  57 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  58     description => "Some command save/restore state from this location.",
  59     type => 'string',
  60     maxLength => 128,
  61     optional => 1,
  62 });
  63
  64 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  65     description => "The name of the snapshot.",
  66     type => 'string', format => 'pve-configid',
  67     maxLength => 40,
  68 });
  69
  70 #no warnings 'redefine';
  71
  72 sub cgroups_write {
  73    my ($controller, $vmid, $option, $value) = @_;
  74
  75    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  76    PVE::ProcFSTools::write_proc_entry($path, $value);
  77
  78 }
  79
  80 my $nodename = PVE::INotify::nodename();
  81
  82 mkdir "/etc/pve/nodes/$nodename";
  83 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  84 mkdir $confdir;
  85
  86 my $var_run_tmpdir = "/var/run/qemu-server";
  87 mkdir $var_run_tmpdir;
  88
  89 my $lock_dir = "/var/lock/qemu-server";
  90 mkdir $lock_dir;
  91
  92 my $pcisysfs = "/sys/bus/pci";
  93
  94 my $cpudesc = {
  95     cputype => {
  96         description => "Emulated CPU type.",
  97         type => 'string',
  98         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  99         default => 'kvm64',
 100         default_key => 1,
 101     },
 102     hidden => {
 103         description => "Do not identify as a KVM virtual machine.",
 104         type => 'boolean',
 105         optional => 1,
 106         default => 0
 107     },
 108 };
 109
 110 my $confdesc = {
 111     onboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Specifies whether a VM will be started during system bootup.",
 115         default => 0,
 116     },
 117     autostart => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Automatic restart after crash (currently ignored).",
 121         default => 0,
 122     },
 123     hotplug => {
 124         optional => 1,
 125         type => 'string', format => 'pve-hotplug-features',
 126         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 127         default => 'network,disk,usb',
 128     },
 129     reboot => {
 130         optional => 1,
 131         type => 'boolean',
 132         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 133         default => 1,
 134     },
 135     lock => {
 136         optional => 1,
 137         type => 'string',
 138         description => "Lock/unlock the VM.",
 139         enum => [qw(migrate backup snapshot rollback)],
 140     },
 141     cpulimit => {
 142         optional => 1,
 143         type => 'number',
 144         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 145         minimum => 0,
 146         maximum => 128,
 147         default => 0,
 148     },
 149     cpuunits => {
 150         optional => 1,
 151         type => 'integer',
 152         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 153         minimum => 0,
 154         maximum => 500000,
 155         default => 1000,
 156     },
 157     memory => {
 158         optional => 1,
 159         type => 'integer',
 160         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 161         minimum => 16,
 162         default => 512,
 163     },
 164     balloon => {
 165         optional => 1,
 166         type => 'integer',
 167         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 168         minimum => 0,
 169     },
 170     shares => {
 171         optional => 1,
 172         type => 'integer',
 173         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 174         minimum => 0,
 175         maximum => 50000,
 176         default => 1000,
 177     },
 178     keyboard => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 182         enum => PVE::Tools::kvmkeymaplist(),
 183         default => 'en-us',
 184     },
 185     name => {
 186         optional => 1,
 187         type => 'string', format => 'dns-name',
 188         description => "Set a name for the VM. Only used on the configuration web interface.",
 189     },
 190     scsihw => {
 191         optional => 1,
 192         type => 'string',
 193         description => "scsi controller model",
 194         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 195         default => 'lsi',
 196     },
 197     description => {
 198         optional => 1,
 199         type => 'string',
 200         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 201     },
 202     ostype => {
 203         optional => 1,
 204         type => 'string',
 205         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 206         description => <<EODESC,
 207 Used to enable special optimization/features for specific
 208 operating systems:
 209
 210 other  => unspecified OS
 211 wxp    => Microsoft Windows XP
 212 w2k    => Microsoft Windows 2000
 213 w2k3   => Microsoft Windows 2003
 214 w2k8   => Microsoft Windows 2008
 215 wvista => Microsoft Windows Vista
 216 win7   => Microsoft Windows 7
 217 win8   => Microsoft Windows 8/2012
 218 l24    => Linux 2.4 Kernel
 219 l26    => Linux 2.6/3.X Kernel
 220 solaris => solaris/opensolaris/openindiania kernel
 221
 222 other|l24|l26|solaris                       ... no special behaviour
 223 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 224 EODESC
 225     },
 226     boot => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 230         pattern => '[acdn]{1,4}',
 231         default => 'cdn',
 232     },
 233     bootdisk => {
 234         optional => 1,
 235         type => 'string', format => 'pve-qm-bootdisk',
 236         description => "Enable booting from specified disk.",
 237         pattern => '(ide|sata|scsi|virtio)\d+',
 238     },
 239     smp => {
 240         optional => 1,
 241         type => 'integer',
 242         description => "The number of CPUs. Please use option -sockets instead.",
 243         minimum => 1,
 244         default => 1,
 245     },
 246     sockets => {
 247         optional => 1,
 248         type => 'integer',
 249         description => "The number of CPU sockets.",
 250         minimum => 1,
 251         default => 1,
 252     },
 253     cores => {
 254         optional => 1,
 255         type => 'integer',
 256         description => "The number of cores per socket.",
 257         minimum => 1,
 258         default => 1,
 259     },
 260     numa => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Enable/disable Numa.",
 264         default => 0,
 265     },
 266     vcpus => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "Number of hotplugged vcpus.",
 270         minimum => 1,
 271         default => 0,
 272     },
 273     acpi => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable ACPI.",
 277         default => 1,
 278     },
 279     agent => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Enable/disable Qemu GuestAgent.",
 283         default => 0,
 284     },
 285     kvm => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Enable/disable KVM hardware virtualization.",
 289         default => 1,
 290     },
 291     tdf => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Enable/disable time drift fix.",
 295         default => 0,
 296     },
 297     localtime => {
 298         optional => 1,
 299         type => 'boolean',
 300         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 301     },
 302     freeze => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 306     },
 307     vga => {
 308         optional => 1,
 309         type => 'string',
 310         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 311         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 312     },
 313     watchdog => {
 314         optional => 1,
 315         type => 'string', format => 'pve-qm-watchdog',
 316         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 317         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 318     },
 319     startdate => {
 320         optional => 1,
 321         type => 'string',
 322         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 323         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 324         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 325         default => 'now',
 326     },
 327     startup =>  get_standard_option('pve-startup-order'),
 328     template => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable Template.",
 332         default => 0,
 333     },
 334     args => {
 335         optional => 1,
 336         type => 'string',
 337         description => <<EODESCR,
 338 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 339
 340 args: -no-reboot -no-hpet
 341 EODESCR
 342     },
 343     tablet => {
 344         optional => 1,
 345         type => 'boolean',
 346         default => 1,
 347         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 348     },
 349     migrate_speed => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 353         minimum => 0,
 354         default => 0,
 355     },
 356     migrate_downtime => {
 357         optional => 1,
 358         type => 'number',
 359         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 360         minimum => 0,
 361         default => 0.1,
 362     },
 363     cdrom => {
 364         optional => 1,
 365         type => 'string', format => 'pve-qm-drive',
 366         typetext => 'volume',
 367         description => "This is an alias for option -ide2",
 368     },
 369     cpu => {
 370         optional => 1,
 371         description => "Emulated CPU type.",
 372         type => 'string',
 373         format => $cpudesc,
 374     },
 375     parent => get_standard_option('pve-snapshot-name', {
 376         optional => 1,
 377         description => "Parent snapshot name. This is used internally, and should not be modified.",
 378     }),
 379     snaptime => {
 380         optional => 1,
 381         description => "Timestamp for snapshots.",
 382         type => 'integer',
 383         minimum => 0,
 384     },
 385     vmstate => {
 386         optional => 1,
 387         type => 'string', format => 'pve-volume-id',
 388         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 389     },
 390     machine => {
 391         description => "Specific the Qemu machine type.",
 392         type => 'string',
 393         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 394         maxLength => 40,
 395         optional => 1,
 396     },
 397     smbios1 => {
 398         description => "Specify SMBIOS type 1 fields.",
 399         type => 'string', format => 'pve-qm-smbios1',
 400         maxLength => 256,
 401         optional => 1,
 402     },
 403     protection => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 407         default => 0,
 408     },
 409     bios => {
 410         optional => 1,
 411         type => 'string',
 412         enum => [ qw(seabios ovmf) ],
 413         description => "Select BIOS implementation.",
 414         default => 'seabios',
 415     },
 416 };
 417
 418 # what about other qemu settings ?
 419 #cpu => 'string',
 420 #machine => 'string',
 421 #fda => 'file',
 422 #fdb => 'file',
 423 #mtdblock => 'file',
 424 #sd => 'file',
 425 #pflash => 'file',
 426 #snapshot => 'bool',
 427 #bootp => 'file',
 428 ##tftp => 'dir',
 429 ##smb => 'dir',
 430 #kernel => 'file',
 431 #append => 'string',
 432 #initrd => 'file',
 433 ##soundhw => 'string',
 434
 435 while (my ($k, $v) = each %$confdesc) {
 436     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 437 }
 438
 439 my $MAX_IDE_DISKS = 4;
 440 my $MAX_SCSI_DISKS = 14;
 441 my $MAX_VIRTIO_DISKS = 16;
 442 my $MAX_SATA_DISKS = 6;
 443 my $MAX_USB_DEVICES = 5;
 444 my $MAX_NETS = 32;
 445 my $MAX_UNUSED_DISKS = 8;
 446 my $MAX_HOSTPCI_DEVICES = 4;
 447 my $MAX_SERIAL_PORTS = 4;
 448 my $MAX_PARALLEL_PORTS = 3;
 449 my $MAX_NUMA = 8;
 450 my $MAX_MEM = 4194304;
 451 my $STATICMEM = 1024;
 452
 453 my $numadesc = {
 454     optional => 1,
 455     type => 'string', format => 'pve-qm-numanode',
 456     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 457     description => "numa topology",
 458 };
 459 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 460
 461 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 462     $confdesc->{"numa$i"} = $numadesc;
 463 }
 464
 465 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 466                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 467                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 468 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 469
 470 my $netdesc = {
 471     optional => 1,
 472     type => 'string', format => 'pve-qm-net',
 473     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 474     description => <<EODESCR,
 475 Specify network devices.
 476
 477 MODEL is one of: $nic_model_list_txt
 478
 479 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 480 automatically generated if not specified.
 481
 482 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 483
 484 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 485
 486 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 487
 488 10.0.2.2   Gateway
 489 10.0.2.3   DNS Server
 490 10.0.2.4   SMB Server
 491
 492 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 493
 494 EODESCR
 495 };
 496 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 497
 498 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 499     $confdesc->{"net$i"} = $netdesc;
 500 }
 501
 502 my $drivename_hash;
 503
 504 my %drivedesc_base = (
 505     volume => { alias => 'file' },
 506     file => {
 507         type => 'pve-volume-id',
 508         default_key => 1,
 509         format_description => 'volume',
 510         description => "The drive's backing volume.",
 511     },
 512     media => {
 513         type => 'string',
 514         format_description => 'cdrom|disk',
 515         enum => [qw(cdrom disk)],
 516         description => "The drive's media type.",
 517         default => 'disk',
 518         optional => 1
 519     },
 520     cyls => {
 521         type => 'integer',
 522         format_description => 'count',
 523         description => "Force the drive's physical geometry to have a specific cylinder count.",
 524         optional => 1
 525     },
 526     heads => {
 527         type => 'integer',
 528         format_description => 'count',
 529         description => "Force the drive's physical geometry to have a specific head count.",
 530         optional => 1
 531     },
 532     secs => {
 533         type => 'integer',
 534         format_description => 'count',
 535         description => "Force the drive's physical geometry to have a specific sector count.",
 536         optional => 1
 537     },
 538     trans => {
 539         type => 'string',
 540         format_description => 'none|lba|auto',
 541         enum => [qw(none lba auto)],
 542         description => "Force disk geometry bios translation mode.",
 543         optional => 1,
 544     },
 545     snapshot => {
 546         type => 'boolean',
 547         format_description => 'on|off',
 548         description => "Whether the drive should be included when making snapshots.",
 549         optional => 1,
 550     },
 551     cache => {
 552         type => 'string',
 553         format_description => 'none|writethrough|writeback|unsafe|directsync',
 554         enum => [qw(none writethrough writeback unsafe directsync)],
 555         description => "The drive's cache mode",
 556         optional => 1,
 557     },
 558     format => {
 559         type => 'string',
 560         format_description => 'drive format',
 561         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 562         description => "The drive's backing file's data format.",
 563         optional => 1,
 564     },
 565     size => {
 566         type => 'string',
 567         format => 'disk-size',
 568         description => "Disk size. This is purely informational and has no effect.",
 569         optional => 1,
 570     },
 571     backup => {
 572         type => 'boolean',
 573         format_description => 'on|off',
 574         description => "Whether the drive should be included when making backups.",
 575         optional => 1,
 576     },
 577     werror => {
 578         type => 'string',
 579         format_description => 'enospc|ignore|report|stop',
 580         enum => [qw(enospc ignore report stop)],
 581         description => 'Write error action.',
 582         optional => 1,
 583     },
 584     aio => {
 585         type => 'string',
 586         format_description => 'native|threads',
 587         enum => [qw(native threads)],
 588         description => 'AIO type to use.',
 589         optional => 1,
 590     },
 591     discard => {
 592         type => 'string',
 593         format_description => 'ignore|on',
 594         enum => [qw(ignore on)],
 595         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 596         optional => 1,
 597     },
 598     detect_zeroes => {
 599         type => 'boolean',
 600         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 601         optional => 1,
 602     },
 603     serial => {
 604         type => 'string',
 605         format_description => 'serial',
 606         description => "The drive's reported serial number.",
 607         optional => 1,
 608     }
 609 );
 610
 611 my %rerror_fmt = (
 612     rerror => {
 613         type => 'string',
 614         format_description => 'ignore|report|stop',
 615         enum => [qw(ignore report stop)],
 616         description => 'Read error action.',
 617         optional => 1,
 618     },
 619 );
 620
 621 my %iothread_fmt = ( iothread => {
 622         type => 'boolean',
 623         format_description => 'off|on',
 624         description => "Whether to use iothreads for this drive",
 625         optional => 1,
 626 });
 627
 628 my %model_fmt = (
 629     model => {
 630         type => 'string',
 631         format_description => 'model',
 632         description => "The drive's reported model name.",
 633         optional => 1,
 634     },
 635 );
 636
 637 my %queues_fmt = (
 638     queues => {
 639         type => 'integer',
 640         format_description => 'nbqueues',
 641         description => "Number of queues.",
 642         minimum => 2,
 643         optional => 1
 644     }
 645 );
 646
 647 my $add_throttle_desc = sub {
 648     my ($key, $type, $what, $size, $longsize) = @_;
 649     $drivedesc_base{$key} = {
 650         type => $type,
 651         format_description => $size,
 652         description => "Maximum $what speed in $longsize per second.",
 653         optional => 1,
 654     };
 655 };
 656 # throughput: (leaky bucket)
 657 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 658 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 659 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 660 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 662 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 663 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 664 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 665 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 666
 667 # pools: (pool of IO before throttling starts taking effect)
 668 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 670 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 671 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 672 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 673 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 674
 675 my $ide_fmt = {
 676     %drivedesc_base,
 677     %rerror_fmt,
 678     %model_fmt,
 679 };
 680
 681 my $idedesc = {
 682     optional => 1,
 683     type => 'string', format => $ide_fmt,
 684     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 685 };
 686 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 687
 688 my $scsi_fmt = {
 689     %drivedesc_base,
 690     %iothread_fmt,
 691     %queues_fmt,
 692 };
 693 my $scsidesc = {
 694     optional => 1,
 695     type => 'string', format => $scsi_fmt,
 696     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 697 };
 698 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 699
 700 my $sata_fmt = {
 701     %drivedesc_base,
 702     %rerror_fmt,
 703 };
 704 my $satadesc = {
 705     optional => 1,
 706     type => 'string', format => $sata_fmt,
 707     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 708 };
 709 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 710
 711 my $virtio_fmt = {
 712     %drivedesc_base,
 713     %iothread_fmt,
 714     %rerror_fmt,
 715 };
 716 my $virtiodesc = {
 717     optional => 1,
 718     type => 'string', format => $virtio_fmt,
 719     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 720 };
 721 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 722
 723 my $alldrive_fmt = {
 724     %drivedesc_base,
 725     %rerror_fmt,
 726     %iothread_fmt,
 727     %model_fmt,
 728     %queues_fmt,
 729 };
 730
 731 my $usbformat = {
 732     host => {
 733         default_key => 1,
 734         type => 'string', format => 'pve-qm-usb-device',
 735         format_description => 'HOSTUSBDEVICE|spice',
 736         description => 'The Host USB device or port or the value spice',
 737     },
 738     usb3 => {
 739         optional => 1,
 740         type => 'boolean',
 741         format_description => 'yes|no',
 742         description => 'Specifies whether if given host option is a USB3 device or port',
 743     },
 744 };
 745
 746 my $usbdesc = {
 747     optional => 1,
 748     type => 'string', format => $usbformat,
 749     description => <<EODESCR,
 750 Configure an USB device (n is 0 to 4). This can be used to
 751 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 752
 753 'bus-port(.port)*' (decimal numbers) or
 754 'vendor_id:product_id' (hexadeciaml numbers) or
 755 'spice'
 756
 757 You can use the 'lsusb -t' command to list existing usb devices.
 758
 759 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 760
 761 The value 'spice' can be used to add a usb redirection devices for spice.
 762
 763 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 764
 765 EODESCR
 766 };
 767 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 768
 769 my $hostpcidesc = {
 770         optional => 1,
 771         type => 'string', format => 'pve-qm-hostpci',
 772         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 773         description => <<EODESCR,
 774 Map host pci devices. HOSTPCIDEVICE syntax is:
 775
 776 'bus:dev.func' (hexadecimal numbers)
 777
 778 You can us the 'lspci' command to list existing pci devices.
 779
 780 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 781
 782 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 783
 784 Experimental: user reported problems with this option.
 785 EODESCR
 786 };
 787 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 788
 789 my $serialdesc = {
 790         optional => 1,
 791         type => 'string',
 792         pattern => '(/dev/.+|socket)',
 793         description =>  <<EODESCR,
 794 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 795
 796 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 797
 798 Experimental: user reported problems with this option.
 799 EODESCR
 800 };
 801
 802 my $paralleldesc= {
 803         optional => 1,
 804         type => 'string',
 805         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 806         description =>  <<EODESCR,
 807 Map host parallel devices (n is 0 to 2).
 808
 809 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 810
 811 Experimental: user reported problems with this option.
 812 EODESCR
 813 };
 814
 815 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 816     $confdesc->{"parallel$i"} = $paralleldesc;
 817 }
 818
 819 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 820     $confdesc->{"serial$i"} = $serialdesc;
 821 }
 822
 823 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 824     $confdesc->{"hostpci$i"} = $hostpcidesc;
 825 }
 826
 827 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 828     $drivename_hash->{"ide$i"} = 1;
 829     $confdesc->{"ide$i"} = $idedesc;
 830 }
 831
 832 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 833     $drivename_hash->{"sata$i"} = 1;
 834     $confdesc->{"sata$i"} = $satadesc;
 835 }
 836
 837 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 838     $drivename_hash->{"scsi$i"} = 1;
 839     $confdesc->{"scsi$i"} = $scsidesc ;
 840 }
 841
 842 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 843     $drivename_hash->{"virtio$i"} = 1;
 844     $confdesc->{"virtio$i"} = $virtiodesc;
 845 }
 846
 847 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 848     $confdesc->{"usb$i"} = $usbdesc;
 849 }
 850
 851 my $unuseddesc = {
 852     optional => 1,
 853     type => 'string', format => 'pve-volume-id',
 854     description => "Reference to unused volumes.",
 855 };
 856
 857 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 858     $confdesc->{"unused$i"} = $unuseddesc;
 859 }
 860
 861 my $kvm_api_version = 0;
 862
 863 sub kvm_version {
 864
 865     return $kvm_api_version if $kvm_api_version;
 866
 867     my $fh = IO::File->new("</dev/kvm") ||
 868         return 0;
 869
 870     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 871         $kvm_api_version = $v;
 872     }
 873
 874     $fh->close();
 875
 876     return  $kvm_api_version;
 877 }
 878
 879 my $kvm_user_version;
 880
 881 sub kvm_user_version {
 882
 883     return $kvm_user_version if $kvm_user_version;
 884
 885     $kvm_user_version = 'unknown';
 886
 887     my $code = sub {
 888         my $line = shift;
 889         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 890             $kvm_user_version = $2;
 891         }
 892     };
 893
 894     eval { run_command("kvm -version", outfunc => $code); };
 895     warn $@ if $@;
 896
 897     return $kvm_user_version;
 898
 899 }
 900
 901 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 902
 903 sub valid_drive_names {
 904     # order is important - used to autoselect boot disk
 905     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 906             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 907             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 908             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 909 }
 910
 911 sub is_valid_drivename {
 912     my $dev = shift;
 913
 914     return defined($drivename_hash->{$dev});
 915 }
 916
 917 sub option_exists {
 918     my $key = shift;
 919     return defined($confdesc->{$key});
 920 }
 921
 922 sub nic_models {
 923     return $nic_model_list;
 924 }
 925
 926 sub os_list_description {
 927
 928     return {
 929         other => 'Other',
 930         wxp => 'Windows XP',
 931         w2k => 'Windows 2000',
 932         w2k3 =>, 'Windows 2003',
 933         w2k8 => 'Windows 2008',
 934         wvista => 'Windows Vista',
 935         win7 => 'Windows 7',
 936         win8 => 'Windows 8/2012',
 937         l24 => 'Linux 2.4',
 938         l26 => 'Linux 2.6',
 939     };
 940 }
 941
 942 my $cdrom_path;
 943
 944 sub get_cdrom_path {
 945
 946     return  $cdrom_path if $cdrom_path;
 947
 948     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 949     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 950     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 951 }
 952
 953 sub get_iso_path {
 954     my ($storecfg, $vmid, $cdrom) = @_;
 955
 956     if ($cdrom eq 'cdrom') {
 957         return get_cdrom_path();
 958     } elsif ($cdrom eq 'none') {
 959         return '';
 960     } elsif ($cdrom =~ m|^/|) {
 961         return $cdrom;
 962     } else {
 963         return PVE::Storage::path($storecfg, $cdrom);
 964     }
 965 }
 966
 967 # try to convert old style file names to volume IDs
 968 sub filename_to_volume_id {
 969     my ($vmid, $file, $media) = @_;
 970
 971     if (!($file eq 'none' || $file eq 'cdrom' ||
 972           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 973
 974         return undef if $file =~ m|/|;
 975
 976         if ($media && $media eq 'cdrom') {
 977             $file = "local:iso/$file";
 978         } else {
 979             $file = "local:$vmid/$file";
 980         }
 981     }
 982
 983     return $file;
 984 }
 985
 986 sub verify_media_type {
 987     my ($opt, $vtype, $media) = @_;
 988
 989     return if !$media;
 990
 991     my $etype;
 992     if ($media eq 'disk') {
 993         $etype = 'images';
 994     } elsif ($media eq 'cdrom') {
 995         $etype = 'iso';
 996     } else {
 997         die "internal error";
 998     }
 999
1000     return if ($vtype eq $etype);
1001
1002     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1003 }
1004
1005 sub cleanup_drive_path {
1006     my ($opt, $storecfg, $drive) = @_;
1007
1008     # try to convert filesystem paths to volume IDs
1009
1010     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1011         ($drive->{file} !~ m|^/dev/.+|) &&
1012         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1013         ($drive->{file} !~ m/^\d+$/)) {
1014         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1015         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1016         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1017         verify_media_type($opt, $vtype, $drive->{media});
1018         $drive->{file} = $volid;
1019     }
1020
1021     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1022 }
1023
1024 sub parse_hotplug_features {
1025     my ($data) = @_;
1026
1027     my $res = {};
1028
1029     return $res if $data eq '0';
1030
1031     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1032
1033     foreach my $feature (PVE::Tools::split_list($data)) {
1034         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1035             $res->{$1} = 1;
1036         } else {
1037             warn "ignoring unknown hotplug feature '$feature'\n";
1038         }
1039     }
1040     return $res;
1041 }
1042
1043 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1044 sub pve_verify_hotplug_features {
1045     my ($value, $noerr) = @_;
1046
1047     return $value if parse_hotplug_features($value);
1048
1049     return undef if $noerr;
1050
1051     die "unable to parse hotplug option\n";
1052 }
1053
1054 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1055 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1056 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1057 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1058 #        [,iothread=on][,serial=serial][,model=model]
1059
1060 sub parse_drive {
1061     my ($key, $data) = @_;
1062
1063     my ($interface, $index);
1064
1065     if ($key =~ m/^([^\d]+)(\d+)$/) {
1066         $interface = $1;
1067         $index = $2;
1068     } else {
1069         return undef;
1070     }
1071
1072     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1073                                      : $confdesc->{$key}->{format};
1074     if (!$desc) {
1075         warn "invalid drive key: $key\n";
1076         return undef;
1077     }
1078     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1079     return undef if !$res;
1080     $res->{interface} = $interface;
1081     $res->{index} = $index;
1082
1083     my $error = 0;
1084     foreach my $opt (qw(bps bps_rd bps_wr)) {
1085         if (my $bps = defined(delete $res->{$opt})) {
1086             if (defined($res->{"m$opt"})) {
1087                 warn "both $opt and m$opt specified\n";
1088                 ++$error;
1089                 next;
1090             }
1091             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1092         }
1093     }
1094     return undef if $error;
1095
1096     return undef if $res->{mbps_rd} && $res->{mbps};
1097     return undef if $res->{mbps_wr} && $res->{mbps};
1098     return undef if $res->{iops_rd} && $res->{iops};
1099     return undef if $res->{iops_wr} && $res->{iops};
1100
1101     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1102         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1103         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1104         return undef if $res->{interface} eq 'virtio';
1105     }
1106
1107     if (my $size = $res->{size}) {
1108         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1109     }
1110
1111     return $res;
1112 }
1113
1114 sub print_drive {
1115     my ($vmid, $drive) = @_;
1116     my $data = { %$drive };
1117     delete $data->{$_} for qw(index interface);
1118     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1119 }
1120
1121 sub scsi_inquiry {
1122     my($fh, $noerr) = @_;
1123
1124     my $SG_IO = 0x2285;
1125     my $SG_GET_VERSION_NUM = 0x2282;
1126
1127     my $versionbuf = "\x00" x 8;
1128     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1129     if (!$ret) {
1130         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1131         return undef;
1132     }
1133     my $version = unpack("I", $versionbuf);
1134     if ($version < 30000) {
1135         die "scsi generic interface too old\n"  if !$noerr;
1136         return undef;
1137     }
1138
1139     my $buf = "\x00" x 36;
1140     my $sensebuf = "\x00" x 8;
1141     my $cmd = pack("C x3 C x1", 0x12, 36);
1142
1143     # see /usr/include/scsi/sg.h
1144     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1145
1146     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1147                       length($sensebuf), 0, length($buf), $buf,
1148                       $cmd, $sensebuf, 6000);
1149
1150     $ret = ioctl($fh, $SG_IO, $packet);
1151     if (!$ret) {
1152         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1153         return undef;
1154     }
1155
1156     my @res = unpack($sg_io_hdr_t, $packet);
1157     if ($res[17] || $res[18]) {
1158         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1159         return undef;
1160     }
1161
1162     my $res = {};
1163     (my $byte0, my $byte1, $res->{vendor},
1164      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1165
1166     $res->{removable} = $byte1 & 128 ? 1 : 0;
1167     $res->{type} = $byte0 & 31;
1168
1169     return $res;
1170 }
1171
1172 sub path_is_scsi {
1173     my ($path) = @_;
1174
1175     my $fh = IO::File->new("+<$path") || return undef;
1176     my $res = scsi_inquiry($fh, 1);
1177     close($fh);
1178
1179     return $res;
1180 }
1181
1182 sub machine_type_is_q35 {
1183     my ($conf) = @_;
1184
1185     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1186 }
1187
1188 sub print_tabletdevice_full {
1189     my ($conf) = @_;
1190
1191     my $q35 = machine_type_is_q35($conf);
1192
1193     # we use uhci for old VMs because tablet driver was buggy in older qemu
1194     my $usbbus = $q35 ? "ehci" : "uhci";
1195
1196     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1197 }
1198
1199 sub print_drivedevice_full {
1200     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1201
1202     my $device = '';
1203     my $maxdev = 0;
1204
1205     if ($drive->{interface} eq 'virtio') {
1206         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1207         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1208         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1209     } elsif ($drive->{interface} eq 'scsi') {
1210
1211         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1212         my $unit = $drive->{index} % $maxdev;
1213         my $devicetype = 'hd';
1214         my $path = '';
1215         if (drive_is_cdrom($drive)) {
1216             $devicetype = 'cd';
1217         } else {
1218             if ($drive->{file} =~ m|^/|) {
1219                 $path = $drive->{file};
1220                 if (my $info = path_is_scsi($path)) {
1221                     if ($info->{type} == 0) {
1222                         $devicetype = 'block';
1223                     } elsif ($info->{type} == 1) { # tape
1224                         $devicetype = 'generic';
1225                     }
1226                 }
1227             } else {
1228                  $path = PVE::Storage::path($storecfg, $drive->{file});
1229             }
1230
1231             if($path =~ m/^iscsi\:\/\//){
1232                 $devicetype = 'generic';
1233             }
1234         }
1235
1236         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1237             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1238         } else {
1239             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1240         }
1241
1242     } elsif ($drive->{interface} eq 'ide'){
1243         $maxdev = 2;
1244         my $controller = int($drive->{index} / $maxdev);
1245         my $unit = $drive->{index} % $maxdev;
1246         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1247
1248         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1249         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1250             $device .= ",model=$model";
1251         }
1252     } elsif ($drive->{interface} eq 'sata'){
1253         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1254         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1255         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1256     } elsif ($drive->{interface} eq 'usb') {
1257         die "implement me";
1258         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1259     } else {
1260         die "unsupported interface type";
1261     }
1262
1263     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1264
1265     return $device;
1266 }
1267
1268 sub get_initiator_name {
1269     my $initiator;
1270
1271     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1272     while (defined(my $line = <$fh>)) {
1273         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1274         $initiator = $1;
1275         last;
1276     }
1277     $fh->close();
1278
1279     return $initiator;
1280 }
1281
1282 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1283 sub print_drive_full {
1284     my ($storecfg, $vmid, $drive) = @_;
1285
1286     my $path;
1287     my $volid = $drive->{file};
1288     my $format;
1289
1290     if (drive_is_cdrom($drive)) {
1291         $path = get_iso_path($storecfg, $vmid, $volid);
1292     } else {
1293         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1294         if ($storeid) {
1295             $path = PVE::Storage::path($storecfg, $volid);
1296             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1297             $format = qemu_img_format($scfg, $volname);
1298         } else {
1299             $path = $volid;
1300             $format = "raw";
1301         }
1302    }
1303
1304     my $opts = '';
1305     foreach my $o (@qemu_drive_options) {
1306         next if $o eq 'bootindex';
1307         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1308     }
1309
1310     $opts .= ",format=$format" if $format && !$drive->{format};
1311
1312     foreach my $o (qw(bps bps_rd bps_wr)) {
1313         my $v = $drive->{"m$o"};
1314         $opts .= ",$o=" . int($v*1024*1024) if $v;
1315     }
1316
1317     my $cache_direct = 0;
1318
1319     if (my $cache = $drive->{cache}) {
1320         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1321     } elsif (!drive_is_cdrom($drive)) {
1322         $opts .= ",cache=none";
1323         $cache_direct = 1;
1324     }
1325
1326     # aio native works only with O_DIRECT
1327     if (!$drive->{aio}) {
1328         if($cache_direct) {
1329             $opts .= ",aio=native";
1330         } else {
1331             $opts .= ",aio=threads";
1332         }
1333     }
1334
1335     if (!drive_is_cdrom($drive)) {
1336         my $detectzeroes;
1337         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1338             $detectzeroes = 'off';
1339         } elsif ($drive->{discard}) {
1340             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1341         } else {
1342             # This used to be our default with discard not being specified:
1343             $detectzeroes = 'on';
1344         }
1345         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1346     }
1347
1348     my $pathinfo = $path ? "file=$path," : '';
1349
1350     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1351 }
1352
1353 sub print_netdevice_full {
1354     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1355
1356     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1357
1358     my $device = $net->{model};
1359     if ($net->{model} eq 'virtio') {
1360          $device = 'virtio-net-pci';
1361      };
1362
1363     my $pciaddr = print_pci_addr("$netid", $bridges);
1364     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1365     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1366         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1367         my $vectors = $net->{queues} * 2 + 2;
1368         $tmpstr .= ",vectors=$vectors,mq=on";
1369     }
1370     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1371
1372     if ($use_old_bios_files) {
1373         my $romfile;
1374         if ($device eq 'virtio-net-pci') {
1375             $romfile = 'pxe-virtio.rom';
1376         } elsif ($device eq 'e1000') {
1377             $romfile = 'pxe-e1000.rom';
1378         } elsif ($device eq 'ne2k') {
1379             $romfile = 'pxe-ne2k_pci.rom';
1380         } elsif ($device eq 'pcnet') {
1381             $romfile = 'pxe-pcnet.rom';
1382         } elsif ($device eq 'rtl8139') {
1383             $romfile = 'pxe-rtl8139.rom';
1384         }
1385         $tmpstr .= ",romfile=$romfile" if $romfile;
1386     }
1387
1388     return $tmpstr;
1389 }
1390
1391 sub print_netdev_full {
1392     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1393
1394     my $i = '';
1395     if ($netid =~ m/^net(\d+)$/) {
1396         $i = int($1);
1397     }
1398
1399     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1400
1401     my $ifname = "tap${vmid}i$i";
1402
1403     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1404     die "interface name '$ifname' is too long (max 15 character)\n"
1405         if length($ifname) >= 16;
1406
1407     my $vhostparam = '';
1408     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1409
1410     my $vmname = $conf->{name} || "vm$vmid";
1411
1412     my $netdev = "";
1413     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1414
1415     if ($net->{bridge}) {
1416         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1417     } else {
1418         $netdev = "type=user,id=$netid,hostname=$vmname";
1419     }
1420
1421     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1422
1423     return $netdev;
1424 }
1425
1426 sub drive_is_cdrom {
1427     my ($drive) = @_;
1428
1429     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1430
1431 }
1432
1433 sub parse_numa {
1434     my ($data) = @_;
1435
1436     my $res = {};
1437
1438     foreach my $kvp (split(/,/, $data)) {
1439
1440         if ($kvp =~ m/^memory=(\S+)$/) {
1441             $res->{memory} = $1;
1442         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1443             $res->{policy} = $1;
1444         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1445             $res->{cpus}->{start} = $1;
1446             $res->{cpus}->{end} = $3;
1447         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1448             $res->{hostnodes}->{start} = $1;
1449             $res->{hostnodes}->{end} = $3;
1450         } else {
1451             return undef;
1452         }
1453     }
1454
1455     return $res;
1456 }
1457
1458 sub parse_hostpci {
1459     my ($value) = @_;
1460
1461     return undef if !$value;
1462
1463
1464     my @list = split(/,/, $value);
1465     my $found;
1466
1467     my $res = {};
1468     foreach my $kv (@list) {
1469
1470         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1471             $found = 1;
1472             if(defined($4)){
1473                 push @{$res->{pciid}}, { id => $2 , function => $4};
1474
1475             }else{
1476                 my $pcidevices = lspci($2);
1477                 $res->{pciid} = $pcidevices->{$2};
1478             }
1479         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1480             $res->{rombar} = $1;
1481         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1482             $res->{'x-vga'} = $1;
1483         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1484             $res->{pcie} = 1 if $1 == 1;
1485         } else {
1486             warn "unknown hostpci setting '$kv'\n";
1487         }
1488     }
1489
1490     return undef if !$found;
1491
1492     return $res;
1493 }
1494
1495 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1496 sub parse_net {
1497     my ($data) = @_;
1498
1499     my $res = {};
1500
1501     foreach my $kvp (split(/,/, $data)) {
1502
1503         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1504             my $model = lc($1);
1505             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1506             $res->{model} = $model;
1507             $res->{macaddr} = $mac;
1508         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1509             $res->{bridge} = $1;
1510         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1511             $res->{queues} = $1;
1512         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1513             $res->{rate} = $1;
1514         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1515             $res->{tag} = $1;
1516         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1517             $res->{trunks} = $1;
1518         } elsif ($kvp =~ m/^firewall=([01])$/) {
1519             $res->{firewall} = $1;
1520         } elsif ($kvp =~ m/^link_down=([01])$/) {
1521             $res->{link_down} = $1;
1522         } else {
1523             return undef;
1524         }
1525
1526     }
1527
1528     return undef if !$res->{model};
1529
1530     return $res;
1531 }
1532
1533 sub print_net {
1534     my $net = shift;
1535
1536     my $res = "$net->{model}";
1537     $res .= "=$net->{macaddr}" if $net->{macaddr};
1538     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1539     $res .= ",rate=$net->{rate}" if $net->{rate};
1540     $res .= ",tag=$net->{tag}" if $net->{tag};
1541     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1542     $res .= ",firewall=1" if $net->{firewall};
1543     $res .= ",link_down=1" if $net->{link_down};
1544     $res .= ",queues=$net->{queues}" if $net->{queues};
1545
1546     return $res;
1547 }
1548
1549 sub add_random_macs {
1550     my ($settings) = @_;
1551
1552     foreach my $opt (keys %$settings) {
1553         next if $opt !~ m/^net(\d+)$/;
1554         my $net = parse_net($settings->{$opt});
1555         next if !$net;
1556         $settings->{$opt} = print_net($net);
1557     }
1558 }
1559
1560 sub add_unused_volume {
1561     my ($config, $volid) = @_;
1562
1563     my $key;
1564     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1565         my $test = "unused$ind";
1566         if (my $vid = $config->{$test}) {
1567             return if $vid eq $volid; # do not add duplicates
1568         } else {
1569             $key = $test;
1570         }
1571     }
1572
1573     die "To many unused volume - please delete them first.\n" if !$key;
1574
1575     $config->{$key} = $volid;
1576
1577     return $key;
1578 }
1579
1580 sub vm_is_volid_owner {
1581     my ($storecfg, $vmid, $volid) = @_;
1582
1583     if ($volid !~  m|^/|) {
1584         my ($path, $owner);
1585         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1586         if ($owner && ($owner == $vmid)) {
1587             return 1;
1588         }
1589     }
1590
1591     return undef;
1592 }
1593
1594 sub split_flagged_list {
1595     my $text = shift || '';
1596     $text =~ s/[,;]/ /g;
1597     $text =~ s/^\s+//;
1598     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1599 }
1600
1601 sub join_flagged_list {
1602     my ($how, $lst) = @_;
1603     join $how, map { $lst->{$_} . $_ } keys %$lst;
1604 }
1605
1606 sub vmconfig_delete_pending_option {
1607     my ($conf, $key, $force) = @_;
1608
1609     delete $conf->{pending}->{$key};
1610     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1611     $pending_delete_hash->{$key} = $force ? '!' : '';
1612     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1613 }
1614
1615 sub vmconfig_undelete_pending_option {
1616     my ($conf, $key) = @_;
1617
1618     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1619     delete $pending_delete_hash->{$key};
1620
1621     if (%$pending_delete_hash) {
1622         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1623     } else {
1624         delete $conf->{pending}->{delete};
1625     }
1626 }
1627
1628 sub vmconfig_register_unused_drive {
1629     my ($storecfg, $vmid, $conf, $drive) = @_;
1630
1631     if (!drive_is_cdrom($drive)) {
1632         my $volid = $drive->{file};
1633         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1634             add_unused_volume($conf, $volid, $vmid);
1635         }
1636     }
1637 }
1638
1639 sub vmconfig_cleanup_pending {
1640     my ($conf) = @_;
1641
1642     # remove pending changes when nothing changed
1643     my $changes;
1644     foreach my $opt (keys %{$conf->{pending}}) {
1645         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1646             $changes = 1;
1647             delete $conf->{pending}->{$opt};
1648         }
1649     }
1650
1651     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1652     my $pending_delete_hash = {};
1653     while (my ($opt, $force) = each %$current_delete_hash) {
1654         if (defined($conf->{$opt})) {
1655             $pending_delete_hash->{$opt} = $force;
1656         } else {
1657             $changes = 1;
1658         }
1659     }
1660
1661     if (%$pending_delete_hash) {
1662         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1663     } else {
1664         delete $conf->{pending}->{delete};
1665     }
1666
1667     return $changes;
1668 }
1669
1670 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1671 my $smbios1_desc = {
1672     uuid => {
1673         type => 'string',
1674         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1675         format_description => 'UUID',
1676         optional => 1,
1677     },
1678     version => {
1679         type => 'string',
1680         pattern => '\S+',
1681         format_description => 'str',
1682         optional => 1,
1683     },
1684     serial => {
1685         type => 'string',
1686         pattern => '\S+',
1687         format_description => 'str',
1688         optional => 1,
1689     },
1690     manufacturer => {
1691         type => 'string',
1692         pattern => '\S+',
1693         format_description => 'name',
1694         optional => 1,
1695     },
1696     product => {
1697         type => 'string',
1698         pattern => '\S+',
1699         format_description => 'name',
1700         optional => 1,
1701     },
1702     sku => {
1703         type => 'string',
1704         pattern => '\S+',
1705         format_description => 'str',
1706         optional => 1,
1707     },
1708     family => {
1709         type => 'string',
1710         pattern => '\S+',
1711         format_description => 'str',
1712         optional => 1,
1713     },
1714 };
1715
1716 sub parse_smbios1 {
1717     my ($data) = @_;
1718
1719     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1720     warn $@ if $@;
1721     return $res;
1722 }
1723
1724 sub print_smbios1 {
1725     my ($smbios1) = @_;
1726     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1727 }
1728
1729 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1730
1731 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1732 sub verify_bootdisk {
1733     my ($value, $noerr) = @_;
1734
1735     return $value if is_valid_drivename($value);
1736
1737     return undef if $noerr;
1738
1739     die "invalid boot disk '$value'\n";
1740 }
1741
1742 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1743 sub verify_numa {
1744     my ($value, $noerr) = @_;
1745
1746     return $value if parse_numa($value);
1747
1748     return undef if $noerr;
1749
1750     die "unable to parse numa options\n";
1751 }
1752
1753 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1754 sub verify_net {
1755     my ($value, $noerr) = @_;
1756
1757     return $value if parse_net($value);
1758
1759     return undef if $noerr;
1760
1761     die "unable to parse network options\n";
1762 }
1763
1764 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1765 sub verify_hostpci {
1766     my ($value, $noerr) = @_;
1767
1768     return $value if parse_hostpci($value);
1769
1770     return undef if $noerr;
1771
1772     die "unable to parse pci id\n";
1773 }
1774
1775 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1776 sub verify_watchdog {
1777     my ($value, $noerr) = @_;
1778
1779     return $value if parse_watchdog($value);
1780
1781     return undef if $noerr;
1782
1783     die "unable to parse watchdog options\n";
1784 }
1785
1786 sub parse_watchdog {
1787     my ($value) = @_;
1788
1789     return undef if !$value;
1790
1791     my $res = {};
1792
1793     foreach my $p (split(/,/, $value)) {
1794         next if $p =~ m/^\s*$/;
1795
1796         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1797             $res->{model} = $2;
1798         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1799             $res->{action} = $2;
1800         } else {
1801             return undef;
1802         }
1803     }
1804
1805     return $res;
1806 }
1807
1808 sub parse_usb_device {
1809     my ($value) = @_;
1810
1811     return undef if !$value;
1812
1813     my $res = {};
1814     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1815         $res->{vendorid} = $2;
1816         $res->{productid} = $4;
1817     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1818         $res->{hostbus} = $1;
1819         $res->{hostport} = $2;
1820     } elsif ($value =~ m/^spice$/i) {
1821         $res->{spice} = 1;
1822     } else {
1823         return undef;
1824     }
1825
1826     return $res;
1827 }
1828
1829 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1830 sub verify_usb_device {
1831     my ($value, $noerr) = @_;
1832
1833     return $value if parse_usb_device($value);
1834
1835     return undef if $noerr;
1836
1837     die "unable to parse usb device\n";
1838 }
1839
1840 # add JSON properties for create and set function
1841 sub json_config_properties {
1842     my $prop = shift;
1843
1844     foreach my $opt (keys %$confdesc) {
1845         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1846         $prop->{$opt} = $confdesc->{$opt};
1847     }
1848
1849     return $prop;
1850 }
1851
1852 sub check_type {
1853     my ($key, $value) = @_;
1854
1855     die "unknown setting '$key'\n" if !$confdesc->{$key};
1856
1857     my $type = $confdesc->{$key}->{type};
1858
1859     if (!defined($value)) {
1860         die "got undefined value\n";
1861     }
1862
1863     if ($value =~ m/[\n\r]/) {
1864         die "property contains a line feed\n";
1865     }
1866
1867     if ($type eq 'boolean') {
1868         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1869         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1870         die "type check ('boolean') failed - got '$value'\n";
1871     } elsif ($type eq 'integer') {
1872         return int($1) if $value =~ m/^(\d+)$/;
1873         die "type check ('integer') failed - got '$value'\n";
1874     } elsif ($type eq 'number') {
1875         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1876         die "type check ('number') failed - got '$value'\n";
1877     } elsif ($type eq 'string') {
1878         if (my $fmt = $confdesc->{$key}->{format}) {
1879             if ($fmt eq 'pve-qm-drive') {
1880                 # special case - we need to pass $key to parse_drive()
1881                 my $drive = parse_drive($key, $value);
1882                 return $value if $drive;
1883                 die "unable to parse drive options\n";
1884             }
1885             PVE::JSONSchema::check_format($fmt, $value);
1886             return $value;
1887         }
1888         $value =~ s/^\"(.*)\"$/$1/;
1889         return $value;
1890     } else {
1891         die "internal error"
1892     }
1893 }
1894
1895 sub check_iommu_support{
1896     #fixme : need to check IOMMU support
1897     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1898
1899     my $iommu=1;
1900     return $iommu;
1901
1902 }
1903
1904 sub touch_config {
1905     my ($vmid) = @_;
1906
1907     my $conf = PVE::QemuConfig->config_file($vmid);
1908     utime undef, undef, $conf;
1909 }
1910
1911 sub destroy_vm {
1912     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1913
1914     my $conffile = PVE::QemuConfig->config_file($vmid);
1915
1916     my $conf = PVE::QemuConfig->load_config($vmid);
1917
1918     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1919
1920     # only remove disks owned by this VM
1921     foreach_drive($conf, sub {
1922         my ($ds, $drive) = @_;
1923
1924         return if drive_is_cdrom($drive);
1925
1926         my $volid = $drive->{file};
1927
1928         return if !$volid || $volid =~ m|^/|;
1929
1930         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1931         return if !$path || !$owner || ($owner != $vmid);
1932
1933         PVE::Storage::vdisk_free($storecfg, $volid);
1934     });
1935
1936     if ($keep_empty_config) {
1937         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1938     } else {
1939         unlink $conffile;
1940     }
1941
1942     # also remove unused disk
1943     eval {
1944         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1945
1946         eval {
1947             PVE::Storage::foreach_volid($dl, sub {
1948                 my ($volid, $sid, $volname, $d) = @_;
1949                 PVE::Storage::vdisk_free($storecfg, $volid);
1950             });
1951         };
1952         warn $@ if $@;
1953
1954     };
1955     warn $@ if $@;
1956 }
1957
1958 sub parse_vm_config {
1959     my ($filename, $raw) = @_;
1960
1961     return undef if !defined($raw);
1962
1963     my $res = {
1964         digest => Digest::SHA::sha1_hex($raw),
1965         snapshots => {},
1966         pending => {},
1967     };
1968
1969     $filename =~ m|/qemu-server/(\d+)\.conf$|
1970         || die "got strange filename '$filename'";
1971
1972     my $vmid = $1;
1973
1974     my $conf = $res;
1975     my $descr;
1976     my $section = '';
1977
1978     my @lines = split(/\n/, $raw);
1979     foreach my $line (@lines) {
1980         next if $line =~ m/^\s*$/;
1981
1982         if ($line =~ m/^\[PENDING\]\s*$/i) {
1983             $section = 'pending';
1984             if (defined($descr)) {
1985                 $descr =~ s/\s+$//;
1986                 $conf->{description} = $descr;
1987             }
1988             $descr = undef;
1989             $conf = $res->{$section} = {};
1990             next;
1991
1992         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1993             $section = $1;
1994             if (defined($descr)) {
1995                 $descr =~ s/\s+$//;
1996                 $conf->{description} = $descr;
1997             }
1998             $descr = undef;
1999             $conf = $res->{snapshots}->{$section} = {};
2000             next;
2001         }
2002
2003         if ($line =~ m/^\#(.*)\s*$/) {
2004             $descr = '' if !defined($descr);
2005             $descr .= PVE::Tools::decode_text($1) . "\n";
2006             next;
2007         }
2008
2009         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2010             $descr = '' if !defined($descr);
2011             $descr .= PVE::Tools::decode_text($2);
2012         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2013             $conf->{snapstate} = $1;
2014         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2015             my $key = $1;
2016             my $value = $2;
2017             $conf->{$key} = $value;
2018         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2019             my $value = $1;
2020             if ($section eq 'pending') {
2021                 $conf->{delete} = $value; # we parse this later
2022             } else {
2023                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2024             }
2025         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2026             my $key = $1;
2027             my $value = $2;
2028             eval { $value = check_type($key, $value); };
2029             if ($@) {
2030                 warn "vm $vmid - unable to parse value of '$key' - $@";
2031             } else {
2032                 my $fmt = $confdesc->{$key}->{format};
2033                 if ($fmt && $fmt eq 'pve-qm-drive') {
2034                     my $v = parse_drive($key, $value);
2035                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2036                         $v->{file} = $volid;
2037                         $value = print_drive($vmid, $v);
2038                     } else {
2039                         warn "vm $vmid - unable to parse value of '$key'\n";
2040                         next;
2041                     }
2042                 }
2043
2044                 if ($key eq 'cdrom') {
2045                     $conf->{ide2} = $value;
2046                 } else {
2047                     $conf->{$key} = $value;
2048                 }
2049             }
2050         }
2051     }
2052
2053     if (defined($descr)) {
2054         $descr =~ s/\s+$//;
2055         $conf->{description} = $descr;
2056     }
2057     delete $res->{snapstate}; # just to be sure
2058
2059     return $res;
2060 }
2061
2062 sub write_vm_config {
2063     my ($filename, $conf) = @_;
2064
2065     delete $conf->{snapstate}; # just to be sure
2066
2067     if ($conf->{cdrom}) {
2068         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2069         $conf->{ide2} = $conf->{cdrom};
2070         delete $conf->{cdrom};
2071     }
2072
2073     # we do not use 'smp' any longer
2074     if ($conf->{sockets}) {
2075         delete $conf->{smp};
2076     } elsif ($conf->{smp}) {
2077         $conf->{sockets} = $conf->{smp};
2078         delete $conf->{cores};
2079         delete $conf->{smp};
2080     }
2081
2082     my $used_volids = {};
2083
2084     my $cleanup_config = sub {
2085         my ($cref, $pending, $snapname) = @_;
2086
2087         foreach my $key (keys %$cref) {
2088             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2089                 $key eq 'snapstate' || $key eq 'pending';
2090             my $value = $cref->{$key};
2091             if ($key eq 'delete') {
2092                 die "propertry 'delete' is only allowed in [PENDING]\n"
2093                     if !$pending;
2094                 # fixme: check syntax?
2095                 next;
2096             }
2097             eval { $value = check_type($key, $value); };
2098             die "unable to parse value of '$key' - $@" if $@;
2099
2100             $cref->{$key} = $value;
2101
2102             if (!$snapname && is_valid_drivename($key)) {
2103                 my $drive = parse_drive($key, $value);
2104                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2105             }
2106         }
2107     };
2108
2109     &$cleanup_config($conf);
2110
2111     &$cleanup_config($conf->{pending}, 1);
2112
2113     foreach my $snapname (keys %{$conf->{snapshots}}) {
2114         die "internal error" if $snapname eq 'pending';
2115         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2116     }
2117
2118     # remove 'unusedX' settings if we re-add a volume
2119     foreach my $key (keys %$conf) {
2120         my $value = $conf->{$key};
2121         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2122             delete $conf->{$key};
2123         }
2124     }
2125
2126     my $generate_raw_config = sub {
2127         my ($conf, $pending) = @_;
2128
2129         my $raw = '';
2130
2131         # add description as comment to top of file
2132         if (defined(my $descr = $conf->{description})) {
2133             if ($descr) {
2134                 foreach my $cl (split(/\n/, $descr)) {
2135                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2136                 }
2137             } else {
2138                 $raw .= "#\n" if $pending;
2139             }
2140         }
2141
2142         foreach my $key (sort keys %$conf) {
2143             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2144             $raw .= "$key: $conf->{$key}\n";
2145         }
2146         return $raw;
2147     };
2148
2149     my $raw = &$generate_raw_config($conf);
2150
2151     if (scalar(keys %{$conf->{pending}})){
2152         $raw .= "\n[PENDING]\n";
2153         $raw .= &$generate_raw_config($conf->{pending}, 1);
2154     }
2155
2156     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2157         $raw .= "\n[$snapname]\n";
2158         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2159     }
2160
2161     return $raw;
2162 }
2163
2164 sub load_defaults {
2165
2166     my $res = {};
2167
2168     # we use static defaults from our JSON schema configuration
2169     foreach my $key (keys %$confdesc) {
2170         if (defined(my $default = $confdesc->{$key}->{default})) {
2171             $res->{$key} = $default;
2172         }
2173     }
2174
2175     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2176     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2177
2178     return $res;
2179 }
2180
2181 sub config_list {
2182     my $vmlist = PVE::Cluster::get_vmlist();
2183     my $res = {};
2184     return $res if !$vmlist || !$vmlist->{ids};
2185     my $ids = $vmlist->{ids};
2186
2187     foreach my $vmid (keys %$ids) {
2188         my $d = $ids->{$vmid};
2189         next if !$d->{node} || $d->{node} ne $nodename;
2190         next if !$d->{type} || $d->{type} ne 'qemu';
2191         $res->{$vmid}->{exists} = 1;
2192     }
2193     return $res;
2194 }
2195
2196 # test if VM uses local resources (to prevent migration)
2197 sub check_local_resources {
2198     my ($conf, $noerr) = @_;
2199
2200     my $loc_res = 0;
2201
2202     $loc_res = 1 if $conf->{hostusb}; # old syntax
2203     $loc_res = 1 if $conf->{hostpci}; # old syntax
2204
2205     foreach my $k (keys %$conf) {
2206         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2207         # sockets are safe: they will recreated be on the target side post-migrate
2208         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2209         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2210     }
2211
2212     die "VM uses local resources\n" if $loc_res && !$noerr;
2213
2214     return $loc_res;
2215 }
2216
2217 # check if used storages are available on all nodes (use by migrate)
2218 sub check_storage_availability {
2219     my ($storecfg, $conf, $node) = @_;
2220
2221     foreach_drive($conf, sub {
2222         my ($ds, $drive) = @_;
2223
2224         my $volid = $drive->{file};
2225         return if !$volid;
2226
2227         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2228         return if !$sid;
2229
2230         # check if storage is available on both nodes
2231         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2232         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2233    });
2234 }
2235
2236 # list nodes where all VM images are available (used by has_feature API)
2237 sub shared_nodes {
2238     my ($conf, $storecfg) = @_;
2239
2240     my $nodelist = PVE::Cluster::get_nodelist();
2241     my $nodehash = { map { $_ => 1 } @$nodelist };
2242     my $nodename = PVE::INotify::nodename();
2243
2244     foreach_drive($conf, sub {
2245         my ($ds, $drive) = @_;
2246
2247         my $volid = $drive->{file};
2248         return if !$volid;
2249
2250         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2251         if ($storeid) {
2252             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2253             if ($scfg->{disable}) {
2254                 $nodehash = {};
2255             } elsif (my $avail = $scfg->{nodes}) {
2256                 foreach my $node (keys %$nodehash) {
2257                     delete $nodehash->{$node} if !$avail->{$node};
2258                 }
2259             } elsif (!$scfg->{shared}) {
2260                 foreach my $node (keys %$nodehash) {
2261                     delete $nodehash->{$node} if $node ne $nodename
2262                 }
2263             }
2264         }
2265     });
2266
2267     return $nodehash
2268 }
2269
2270 sub check_cmdline {
2271     my ($pidfile, $pid) = @_;
2272
2273     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2274     if (defined($fh)) {
2275         my $line = <$fh>;
2276         $fh->close;
2277         return undef if !$line;
2278         my @param = split(/\0/, $line);
2279
2280         my $cmd = $param[0];
2281         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2282
2283         for (my $i = 0; $i < scalar (@param); $i++) {
2284             my $p = $param[$i];
2285             next if !$p;
2286             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2287                 my $p = $param[$i+1];
2288                 return 1 if $p && ($p eq $pidfile);
2289                 return undef;
2290             }
2291         }
2292     }
2293     return undef;
2294 }
2295
2296 sub check_running {
2297     my ($vmid, $nocheck, $node) = @_;
2298
2299     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2300
2301     die "unable to find configuration file for VM $vmid - no such machine\n"
2302         if !$nocheck && ! -f $filename;
2303
2304     my $pidfile = pidfile_name($vmid);
2305
2306     if (my $fd = IO::File->new("<$pidfile")) {
2307         my $st = stat($fd);
2308         my $line = <$fd>;
2309         close($fd);
2310
2311         my $mtime = $st->mtime;
2312         if ($mtime > time()) {
2313             warn "file '$filename' modified in future\n";
2314         }
2315
2316         if ($line =~ m/^(\d+)$/) {
2317             my $pid = $1;
2318             if (check_cmdline($pidfile, $pid)) {
2319                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2320                     return $pid;
2321                 }
2322             }
2323         }
2324     }
2325
2326     return undef;
2327 }
2328
2329 sub vzlist {
2330
2331     my $vzlist = config_list();
2332
2333     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2334
2335     while (defined(my $de = $fd->read)) {
2336         next if $de !~ m/^(\d+)\.pid$/;
2337         my $vmid = $1;
2338         next if !defined($vzlist->{$vmid});
2339         if (my $pid = check_running($vmid)) {
2340             $vzlist->{$vmid}->{pid} = $pid;
2341         }
2342     }
2343
2344     return $vzlist;
2345 }
2346
2347 sub disksize {
2348     my ($storecfg, $conf) = @_;
2349
2350     my $bootdisk = $conf->{bootdisk};
2351     return undef if !$bootdisk;
2352     return undef if !is_valid_drivename($bootdisk);
2353
2354     return undef if !$conf->{$bootdisk};
2355
2356     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2357     return undef if !defined($drive);
2358
2359     return undef if drive_is_cdrom($drive);
2360
2361     my $volid = $drive->{file};
2362     return undef if !$volid;
2363
2364     return $drive->{size};
2365 }
2366
2367 my $last_proc_pid_stat;
2368
2369 # get VM status information
2370 # This must be fast and should not block ($full == false)
2371 # We only query KVM using QMP if $full == true (this can be slow)
2372 sub vmstatus {
2373     my ($opt_vmid, $full) = @_;
2374
2375     my $res = {};
2376
2377     my $storecfg = PVE::Storage::config();
2378
2379     my $list = vzlist();
2380     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2381
2382     my $cpucount = $cpuinfo->{cpus} || 1;
2383
2384     foreach my $vmid (keys %$list) {
2385         next if $opt_vmid && ($vmid ne $opt_vmid);
2386
2387         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2388         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2389
2390         my $d = {};
2391         $d->{pid} = $list->{$vmid}->{pid};
2392
2393         # fixme: better status?
2394         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2395
2396         my $size = disksize($storecfg, $conf);
2397         if (defined($size)) {
2398             $d->{disk} = 0; # no info available
2399             $d->{maxdisk} = $size;
2400         } else {
2401             $d->{disk} = 0;
2402             $d->{maxdisk} = 0;
2403         }
2404
2405         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2406         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2407         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2408
2409         $d->{name} = $conf->{name} || "VM $vmid";
2410         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2411
2412         if ($conf->{balloon}) {
2413             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2414             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2415         }
2416
2417         $d->{uptime} = 0;
2418         $d->{cpu} = 0;
2419         $d->{mem} = 0;
2420
2421         $d->{netout} = 0;
2422         $d->{netin} = 0;
2423
2424         $d->{diskread} = 0;
2425         $d->{diskwrite} = 0;
2426
2427         $d->{template} = PVE::QemuConfig->is_template($conf);
2428
2429         $res->{$vmid} = $d;
2430     }
2431
2432     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2433     foreach my $dev (keys %$netdev) {
2434         next if $dev !~ m/^tap([1-9]\d*)i/;
2435         my $vmid = $1;
2436         my $d = $res->{$vmid};
2437         next if !$d;
2438
2439         $d->{netout} += $netdev->{$dev}->{receive};
2440         $d->{netin} += $netdev->{$dev}->{transmit};
2441
2442         if ($full) {
2443             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2444             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2445         }
2446
2447     }
2448
2449     my $ctime = gettimeofday;
2450
2451     foreach my $vmid (keys %$list) {
2452
2453         my $d = $res->{$vmid};
2454         my $pid = $d->{pid};
2455         next if !$pid;
2456
2457         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2458         next if !$pstat; # not running
2459
2460         my $used = $pstat->{utime} + $pstat->{stime};
2461
2462         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2463
2464         if ($pstat->{vsize}) {
2465             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2466         }
2467
2468         my $old = $last_proc_pid_stat->{$pid};
2469         if (!$old) {
2470             $last_proc_pid_stat->{$pid} = {
2471                 time => $ctime,
2472                 used => $used,
2473                 cpu => 0,
2474             };
2475             next;
2476         }
2477
2478         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2479
2480         if ($dtime > 1000) {
2481             my $dutime = $used -  $old->{used};
2482
2483             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2484             $last_proc_pid_stat->{$pid} = {
2485                 time => $ctime,
2486                 used => $used,
2487                 cpu => $d->{cpu},
2488             };
2489         } else {
2490             $d->{cpu} = $old->{cpu};
2491         }
2492     }
2493
2494     return $res if !$full;
2495
2496     my $qmpclient = PVE::QMPClient->new();
2497
2498     my $ballooncb = sub {
2499         my ($vmid, $resp) = @_;
2500
2501         my $info = $resp->{'return'};
2502         return if !$info->{max_mem};
2503
2504         my $d = $res->{$vmid};
2505
2506         # use memory assigned to VM
2507         $d->{maxmem} = $info->{max_mem};
2508         $d->{balloon} = $info->{actual};
2509
2510         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2511             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2512             $d->{freemem} = $info->{free_mem};
2513         }
2514
2515         $d->{ballooninfo} = $info;
2516     };
2517
2518     my $blockstatscb = sub {
2519         my ($vmid, $resp) = @_;
2520         my $data = $resp->{'return'} || [];
2521         my $totalrdbytes = 0;
2522         my $totalwrbytes = 0;
2523
2524         for my $blockstat (@$data) {
2525             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2526             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2527
2528             $blockstat->{device} =~ s/drive-//;
2529             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2530         }
2531         $res->{$vmid}->{diskread} = $totalrdbytes;
2532         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2533     };
2534
2535     my $statuscb = sub {
2536         my ($vmid, $resp) = @_;
2537
2538         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2539         # this fails if ballon driver is not loaded, so this must be
2540         # the last commnand (following command are aborted if this fails).
2541         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2542
2543         my $status = 'unknown';
2544         if (!defined($status = $resp->{'return'}->{status})) {
2545             warn "unable to get VM status\n";
2546             return;
2547         }
2548
2549         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2550     };
2551
2552     foreach my $vmid (keys %$list) {
2553         next if $opt_vmid && ($vmid ne $opt_vmid);
2554         next if !$res->{$vmid}->{pid}; # not running
2555         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2556     }
2557
2558     $qmpclient->queue_execute(undef, 1);
2559
2560     foreach my $vmid (keys %$list) {
2561         next if $opt_vmid && ($vmid ne $opt_vmid);
2562         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2563     }
2564
2565     return $res;
2566 }
2567
2568 sub foreach_dimm {
2569     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2570
2571     my $dimm_id = 0;
2572     my $current_size = 1024;
2573     my $dimm_size = 512;
2574     return if $current_size == $memory;
2575
2576     for (my $j = 0; $j < 8; $j++) {
2577         for (my $i = 0; $i < 32; $i++) {
2578             my $name = "dimm${dimm_id}";
2579             $dimm_id++;
2580             my $numanode = $i % $sockets;
2581             $current_size += $dimm_size;
2582             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2583             return  $current_size if $current_size >= $memory;
2584         }
2585         $dimm_size *= 2;
2586     }
2587 }
2588
2589 sub foreach_reverse_dimm {
2590     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2591
2592     my $dimm_id = 253;
2593     my $current_size = 4177920;
2594     my $dimm_size = 65536;
2595     return if $current_size == $memory;
2596
2597     for (my $j = 0; $j < 8; $j++) {
2598         for (my $i = 0; $i < 32; $i++) {
2599             my $name = "dimm${dimm_id}";
2600             $dimm_id--;
2601             my $numanode = $i % $sockets;
2602             $current_size -= $dimm_size;
2603             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2604             return  $current_size if $current_size <= $memory;
2605         }
2606         $dimm_size /= 2;
2607     }
2608 }
2609
2610 sub foreach_drive {
2611     my ($conf, $func) = @_;
2612
2613     foreach my $ds (valid_drive_names()) {
2614         next if !defined($conf->{$ds});
2615
2616         my $drive = parse_drive($ds, $conf->{$ds});
2617         next if !$drive;
2618
2619         &$func($ds, $drive);
2620     }
2621 }
2622
2623 sub foreach_volid {
2624     my ($conf, $func) = @_;
2625
2626     my $volhash = {};
2627
2628     my $test_volid = sub {
2629         my ($volid, $is_cdrom) = @_;
2630
2631         return if !$volid;
2632
2633         $volhash->{$volid} = $is_cdrom || 0;
2634     };
2635
2636     foreach_drive($conf, sub {
2637         my ($ds, $drive) = @_;
2638         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2639     });
2640
2641     foreach my $snapname (keys %{$conf->{snapshots}}) {
2642         my $snap = $conf->{snapshots}->{$snapname};
2643         &$test_volid($snap->{vmstate}, 0);
2644         foreach_drive($snap, sub {
2645             my ($ds, $drive) = @_;
2646             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2647         });
2648     }
2649
2650     foreach my $volid (keys %$volhash) {
2651         &$func($volid, $volhash->{$volid});
2652     }
2653 }
2654
2655 sub vga_conf_has_spice {
2656     my ($vga) = @_;
2657
2658     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2659
2660     return $1 || 1;
2661 }
2662
2663 sub config_to_command {
2664     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2665
2666     my $cmd = [];
2667     my $globalFlags = [];
2668     my $machineFlags = [];
2669     my $rtcFlags = [];
2670     my $cpuFlags = [];
2671     my $devices = [];
2672     my $pciaddr = '';
2673     my $bridges = {};
2674     my $kvmver = kvm_user_version();
2675     my $vernum = 0; # unknown
2676     my $ostype = $conf->{ostype};
2677     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2678         $vernum = $1*1000000+$2*1000;
2679     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2680         $vernum = $1*1000000+$2*1000+$3;
2681     }
2682
2683     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2684
2685     my $have_ovz = -f '/proc/vz/vestat';
2686
2687     my $q35 = machine_type_is_q35($conf);
2688     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2689     my $machine_type = $forcemachine || $conf->{machine};
2690     my $use_old_bios_files = undef;
2691     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2692
2693     my $cpuunits = defined($conf->{cpuunits}) ?
2694             $conf->{cpuunits} : $defaults->{cpuunits};
2695
2696     push @$cmd, '/usr/bin/systemd-run';
2697     push @$cmd, '--scope';
2698     push @$cmd, '--slice', "qemu";
2699     push @$cmd, '--unit', $vmid;
2700     # set KillMode=none, so that systemd don't kill those scopes
2701     # at shutdown (pve-manager service should stop the VMs instead)
2702     push @$cmd, '-p', "KillMode=none";
2703     push @$cmd, '-p', "CPUShares=$cpuunits";
2704     if ($conf->{cpulimit}) {
2705         my $cpulimit = int($conf->{cpulimit} * 100);
2706         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2707     }
2708
2709     push @$cmd, '/usr/bin/kvm';
2710
2711     push @$cmd, '-id', $vmid;
2712
2713     my $use_virtio = 0;
2714
2715     my $qmpsocket = qmp_socket($vmid);
2716     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2717     push @$cmd, '-mon', "chardev=qmp,mode=control";
2718
2719
2720     push @$cmd, '-pidfile' , pidfile_name($vmid);
2721
2722     push @$cmd, '-daemonize';
2723
2724     if ($conf->{smbios1}) {
2725         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2726     }
2727
2728     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2729         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2730         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2731         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2732         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2733         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2734         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2735     }
2736
2737     if ($q35) {
2738         # the q35 chipset support native usb2, so we enable usb controller
2739         # by default for this machine type
2740         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2741     } else {
2742         $pciaddr = print_pci_addr("piix3", $bridges);
2743         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2744
2745         my $use_usb2 = 0;
2746         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2747             next if !$conf->{"usb$i"};
2748             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2749             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2750             $use_usb2 = 1;
2751         }
2752         # include usb device config
2753         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2754     }
2755
2756     # add usb3 controller if needed
2757
2758     my $use_usb3 = 0;
2759     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2760         next if !$conf->{"usb$i"};
2761         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2762         next if !$d || !$d->{usb3};
2763         $use_usb3 = 1;
2764     }
2765
2766     $pciaddr = print_pci_addr("xhci", $bridges);
2767     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2768
2769     my $vga = $conf->{vga};
2770
2771     my $qxlnum = vga_conf_has_spice($vga);
2772     $vga = 'qxl' if $qxlnum;
2773
2774     if (!$vga) {
2775         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2776                                 $conf->{ostype} eq 'win7' ||
2777                                 $conf->{ostype} eq 'w2k8')) {
2778             $vga = 'std';
2779         } else {
2780             $vga = 'cirrus';
2781         }
2782     }
2783
2784     # enable absolute mouse coordinates (needed by vnc)
2785     my $tablet;
2786     if (defined($conf->{tablet})) {
2787         $tablet = $conf->{tablet};
2788     } else {
2789         $tablet = $defaults->{tablet};
2790         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2791         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2792     }
2793
2794     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2795
2796     my $kvm_off = 0;
2797     # host pci devices
2798     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2799         my $d = parse_hostpci($conf->{"hostpci$i"});
2800         next if !$d;
2801
2802         my $pcie = $d->{pcie};
2803         if($pcie){
2804             die "q35 machine model is not enabled" if !$q35;
2805             $pciaddr = print_pcie_addr("hostpci$i");
2806         }else{
2807             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2808         }
2809
2810         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2811         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2812         if ($xvga && $xvga ne '') {
2813             $kvm_off = 1;
2814             $vga = 'none';
2815             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2816                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2817             }
2818             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2819                 $xvga = "";
2820             }
2821         }
2822         my $pcidevices = $d->{pciid};
2823         my $multifunction = 1 if @$pcidevices > 1;
2824
2825         my $j=0;
2826         foreach my $pcidevice (@$pcidevices) {
2827
2828             my $id = "hostpci$i";
2829             $id .= ".$j" if $multifunction;
2830             my $addr = $pciaddr;
2831             $addr .= ".$j" if $multifunction;
2832             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2833
2834             if($j == 0){
2835                 $devicestr .= "$rombar$xvga";
2836                 $devicestr .= ",multifunction=on" if $multifunction;
2837             }
2838
2839             push @$devices, '-device', $devicestr;
2840             $j++;
2841         }
2842     }
2843
2844     # usb devices
2845     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2846         next if !$conf->{"usb$i"};
2847         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2848         next if !$d;
2849
2850         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2851         my $usbbus = '';
2852         if (defined($d->{usb3}) && $d->{usb3}) {
2853             $usbbus = ',bus=xhci.0';
2854         }
2855
2856         if (defined($d->{host})) {
2857             $d = parse_usb_device($d->{host});
2858             if (defined($d->{vendorid}) && defined($d->{productid})) {
2859                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2860             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2861                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2862             } elsif (defined($d->{spice}) && $d->{spice}) {
2863                 # usb redir support for spice, currently no usb3
2864                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2865                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2866             }
2867         }
2868     }
2869
2870     # serial devices
2871     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2872         if (my $path = $conf->{"serial$i"}) {
2873             if ($path eq 'socket') {
2874                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2875                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2876                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2877             } else {
2878                 die "no such serial device\n" if ! -c $path;
2879                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2880                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2881             }
2882         }
2883     }
2884
2885     # parallel devices
2886     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2887         if (my $path = $conf->{"parallel$i"}) {
2888             die "no such parallel device\n" if ! -c $path;
2889             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2890             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2891             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2892         }
2893     }
2894
2895     my $vmname = $conf->{name} || "vm$vmid";
2896
2897     push @$cmd, '-name', $vmname;
2898
2899     my $sockets = 1;
2900     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2901     $sockets = $conf->{sockets} if  $conf->{sockets};
2902
2903     my $cores = $conf->{cores} || 1;
2904
2905     my $maxcpus = $sockets * $cores;
2906
2907     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2908
2909     my $allowed_vcpus = $cpuinfo->{cpus};
2910
2911     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2912         if ($allowed_vcpus < $maxcpus);
2913
2914     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2915
2916     push @$cmd, '-nodefaults';
2917
2918     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2919
2920     my $bootindex_hash = {};
2921     my $i = 1;
2922     foreach my $o (split(//, $bootorder)) {
2923         $bootindex_hash->{$o} = $i*100;
2924         $i++;
2925     }
2926
2927     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2928
2929     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2930
2931     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2932
2933     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2934
2935     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2936         my $socket = vnc_socket($vmid);
2937         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2938     } else {
2939         push @$cmd, '-nographic';
2940     }
2941
2942     # time drift fix
2943     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2944
2945     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2946     my $useLocaltime = $conf->{localtime};
2947
2948     if ($ostype) {
2949         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2950
2951         if ($ostype =~ m/^w/) { # windows
2952             $useLocaltime = 1 if !defined($conf->{localtime});
2953
2954             # use time drift fix when acpi is enabled
2955             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2956                 $tdf = 1 if !defined($conf->{tdf});
2957             }
2958         }
2959
2960         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2961             $ostype eq 'wvista') {
2962             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2963             push @$cmd, '-no-hpet';
2964             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2965                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2966                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2967                 push @$cpuFlags , 'hv_time' if !$nokvm;
2968
2969             } else {
2970                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2971             }
2972         }
2973
2974         if ($ostype eq 'win7' || $ostype eq 'win8') {
2975             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2976         }
2977     }
2978
2979     push @$rtcFlags, 'driftfix=slew' if $tdf;
2980
2981     if ($nokvm) {
2982         push @$machineFlags, 'accel=tcg';
2983     } else {
2984         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2985     }
2986
2987     if ($machine_type) {
2988         push @$machineFlags, "type=${machine_type}";
2989     }
2990
2991     if ($conf->{startdate}) {
2992         push @$rtcFlags, "base=$conf->{startdate}";
2993     } elsif ($useLocaltime) {
2994         push @$rtcFlags, 'base=localtime';
2995     }
2996
2997     my $cpu = $nokvm ? "qemu64" : "kvm64";
2998     if (my $cputype = $conf->{cpu}) {
2999         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3000             or die "Cannot parse cpu description: $cputype\n";
3001         $cpu = $cpuconf->{cputype};
3002         $kvm_off = 1 if $cpuconf->{hidden};
3003     }
3004
3005     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3006
3007     push @$cpuFlags , '-x2apic'
3008         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3009
3010     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3011
3012     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3013
3014     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3015
3016         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3017         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3018     }
3019
3020     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3021
3022     push @$cpuFlags, 'kvm=off' if $kvm_off;
3023
3024     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3025
3026     push @$cmd, '-cpu', $cpu;
3027
3028     my $memory = $conf->{memory} || $defaults->{memory};
3029     my $static_memory = 0;
3030     my $dimm_memory = 0;
3031
3032     if ($hotplug_features->{memory}) {
3033         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3034         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3035         $static_memory = $STATICMEM;
3036         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3037         $dimm_memory = $memory - $static_memory;
3038         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3039
3040     } else {
3041
3042         $static_memory = $memory;
3043         push @$cmd, '-m', $static_memory;
3044     }
3045
3046     if ($conf->{numa}) {
3047
3048         my $numa_totalmemory = undef;
3049         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3050             next if !$conf->{"numa$i"};
3051             my $numa = parse_numa($conf->{"numa$i"});
3052             next if !$numa;
3053             # memory
3054             die "missing numa node$i memory value\n" if !$numa->{memory};
3055             my $numa_memory = $numa->{memory};
3056             $numa_totalmemory += $numa_memory;
3057             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3058
3059             # cpus
3060             my $cpus_start = $numa->{cpus}->{start};
3061             die "missing numa node$i cpus\n" if !defined($cpus_start);
3062             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3063             my $cpus = $cpus_start;
3064             if (defined($cpus_end)) {
3065                 $cpus .= "-$cpus_end";
3066                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3067             }
3068
3069             # hostnodes
3070             my $hostnodes_start = $numa->{hostnodes}->{start};
3071             if (defined($hostnodes_start)) {
3072                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3073                 my $hostnodes = $hostnodes_start;
3074                 if (defined($hostnodes_end)) {
3075                     $hostnodes .= "-$hostnodes_end";
3076                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3077                 }
3078
3079                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3080                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3081                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3082                 }
3083
3084                 # policy
3085                 my $policy = $numa->{policy};
3086                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3087                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3088             }
3089
3090             push @$cmd, '-object', $numa_object;
3091             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3092         }
3093
3094         die "total memory for NUMA nodes must be equal to vm static memory\n"
3095             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3096
3097         #if no custom tology, we split memory and cores across numa nodes
3098         if(!$numa_totalmemory) {
3099
3100             my $numa_memory = ($static_memory / $sockets) . "M";
3101
3102             for (my $i = 0; $i < $sockets; $i++)  {
3103
3104                 my $cpustart = ($cores * $i);
3105                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3106                 my $cpus = $cpustart;
3107                 $cpus .= "-$cpuend" if $cpuend;
3108
3109                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3110                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3111             }
3112         }
3113     }
3114
3115     if ($hotplug_features->{memory}) {
3116         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3117             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3118             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3119             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3120
3121             #if dimm_memory is not aligned to dimm map
3122             if($current_size > $memory) {
3123                  $conf->{memory} = $current_size;
3124                  PVE::QemuConfig->write_config($vmid, $conf);
3125             }
3126         });
3127     }
3128
3129     push @$cmd, '-S' if $conf->{freeze};
3130
3131     # set keyboard layout
3132     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3133     push @$cmd, '-k', $kb if $kb;
3134
3135     # enable sound
3136     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3137     #push @$cmd, '-soundhw', 'es1370';
3138     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3139
3140     if($conf->{agent}) {
3141         my $qgasocket = qmp_socket($vmid, 1);
3142         my $pciaddr = print_pci_addr("qga0", $bridges);
3143         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3144         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3145         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3146     }
3147
3148     my $spice_port;
3149
3150     if ($qxlnum) {
3151         if ($qxlnum > 1) {
3152             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3153                 for(my $i = 1; $i < $qxlnum; $i++){
3154                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3155                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3156                 }
3157             } else {
3158                 # assume other OS works like Linux
3159                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3160                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3161             }
3162         }
3163
3164         my $pciaddr = print_pci_addr("spice", $bridges);
3165
3166         my $nodename = PVE::INotify::nodename();
3167         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3168         $spice_port = PVE::Tools::next_spice_port($pfamily);
3169
3170         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3171
3172         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3173         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3174         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3175     }
3176
3177     # enable balloon by default, unless explicitly disabled
3178     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3179         $pciaddr = print_pci_addr("balloon0", $bridges);
3180         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3181     }
3182
3183     if ($conf->{watchdog}) {
3184         my $wdopts = parse_watchdog($conf->{watchdog});
3185         $pciaddr = print_pci_addr("watchdog", $bridges);
3186         my $watchdog = $wdopts->{model} || 'i6300esb';
3187         push @$devices, '-device', "$watchdog$pciaddr";
3188         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3189     }
3190
3191     my $vollist = [];
3192     my $scsicontroller = {};
3193     my $ahcicontroller = {};
3194     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3195
3196     # Add iscsi initiator name if available
3197     if (my $initiator = get_initiator_name()) {
3198         push @$devices, '-iscsi', "initiator-name=$initiator";
3199     }
3200
3201     foreach_drive($conf, sub {
3202         my ($ds, $drive) = @_;
3203
3204         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3205             push @$vollist, $drive->{file};
3206         }
3207
3208         $use_virtio = 1 if $ds =~ m/^virtio/;
3209
3210         if (drive_is_cdrom ($drive)) {
3211             if ($bootindex_hash->{d}) {
3212                 $drive->{bootindex} = $bootindex_hash->{d};
3213                 $bootindex_hash->{d} += 1;
3214             }
3215         } else {
3216             if ($bootindex_hash->{c}) {
3217                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3218                 $bootindex_hash->{c} += 1;
3219             }
3220         }
3221
3222         if($drive->{interface} eq 'virtio'){
3223            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3224         }
3225
3226         if ($drive->{interface} eq 'scsi') {
3227
3228             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3229
3230             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3231             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3232
3233             my $iothread = '';
3234             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3235                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3236                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3237             }
3238
3239             my $queues = '';
3240             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3241                 $queues = ",num_queues=$drive->{queues}";
3242             }
3243
3244             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3245             $scsicontroller->{$controller}=1;
3246         }
3247
3248         if ($drive->{interface} eq 'sata') {
3249            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3250            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3251            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3252            $ahcicontroller->{$controller}=1;
3253         }
3254
3255         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3256         push @$devices, '-drive',$drive_cmd;
3257         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3258     });
3259
3260     for (my $i = 0; $i < $MAX_NETS; $i++) {
3261          next if !$conf->{"net$i"};
3262          my $d = parse_net($conf->{"net$i"});
3263          next if !$d;
3264
3265          $use_virtio = 1 if $d->{model} eq 'virtio';
3266
3267          if ($bootindex_hash->{n}) {
3268             $d->{bootindex} = $bootindex_hash->{n};
3269             $bootindex_hash->{n} += 1;
3270          }
3271
3272          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3273          push @$devices, '-netdev', $netdevfull;
3274
3275          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3276          push @$devices, '-device', $netdevicefull;
3277     }
3278
3279     if (!$q35) {
3280         # add pci bridges
3281         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3282            $bridges->{1} = 1;
3283            $bridges->{2} = 1;
3284         }
3285
3286         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3287
3288         while (my ($k, $v) = each %$bridges) {
3289             $pciaddr = print_pci_addr("pci.$k");
3290             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3291         }
3292     }
3293
3294     # add custom args
3295     if ($conf->{args}) {
3296         my $aa = PVE::Tools::split_args($conf->{args});
3297         push @$cmd, @$aa;
3298     }
3299
3300     push @$cmd, @$devices;
3301     push @$cmd, '-rtc', join(',', @$rtcFlags)
3302         if scalar(@$rtcFlags);
3303     push @$cmd, '-machine', join(',', @$machineFlags)
3304         if scalar(@$machineFlags);
3305     push @$cmd, '-global', join(',', @$globalFlags)
3306         if scalar(@$globalFlags);
3307
3308     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3309 }
3310
3311 sub vnc_socket {
3312     my ($vmid) = @_;
3313     return "${var_run_tmpdir}/$vmid.vnc";
3314 }
3315
3316 sub spice_port {
3317     my ($vmid) = @_;
3318
3319     my $res = vm_mon_cmd($vmid, 'query-spice');
3320
3321     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3322 }
3323
3324 sub qmp_socket {
3325     my ($vmid, $qga) = @_;
3326     my $sockettype = $qga ? 'qga' : 'qmp';
3327     return "${var_run_tmpdir}/$vmid.$sockettype";
3328 }
3329
3330 sub pidfile_name {
3331     my ($vmid) = @_;
3332     return "${var_run_tmpdir}/$vmid.pid";
3333 }
3334
3335 sub vm_devices_list {
3336     my ($vmid) = @_;
3337
3338     my $res = vm_mon_cmd($vmid, 'query-pci');
3339     my $devices = {};
3340     foreach my $pcibus (@$res) {
3341         foreach my $device (@{$pcibus->{devices}}) {
3342             next if !$device->{'qdev_id'};
3343             if ($device->{'pci_bridge'}) {
3344                 $devices->{$device->{'qdev_id'}} = 1;
3345                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3346                     next if !$bridge_device->{'qdev_id'};
3347                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3348                     $devices->{$device->{'qdev_id'}}++;
3349                 }
3350             } else {
3351                 $devices->{$device->{'qdev_id'}} = 1;
3352             }
3353         }
3354     }
3355
3356     my $resblock = vm_mon_cmd($vmid, 'query-block');
3357     foreach my $block (@$resblock) {
3358         if($block->{device} =~ m/^drive-(\S+)/){
3359                 $devices->{$1} = 1;
3360         }
3361     }
3362
3363     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3364     foreach my $mice (@$resmice) {
3365         if ($mice->{name} eq 'QEMU HID Tablet') {
3366             $devices->{tablet} = 1;
3367             last;
3368         }
3369     }
3370
3371     return $devices;
3372 }
3373
3374 sub vm_deviceplug {
3375     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3376
3377     my $q35 = machine_type_is_q35($conf);
3378
3379     my $devices_list = vm_devices_list($vmid);
3380     return 1 if defined($devices_list->{$deviceid});
3381
3382     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3383
3384     if ($deviceid eq 'tablet') {
3385
3386         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3387
3388     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3389
3390         qemu_iothread_add($vmid, $deviceid, $device);
3391
3392         qemu_driveadd($storecfg, $vmid, $device);
3393         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3394
3395         qemu_deviceadd($vmid, $devicefull);
3396         eval { qemu_deviceaddverify($vmid, $deviceid); };
3397         if (my $err = $@) {
3398             eval { qemu_drivedel($vmid, $deviceid); };
3399             warn $@ if $@;
3400             die $err;
3401         }
3402
3403     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3404
3405
3406         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3407         my $pciaddr = print_pci_addr($deviceid);
3408         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3409
3410         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3411
3412         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3413             qemu_iothread_add($vmid, $deviceid, $device);
3414             $devicefull .= ",iothread=iothread-$deviceid";
3415         }
3416
3417         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3418             $devicefull .= ",num_queues=$device->{queues}";
3419         }
3420
3421         qemu_deviceadd($vmid, $devicefull);
3422         qemu_deviceaddverify($vmid, $deviceid);
3423
3424     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3425
3426         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3427         qemu_driveadd($storecfg, $vmid, $device);
3428
3429         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3430         eval { qemu_deviceadd($vmid, $devicefull); };
3431         if (my $err = $@) {
3432             eval { qemu_drivedel($vmid, $deviceid); };
3433             warn $@ if $@;
3434             die $err;
3435         }
3436
3437     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3438
3439         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3440
3441         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3442         my $use_old_bios_files = undef;
3443         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3444
3445         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3446         qemu_deviceadd($vmid, $netdevicefull);
3447         eval { qemu_deviceaddverify($vmid, $deviceid); };
3448         if (my $err = $@) {
3449             eval { qemu_netdevdel($vmid, $deviceid); };
3450             warn $@ if $@;
3451             die $err;
3452         }
3453
3454     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3455
3456         my $bridgeid = $2;
3457         my $pciaddr = print_pci_addr($deviceid);
3458         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3459
3460         qemu_deviceadd($vmid, $devicefull);
3461         qemu_deviceaddverify($vmid, $deviceid);
3462
3463     } else {
3464         die "can't hotplug device '$deviceid'\n";
3465     }
3466
3467     return 1;
3468 }
3469
3470 # fixme: this should raise exceptions on error!
3471 sub vm_deviceunplug {
3472     my ($vmid, $conf, $deviceid) = @_;
3473
3474     my $devices_list = vm_devices_list($vmid);
3475     return 1 if !defined($devices_list->{$deviceid});
3476
3477     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3478
3479     if ($deviceid eq 'tablet') {
3480
3481         qemu_devicedel($vmid, $deviceid);
3482
3483     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3484
3485         qemu_devicedel($vmid, $deviceid);
3486         qemu_devicedelverify($vmid, $deviceid);
3487         qemu_drivedel($vmid, $deviceid);
3488         qemu_iothread_del($conf, $vmid, $deviceid);
3489
3490     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3491
3492         qemu_devicedel($vmid, $deviceid);
3493         qemu_devicedelverify($vmid, $deviceid);
3494         qemu_iothread_del($conf, $vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3497
3498         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3499         my $device = parse_drive($deviceid, $conf->{$deviceid});
3500         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3501
3502         qemu_devicedel($vmid, $deviceid);
3503         qemu_drivedel($vmid, $deviceid);
3504         qemu_deletescsihw($conf, $vmid, $deviceid);
3505
3506     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3507
3508         qemu_devicedel($vmid, $deviceid);
3509         qemu_devicedelverify($vmid, $deviceid);
3510         qemu_netdevdel($vmid, $deviceid);
3511
3512     } else {
3513         die "can't unplug device '$deviceid'\n";
3514     }
3515
3516     return 1;
3517 }
3518
3519 sub qemu_deviceadd {
3520     my ($vmid, $devicefull) = @_;
3521
3522     $devicefull = "driver=".$devicefull;
3523     my %options =  split(/[=,]/, $devicefull);
3524
3525     vm_mon_cmd($vmid, "device_add" , %options);
3526 }
3527
3528 sub qemu_devicedel {
3529     my ($vmid, $deviceid) = @_;
3530
3531     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3532 }
3533
3534 sub qemu_iothread_add {
3535     my($vmid, $deviceid, $device) = @_;
3536
3537     if ($device->{iothread}) {
3538         my $iothreads = vm_iothreads_list($vmid);
3539         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3540     }
3541 }
3542
3543 sub qemu_iothread_del {
3544     my($conf, $vmid, $deviceid) = @_;
3545
3546     my $device = parse_drive($deviceid, $conf->{$deviceid});
3547     if ($device->{iothread}) {
3548         my $iothreads = vm_iothreads_list($vmid);
3549         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3550     }
3551 }
3552
3553 sub qemu_objectadd {
3554     my($vmid, $objectid, $qomtype) = @_;
3555
3556     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3557
3558     return 1;
3559 }
3560
3561 sub qemu_objectdel {
3562     my($vmid, $objectid) = @_;
3563
3564     vm_mon_cmd($vmid, "object-del", id => $objectid);
3565
3566     return 1;
3567 }
3568
3569 sub qemu_driveadd {
3570     my ($storecfg, $vmid, $device) = @_;
3571
3572     my $drive = print_drive_full($storecfg, $vmid, $device);
3573     $drive =~ s/\\/\\\\/g;
3574     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3575
3576     # If the command succeeds qemu prints: "OK"
3577     return 1 if $ret =~ m/OK/s;
3578
3579     die "adding drive failed: $ret\n";
3580 }
3581
3582 sub qemu_drivedel {
3583     my($vmid, $deviceid) = @_;
3584
3585     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3586     $ret =~ s/^\s+//;
3587
3588     return 1 if $ret eq "";
3589
3590     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3591     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3592
3593     die "deleting drive $deviceid failed : $ret\n";
3594 }
3595
3596 sub qemu_deviceaddverify {
3597     my ($vmid, $deviceid) = @_;
3598
3599     for (my $i = 0; $i <= 5; $i++) {
3600          my $devices_list = vm_devices_list($vmid);
3601          return 1 if defined($devices_list->{$deviceid});
3602          sleep 1;
3603     }
3604
3605     die "error on hotplug device '$deviceid'\n";
3606 }
3607
3608
3609 sub qemu_devicedelverify {
3610     my ($vmid, $deviceid) = @_;
3611
3612     # need to verify that the device is correctly removed as device_del
3613     # is async and empty return is not reliable
3614
3615     for (my $i = 0; $i <= 5; $i++) {
3616          my $devices_list = vm_devices_list($vmid);
3617          return 1 if !defined($devices_list->{$deviceid});
3618          sleep 1;
3619     }
3620
3621     die "error on hot-unplugging device '$deviceid'\n";
3622 }
3623
3624 sub qemu_findorcreatescsihw {
3625     my ($storecfg, $conf, $vmid, $device) = @_;
3626
3627     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3628
3629     my $scsihwid="$controller_prefix$controller";
3630     my $devices_list = vm_devices_list($vmid);
3631
3632     if(!defined($devices_list->{$scsihwid})) {
3633         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3634     }
3635
3636     return 1;
3637 }
3638
3639 sub qemu_deletescsihw {
3640     my ($conf, $vmid, $opt) = @_;
3641
3642     my $device = parse_drive($opt, $conf->{$opt});
3643
3644     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3645         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3646         return 1;
3647     }
3648
3649     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3650
3651     my $devices_list = vm_devices_list($vmid);
3652     foreach my $opt (keys %{$devices_list}) {
3653         if (PVE::QemuServer::is_valid_drivename($opt)) {
3654             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3655             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3656                 return 1;
3657             }
3658         }
3659     }
3660
3661     my $scsihwid="scsihw$controller";
3662
3663     vm_deviceunplug($vmid, $conf, $scsihwid);
3664
3665     return 1;
3666 }
3667
3668 sub qemu_add_pci_bridge {
3669     my ($storecfg, $conf, $vmid, $device) = @_;
3670
3671     my $bridges = {};
3672
3673     my $bridgeid;
3674
3675     print_pci_addr($device, $bridges);
3676
3677     while (my ($k, $v) = each %$bridges) {
3678         $bridgeid = $k;
3679     }
3680     return 1 if !defined($bridgeid) || $bridgeid < 1;
3681
3682     my $bridge = "pci.$bridgeid";
3683     my $devices_list = vm_devices_list($vmid);
3684
3685     if (!defined($devices_list->{$bridge})) {
3686         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3687     }
3688
3689     return 1;
3690 }
3691
3692 sub qemu_set_link_status {
3693     my ($vmid, $device, $up) = @_;
3694
3695     vm_mon_cmd($vmid, "set_link", name => $device,
3696                up => $up ? JSON::true : JSON::false);
3697 }
3698
3699 sub qemu_netdevadd {
3700     my ($vmid, $conf, $device, $deviceid) = @_;
3701
3702     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3703     my %options =  split(/[=,]/, $netdev);
3704
3705     vm_mon_cmd($vmid, "netdev_add",  %options);
3706     return 1;
3707 }
3708
3709 sub qemu_netdevdel {
3710     my ($vmid, $deviceid) = @_;
3711
3712     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3713 }
3714
3715 sub qemu_cpu_hotplug {
3716     my ($vmid, $conf, $vcpus) = @_;
3717
3718     my $sockets = 1;
3719     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3720     $sockets = $conf->{sockets} if  $conf->{sockets};
3721     my $cores = $conf->{cores} || 1;
3722     my $maxcpus = $sockets * $cores;
3723
3724     $vcpus = $maxcpus if !$vcpus;
3725
3726     die "you can't add more vcpus than maxcpus\n"
3727         if $vcpus > $maxcpus;
3728
3729     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3730     die "online cpu unplug is not yet possible\n"
3731         if $vcpus < $currentvcpus;
3732
3733     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3734     die "vcpus in running vm is different than configuration\n"
3735         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3736
3737     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3738         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3739     }
3740 }
3741
3742 sub qemu_memory_hotplug {
3743     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3744
3745     return $value if !check_running($vmid);
3746
3747     my $memory = $conf->{memory} || $defaults->{memory};
3748     $value = $defaults->{memory} if !$value;
3749     return $value if $value == $memory;
3750
3751     my $static_memory = $STATICMEM;
3752     my $dimm_memory = $memory - $static_memory;
3753
3754     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3755     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3756
3757
3758     my $sockets = 1;
3759     $sockets = $conf->{sockets} if $conf->{sockets};
3760
3761     if($value > $memory) {
3762
3763         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3764             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3765
3766                 return if $current_size <= $conf->{memory};
3767
3768                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3769                 if (my $err = $@) {
3770                     eval { qemu_objectdel($vmid, "mem-$name"); };
3771                     die $err;
3772                 }
3773
3774                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3775                 if (my $err = $@) {
3776                     eval { qemu_objectdel($vmid, "mem-$name"); };
3777                     die $err;
3778                 }
3779                 #update conf after each succesful module hotplug
3780                 $conf->{memory} = $current_size;
3781                 PVE::QemuConfig->write_config($vmid, $conf);
3782         });
3783
3784     } else {
3785
3786         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3787             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3788
3789                 return if $current_size >= $conf->{memory};
3790                 print "try to unplug memory dimm $name\n";
3791
3792                 my $retry = 0;
3793                 while (1) {
3794                     eval { qemu_devicedel($vmid, $name) };
3795                     sleep 3;
3796                     my $dimm_list = qemu_dimm_list($vmid);
3797                     last if !$dimm_list->{$name};
3798                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3799                     $retry++;
3800                 }
3801
3802                 #update conf after each succesful module unplug
3803                 $conf->{memory} = $current_size;
3804
3805                 eval { qemu_objectdel($vmid, "mem-$name"); };
3806                 PVE::QemuConfig->write_config($vmid, $conf);
3807         });
3808     }
3809 }
3810
3811 sub qemu_dimm_list {
3812     my ($vmid) = @_;
3813
3814     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3815     my $dimms = {};
3816
3817     foreach my $dimm (@$dimmarray) {
3818
3819         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3820         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3821         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3822         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3823         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3824     }
3825     return $dimms;
3826 }
3827
3828 sub qemu_block_set_io_throttle {
3829     my ($vmid, $deviceid,
3830         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3831         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3832
3833     return if !check_running($vmid) ;
3834
3835     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3836         bps => int($bps),
3837         bps_rd => int($bps_rd),
3838         bps_wr => int($bps_wr),
3839         iops => int($iops),
3840         iops_rd => int($iops_rd),
3841         iops_wr => int($iops_wr),
3842         bps_max => int($bps_max),
3843         bps_rd_max => int($bps_rd_max),
3844         bps_wr_max => int($bps_wr_max),
3845         iops_max => int($iops_max),
3846         iops_rd_max => int($iops_rd_max),
3847         iops_wr_max => int($iops_wr_max)
3848     );
3849
3850 }
3851
3852 # old code, only used to shutdown old VM after update
3853 sub __read_avail {
3854     my ($fh, $timeout) = @_;
3855
3856     my $sel = new IO::Select;
3857     $sel->add($fh);
3858
3859     my $res = '';
3860     my $buf;
3861
3862     my @ready;
3863     while (scalar (@ready = $sel->can_read($timeout))) {
3864         my $count;
3865         if ($count = $fh->sysread($buf, 8192)) {
3866             if ($buf =~ /^(.*)\(qemu\) $/s) {
3867                 $res .= $1;
3868                 last;
3869             } else {
3870                 $res .= $buf;
3871             }
3872         } else {
3873             if (!defined($count)) {
3874                 die "$!\n";
3875             }
3876             last;
3877         }
3878     }
3879
3880     die "monitor read timeout\n" if !scalar(@ready);
3881
3882     return $res;
3883 }
3884
3885 # old code, only used to shutdown old VM after update
3886 sub vm_monitor_command {
3887     my ($vmid, $cmdstr, $nocheck) = @_;
3888
3889     my $res;
3890
3891     eval {
3892         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3893
3894         my $sname = "${var_run_tmpdir}/$vmid.mon";
3895
3896         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3897             die "unable to connect to VM $vmid socket - $!\n";
3898
3899         my $timeout = 3;
3900
3901         # hack: migrate sometime blocks the monitor (when migrate_downtime
3902         # is set)
3903         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3904             $timeout = 60*60; # 1 hour
3905         }
3906
3907         # read banner;
3908         my $data = __read_avail($sock, $timeout);
3909
3910         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3911             die "got unexpected qemu monitor banner\n";
3912         }
3913
3914         my $sel = new IO::Select;
3915         $sel->add($sock);
3916
3917         if (!scalar(my @ready = $sel->can_write($timeout))) {
3918             die "monitor write error - timeout";
3919         }
3920
3921         my $fullcmd = "$cmdstr\r";
3922
3923         # syslog('info', "VM $vmid monitor command: $cmdstr");
3924
3925         my $b;
3926         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3927             die "monitor write error - $!";
3928         }
3929
3930         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3931
3932         $timeout = 20;
3933
3934         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3935             $timeout = 60*60; # 1 hour
3936         } elsif ($cmdstr =~ m/^(eject|change)/) {
3937             $timeout = 60; # note: cdrom mount command is slow
3938         }
3939         if ($res = __read_avail($sock, $timeout)) {
3940
3941             my @lines = split("\r?\n", $res);
3942
3943             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3944
3945             $res = join("\n", @lines);
3946             $res .= "\n";
3947         }
3948     };
3949
3950     my $err = $@;
3951
3952     if ($err) {
3953         syslog("err", "VM $vmid monitor command failed - $err");
3954         die $err;
3955     }
3956
3957     return $res;
3958 }
3959
3960 sub qemu_block_resize {
3961     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3962
3963     my $running = check_running($vmid);
3964
3965     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3966
3967     return if !$running;
3968
3969     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3970
3971 }
3972
3973 sub qemu_volume_snapshot {
3974     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3975
3976     my $running = check_running($vmid);
3977
3978     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3979         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3980     } else {
3981         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3982     }
3983 }
3984
3985 sub qemu_volume_snapshot_delete {
3986     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3987
3988     my $running = check_running($vmid);
3989
3990     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3991
3992     return if !$running;
3993
3994     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3995 }
3996
3997 sub set_migration_caps {
3998     my ($vmid) = @_;
3999
4000     my $cap_ref = [];
4001
4002     my $enabled_cap = {
4003         "auto-converge" => 1,
4004         "xbzrle" => 1,
4005         "x-rdma-pin-all" => 0,
4006         "zero-blocks" => 0,
4007         "compress" => 0
4008     };
4009
4010     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4011
4012     for my $supported_capability (@$supported_capabilities) {
4013         push @$cap_ref, {
4014             capability => $supported_capability->{capability},
4015             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4016         };
4017     }
4018
4019     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4020 }
4021
4022 my $fast_plug_option = {
4023     'lock' => 1,
4024     'name' => 1,
4025     'onboot' => 1,
4026     'shares' => 1,
4027     'startup' => 1,
4028     'description' => 1,
4029 };
4030
4031 # hotplug changes in [PENDING]
4032 # $selection hash can be used to only apply specified options, for
4033 # example: { cores => 1 } (only apply changed 'cores')
4034 # $errors ref is used to return error messages
4035 sub vmconfig_hotplug_pending {
4036     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4037
4038     my $defaults = load_defaults();
4039
4040     # commit values which do not have any impact on running VM first
4041     # Note: those option cannot raise errors, we we do not care about
4042     # $selection and always apply them.
4043
4044     my $add_error = sub {
4045         my ($opt, $msg) = @_;
4046         $errors->{$opt} = "hotplug problem - $msg";
4047     };
4048
4049     my $changes = 0;
4050     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4051         if ($fast_plug_option->{$opt}) {
4052             $conf->{$opt} = $conf->{pending}->{$opt};
4053             delete $conf->{pending}->{$opt};
4054             $changes = 1;
4055         }
4056     }
4057
4058     if ($changes) {
4059         PVE::QemuConfig->write_config($vmid, $conf);
4060         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4061     }
4062
4063     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4064
4065     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4066     while (my ($opt, $force) = each %$pending_delete_hash) {
4067         next if $selection && !$selection->{$opt};
4068         eval {
4069             if ($opt eq 'hotplug') {
4070                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4071             } elsif ($opt eq 'tablet') {
4072                 die "skip\n" if !$hotplug_features->{usb};
4073                 if ($defaults->{tablet}) {
4074                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4075                 } else {
4076                     vm_deviceunplug($vmid, $conf, $opt);
4077                 }
4078             } elsif ($opt eq 'vcpus') {
4079                 die "skip\n" if !$hotplug_features->{cpu};
4080                 qemu_cpu_hotplug($vmid, $conf, undef);
4081             } elsif ($opt eq 'balloon') {
4082                 # enable balloon device is not hotpluggable
4083                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4084             } elsif ($fast_plug_option->{$opt}) {
4085                 # do nothing
4086             } elsif ($opt =~ m/^net(\d+)$/) {
4087                 die "skip\n" if !$hotplug_features->{network};
4088                 vm_deviceunplug($vmid, $conf, $opt);
4089             } elsif (is_valid_drivename($opt)) {
4090                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4091                 vm_deviceunplug($vmid, $conf, $opt);
4092                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4093             } elsif ($opt =~ m/^memory$/) {
4094                 die "skip\n" if !$hotplug_features->{memory};
4095                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4096             } elsif ($opt eq 'cpuunits') {
4097                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4098             } elsif ($opt eq 'cpulimit') {
4099                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4100             } else {
4101                 die "skip\n";
4102             }
4103         };
4104         if (my $err = $@) {
4105             &$add_error($opt, $err) if $err ne "skip\n";
4106         } else {
4107             # save new config if hotplug was successful
4108             delete $conf->{$opt};
4109             vmconfig_undelete_pending_option($conf, $opt);
4110             PVE::QemuConfig->write_config($vmid, $conf);
4111             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4112         }
4113     }
4114
4115     foreach my $opt (keys %{$conf->{pending}}) {
4116         next if $selection && !$selection->{$opt};
4117         my $value = $conf->{pending}->{$opt};
4118         eval {
4119             if ($opt eq 'hotplug') {
4120                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4121             } elsif ($opt eq 'tablet') {
4122                 die "skip\n" if !$hotplug_features->{usb};
4123                 if ($value == 1) {
4124                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4125                 } elsif ($value == 0) {
4126                     vm_deviceunplug($vmid, $conf, $opt);
4127                 }
4128             } elsif ($opt eq 'vcpus') {
4129                 die "skip\n" if !$hotplug_features->{cpu};
4130                 qemu_cpu_hotplug($vmid, $conf, $value);
4131             } elsif ($opt eq 'balloon') {
4132                 # enable/disable balloning device is not hotpluggable
4133                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4134                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4135                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4136
4137                 # allow manual ballooning if shares is set to zero
4138                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4139                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4140                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4141                 }
4142             } elsif ($opt =~ m/^net(\d+)$/) {
4143                 # some changes can be done without hotplug
4144                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4145                                     $vmid, $opt, $value);
4146             } elsif (is_valid_drivename($opt)) {
4147                 # some changes can be done without hotplug
4148                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4149                                      $vmid, $opt, $value, 1);
4150             } elsif ($opt =~ m/^memory$/) { #dimms
4151                 die "skip\n" if !$hotplug_features->{memory};
4152                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4153             } elsif ($opt eq 'cpuunits') {
4154                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4155             } elsif ($opt eq 'cpulimit') {
4156                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4157                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4158             } else {
4159                 die "skip\n";  # skip non-hot-pluggable options
4160             }
4161         };
4162         if (my $err = $@) {
4163             &$add_error($opt, $err) if $err ne "skip\n";
4164         } else {
4165             # save new config if hotplug was successful
4166             $conf->{$opt} = $value;
4167             delete $conf->{pending}->{$opt};
4168             PVE::QemuConfig->write_config($vmid, $conf);
4169             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4170         }
4171     }
4172 }
4173
4174 sub try_deallocate_drive {
4175     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4176
4177     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4178         my $volid = $drive->{file};
4179         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4180             my $sid = PVE::Storage::parse_volume_id($volid);
4181             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4182
4183             # check if the disk is really unused
4184             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4185                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4186             PVE::Storage::vdisk_free($storecfg, $volid);
4187             return 1;
4188         } else {
4189             # If vm is not owner of this disk remove from config
4190             return 1;
4191         }
4192     }
4193
4194     return undef;
4195 }
4196
4197 sub vmconfig_delete_or_detach_drive {
4198     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4199
4200     my $drive = parse_drive($opt, $conf->{$opt});
4201
4202     my $rpcenv = PVE::RPCEnvironment::get();
4203     my $authuser = $rpcenv->get_user();
4204
4205     if ($force) {
4206         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4207         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4208     } else {
4209         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4210     }
4211 }
4212
4213 sub vmconfig_apply_pending {
4214     my ($vmid, $conf, $storecfg) = @_;
4215
4216     # cold plug
4217
4218     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4219     while (my ($opt, $force) = each %$pending_delete_hash) {
4220         die "internal error" if $opt =~ m/^unused/;
4221         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4222         if (!defined($conf->{$opt})) {
4223             vmconfig_undelete_pending_option($conf, $opt);
4224             PVE::QemuConfig->write_config($vmid, $conf);
4225         } elsif (is_valid_drivename($opt)) {
4226             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4227             vmconfig_undelete_pending_option($conf, $opt);
4228             delete $conf->{$opt};
4229             PVE::QemuConfig->write_config($vmid, $conf);
4230         } else {
4231             vmconfig_undelete_pending_option($conf, $opt);
4232             delete $conf->{$opt};
4233             PVE::QemuConfig->write_config($vmid, $conf);
4234         }
4235     }
4236
4237     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4238
4239     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4240         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4241
4242         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4243             # skip if nothing changed
4244         } elsif (is_valid_drivename($opt)) {
4245             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4246                 if defined($conf->{$opt});
4247             $conf->{$opt} = $conf->{pending}->{$opt};
4248         } else {
4249             $conf->{$opt} = $conf->{pending}->{$opt};
4250         }
4251
4252         delete $conf->{pending}->{$opt};
4253         PVE::QemuConfig->write_config($vmid, $conf);
4254     }
4255 }
4256
4257 my $safe_num_ne = sub {
4258     my ($a, $b) = @_;
4259
4260     return 0 if !defined($a) && !defined($b);
4261     return 1 if !defined($a);
4262     return 1 if !defined($b);
4263
4264     return $a != $b;
4265 };
4266
4267 my $safe_string_ne = sub {
4268     my ($a, $b) = @_;
4269
4270     return 0 if !defined($a) && !defined($b);
4271     return 1 if !defined($a);
4272     return 1 if !defined($b);
4273
4274     return $a ne $b;
4275 };
4276
4277 sub vmconfig_update_net {
4278     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4279
4280     my $newnet = parse_net($value);
4281
4282     if ($conf->{$opt}) {
4283         my $oldnet = parse_net($conf->{$opt});
4284
4285         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4286             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4287             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4288             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4289
4290             # for non online change, we try to hot-unplug
4291             die "skip\n" if !$hotplug;
4292             vm_deviceunplug($vmid, $conf, $opt);
4293         } else {
4294
4295             die "internal error" if $opt !~ m/net(\d+)/;
4296             my $iface = "tap${vmid}i$1";
4297
4298             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4299                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4300             }
4301
4302             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4303                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4304                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4305                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4306                 PVE::Network::tap_unplug($iface);
4307                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4308             }
4309
4310             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4311                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4312             }
4313
4314             return 1;
4315         }
4316     }
4317
4318     if ($hotplug) {
4319         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4320     } else {
4321         die "skip\n";
4322     }
4323 }
4324
4325 sub vmconfig_update_disk {
4326     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4327
4328     # fixme: do we need force?
4329
4330     my $drive = parse_drive($opt, $value);
4331
4332     if ($conf->{$opt}) {
4333
4334         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4335
4336             my $media = $drive->{media} || 'disk';
4337             my $oldmedia = $old_drive->{media} || 'disk';
4338             die "unable to change media type\n" if $media ne $oldmedia;
4339
4340             if (!drive_is_cdrom($old_drive)) {
4341
4342                 if ($drive->{file} ne $old_drive->{file}) {
4343
4344                     die "skip\n" if !$hotplug;
4345
4346                     # unplug and register as unused
4347                     vm_deviceunplug($vmid, $conf, $opt);
4348                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4349
4350                 } else {
4351                     # update existing disk
4352
4353                     # skip non hotpluggable value
4354                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4355                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4356                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4357                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4358                         die "skip\n";
4359                     }
4360
4361                     # apply throttle
4362                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4363                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4364                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4365                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4366                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4367                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4368                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4369                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4370                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4371                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4372                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4373                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4374
4375                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4376                                                    ($drive->{mbps} || 0)*1024*1024,
4377                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4378                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4379                                                    $drive->{iops} || 0,
4380                                                    $drive->{iops_rd} || 0,
4381                                                    $drive->{iops_wr} || 0,
4382                                                    ($drive->{mbps_max} || 0)*1024*1024,
4383                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4384                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4385                                                    $drive->{iops_max} || 0,
4386                                                    $drive->{iops_rd_max} || 0,
4387                                                    $drive->{iops_wr_max} || 0);
4388
4389                     }
4390
4391                     return 1;
4392                 }
4393
4394             } else { # cdrom
4395
4396                 if ($drive->{file} eq 'none') {
4397                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4398                 } else {
4399                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4400                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4401                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4402                 }
4403
4404                 return 1;
4405             }
4406         }
4407     }
4408
4409     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4410     # hotplug new disks
4411     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4412     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4413 }
4414
4415 sub vm_start {
4416     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4417         $forcemachine, $spice_ticket) = @_;
4418
4419     PVE::QemuConfig->lock_config($vmid, sub {
4420         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4421
4422         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4423
4424         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4425
4426         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4427
4428         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4429             vmconfig_apply_pending($vmid, $conf, $storecfg);
4430             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4431         }
4432
4433         my $defaults = load_defaults();
4434
4435         # set environment variable useful inside network script
4436         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4437
4438         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4439
4440         my $migrate_port = 0;
4441         my $migrate_uri;
4442         if ($statefile) {
4443             if ($statefile eq 'tcp') {
4444                 my $localip = "localhost";
4445                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4446                 my $nodename = PVE::INotify::nodename();
4447                 if ($datacenterconf->{migration_unsecure}) {
4448                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4449                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4450                 }
4451                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4452                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4453                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4454                 push @$cmd, '-incoming', $migrate_uri;
4455                 push @$cmd, '-S';
4456             } else {
4457                 push @$cmd, '-loadstate', $statefile;
4458             }
4459         } elsif ($paused) {
4460             push @$cmd, '-S';
4461         }
4462
4463         # host pci devices
4464         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4465           my $d = parse_hostpci($conf->{"hostpci$i"});
4466           next if !$d;
4467           my $pcidevices = $d->{pciid};
4468           foreach my $pcidevice (@$pcidevices) {
4469                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4470
4471                 my $info = pci_device_info("0000:$pciid");
4472                 die "IOMMU not present\n" if !check_iommu_support();
4473                 die "no pci device info for device '$pciid'\n" if !$info;
4474                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4475                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4476           }
4477         }
4478
4479         PVE::Storage::activate_volumes($storecfg, $vollist);
4480
4481         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4482                     umask => 0077); };
4483
4484         if (my $err = $@) {
4485             # deactivate volumes if start fails
4486             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4487             die "start failed: $err";
4488         }
4489
4490         print "migration listens on $migrate_uri\n" if $migrate_uri;
4491
4492         if ($statefile && $statefile ne 'tcp')  {
4493             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4494             warn $@ if $@;
4495         }
4496
4497         if ($migratedfrom) {
4498
4499             eval {
4500                 set_migration_caps($vmid);
4501             };
4502             warn $@ if $@;
4503
4504             if ($spice_port) {
4505                 print "spice listens on port $spice_port\n";
4506                 if ($spice_ticket) {
4507                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4508                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4509                 }
4510             }
4511
4512         } else {
4513
4514             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4515                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4516                     if $conf->{balloon};
4517             }
4518
4519             foreach my $opt (keys %$conf) {
4520                 next if $opt !~  m/^net\d+$/;
4521                 my $nicconf = parse_net($conf->{$opt});
4522                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4523             }
4524         }
4525
4526         vm_mon_cmd_nocheck($vmid, 'qom-set',
4527                     path => "machine/peripheral/balloon0",
4528                     property => "guest-stats-polling-interval",
4529                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4530
4531     });
4532 }
4533
4534 sub vm_mon_cmd {
4535     my ($vmid, $execute, %params) = @_;
4536
4537     my $cmd = { execute => $execute, arguments => \%params };
4538     vm_qmp_command($vmid, $cmd);
4539 }
4540
4541 sub vm_mon_cmd_nocheck {
4542     my ($vmid, $execute, %params) = @_;
4543
4544     my $cmd = { execute => $execute, arguments => \%params };
4545     vm_qmp_command($vmid, $cmd, 1);
4546 }
4547
4548 sub vm_qmp_command {
4549     my ($vmid, $cmd, $nocheck) = @_;
4550
4551     my $res;
4552
4553     my $timeout;
4554     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4555         $timeout = $cmd->{arguments}->{timeout};
4556         delete $cmd->{arguments}->{timeout};
4557     }
4558
4559     eval {
4560         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4561         my $sname = qmp_socket($vmid);
4562         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4563             my $qmpclient = PVE::QMPClient->new();
4564
4565             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4566         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4567             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4568                 if scalar(%{$cmd->{arguments}});
4569             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4570         } else {
4571             die "unable to open monitor socket\n";
4572         }
4573     };
4574     if (my $err = $@) {
4575         syslog("err", "VM $vmid qmp command failed - $err");
4576         die $err;
4577     }
4578
4579     return $res;
4580 }
4581
4582 sub vm_human_monitor_command {
4583     my ($vmid, $cmdline) = @_;
4584
4585     my $res;
4586
4587     my $cmd = {
4588         execute => 'human-monitor-command',
4589         arguments => { 'command-line' => $cmdline},
4590     };
4591
4592     return vm_qmp_command($vmid, $cmd);
4593 }
4594
4595 sub vm_commandline {
4596     my ($storecfg, $vmid) = @_;
4597
4598     my $conf = PVE::QemuConfig->load_config($vmid);
4599
4600     my $defaults = load_defaults();
4601
4602     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4603
4604     return join(' ', @$cmd);
4605 }
4606
4607 sub vm_reset {
4608     my ($vmid, $skiplock) = @_;
4609
4610     PVE::QemuConfig->lock_config($vmid, sub {
4611
4612         my $conf = PVE::QemuConfig->load_config($vmid);
4613
4614         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4615
4616         vm_mon_cmd($vmid, "system_reset");
4617     });
4618 }
4619
4620 sub get_vm_volumes {
4621     my ($conf) = @_;
4622
4623     my $vollist = [];
4624     foreach_volid($conf, sub {
4625         my ($volid, $is_cdrom) = @_;
4626
4627         return if $volid =~ m|^/|;
4628
4629         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4630         return if !$sid;
4631
4632         push @$vollist, $volid;
4633     });
4634
4635     return $vollist;
4636 }
4637
4638 sub vm_stop_cleanup {
4639     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4640
4641     eval {
4642
4643         if (!$keepActive) {
4644             my $vollist = get_vm_volumes($conf);
4645             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4646         }
4647
4648         foreach my $ext (qw(mon qmp pid vnc qga)) {
4649             unlink "/var/run/qemu-server/${vmid}.$ext";
4650         }
4651
4652         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4653     };
4654     warn $@ if $@; # avoid errors - just warn
4655 }
4656
4657 # Note: use $nockeck to skip tests if VM configuration file exists.
4658 # We need that when migration VMs to other nodes (files already moved)
4659 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4660 sub vm_stop {
4661     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4662
4663     $force = 1 if !defined($force) && !$shutdown;
4664
4665     if ($migratedfrom){
4666         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4667         kill 15, $pid if $pid;
4668         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4669         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4670         return;
4671     }
4672
4673     PVE::QemuConfig->lock_config($vmid, sub {
4674
4675         my $pid = check_running($vmid, $nocheck);
4676         return if !$pid;
4677
4678         my $conf;
4679         if (!$nocheck) {
4680             $conf = PVE::QemuConfig->load_config($vmid);
4681             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4682             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4683                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4684                 $timeout = $opts->{down} if $opts->{down};
4685             }
4686         }
4687
4688         $timeout = 60 if !defined($timeout);
4689
4690         eval {
4691             if ($shutdown) {
4692                 if (defined($conf) && $conf->{agent}) {
4693                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4694                 } else {
4695                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4696                 }
4697             } else {
4698                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4699             }
4700         };
4701         my $err = $@;
4702
4703         if (!$err) {
4704             my $count = 0;
4705             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4706                 $count++;
4707                 sleep 1;
4708             }
4709
4710             if ($count >= $timeout) {
4711                 if ($force) {
4712                     warn "VM still running - terminating now with SIGTERM\n";
4713                     kill 15, $pid;
4714                 } else {
4715                     die "VM quit/powerdown failed - got timeout\n";
4716                 }
4717             } else {
4718                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4719                 return;
4720             }
4721         } else {
4722             if ($force) {
4723                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4724                 kill 15, $pid;
4725             } else {
4726                 die "VM quit/powerdown failed\n";
4727             }
4728         }
4729
4730         # wait again
4731         $timeout = 10;
4732
4733         my $count = 0;
4734         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4735             $count++;
4736             sleep 1;
4737         }
4738
4739         if ($count >= $timeout) {
4740             warn "VM still running - terminating now with SIGKILL\n";
4741             kill 9, $pid;
4742             sleep 1;
4743         }
4744
4745         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4746    });
4747 }
4748
4749 sub vm_suspend {
4750     my ($vmid, $skiplock) = @_;
4751
4752     PVE::QemuConfig->lock_config($vmid, sub {
4753
4754         my $conf = PVE::QemuConfig->load_config($vmid);
4755
4756         PVE::QemuConfig->check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4757
4758         vm_mon_cmd($vmid, "stop");
4759     });
4760 }
4761
4762 sub vm_resume {
4763     my ($vmid, $skiplock, $nocheck) = @_;
4764
4765     PVE::QemuConfig->lock_config($vmid, sub {
4766
4767         if (!$nocheck) {
4768
4769             my $conf = PVE::QemuConfig->load_config($vmid);
4770
4771             PVE::QemuConfig->check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4772
4773             vm_mon_cmd($vmid, "cont");
4774
4775         } else {
4776             vm_mon_cmd_nocheck($vmid, "cont");
4777         }
4778     });
4779 }
4780
4781 sub vm_sendkey {
4782     my ($vmid, $skiplock, $key) = @_;
4783
4784     PVE::QemuConfig->lock_config($vmid, sub {
4785
4786         my $conf = PVE::QemuConfig->load_config($vmid);
4787
4788         # there is no qmp command, so we use the human monitor command
4789         vm_human_monitor_command($vmid, "sendkey $key");
4790     });
4791 }
4792
4793 sub vm_destroy {
4794     my ($storecfg, $vmid, $skiplock) = @_;
4795
4796     PVE::QemuConfig->lock_config($vmid, sub {
4797
4798         my $conf = PVE::QemuConfig->load_config($vmid);
4799
4800         if (!check_running($vmid)) {
4801             destroy_vm($storecfg, $vmid, undef, $skiplock);
4802         } else {
4803             die "VM $vmid is running - destroy failed\n";
4804         }
4805     });
4806 }
4807
4808 # pci helpers
4809
4810 sub file_write {
4811     my ($filename, $buf) = @_;
4812
4813     my $fh = IO::File->new($filename, "w");
4814     return undef if !$fh;
4815
4816     my $res = print $fh $buf;
4817
4818     $fh->close();
4819
4820     return $res;
4821 }
4822
4823 sub pci_device_info {
4824     my ($name) = @_;
4825
4826     my $res;
4827
4828     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4829     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4830
4831     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4832     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4833
4834     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4835     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4836
4837     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4838     return undef if !defined($product) || $product !~ s/^0x//;
4839
4840     $res = {
4841         name => $name,
4842         vendor => $vendor,
4843         product => $product,
4844         domain => $domain,
4845         bus => $bus,
4846         slot => $slot,
4847         func => $func,
4848         irq => $irq,
4849         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4850     };
4851
4852     return $res;
4853 }
4854
4855 sub pci_dev_reset {
4856     my ($dev) = @_;
4857
4858     my $name = $dev->{name};
4859
4860     my $fn = "$pcisysfs/devices/$name/reset";
4861
4862     return file_write($fn, "1");
4863 }
4864
4865 sub pci_dev_bind_to_vfio {
4866     my ($dev) = @_;
4867
4868     my $name = $dev->{name};
4869
4870     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4871
4872     if (!-d $vfio_basedir) {
4873         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4874     }
4875     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4876
4877     my $testdir = "$vfio_basedir/$name";
4878     return 1 if -d $testdir;
4879
4880     my $data = "$dev->{vendor} $dev->{product}";
4881     return undef if !file_write("$vfio_basedir/new_id", $data);
4882
4883     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4884     if (!file_write($fn, $name)) {
4885         return undef if -f $fn;
4886     }
4887
4888     $fn = "$vfio_basedir/bind";
4889     if (! -d $testdir) {
4890         return undef if !file_write($fn, $name);
4891     }
4892
4893     return -d $testdir;
4894 }
4895
4896 sub pci_dev_group_bind_to_vfio {
4897     my ($pciid) = @_;
4898
4899     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4900
4901     if (!-d $vfio_basedir) {
4902         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4903     }
4904     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4905
4906     # get IOMMU group devices
4907     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4908       my @devs = grep /^0000:/, readdir($D);
4909     closedir($D);
4910
4911     foreach my $pciid (@devs) {
4912         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4913
4914         # pci bridges, switches or root ports are not supported
4915         # they have a pci_bus subdirectory so skip them
4916         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4917
4918         my $info = pci_device_info($1);
4919         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4920     }
4921
4922     return 1;
4923 }
4924
4925 sub print_pci_addr {
4926     my ($id, $bridges) = @_;
4927
4928     my $res = '';
4929     my $devices = {
4930         piix3 => { bus => 0, addr => 1 },
4931         #addr2 : first videocard
4932         balloon0 => { bus => 0, addr => 3 },
4933         watchdog => { bus => 0, addr => 4 },
4934         scsihw0 => { bus => 0, addr => 5 },
4935         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4936         scsihw1 => { bus => 0, addr => 6 },
4937         ahci0 => { bus => 0, addr => 7 },
4938         qga0 => { bus => 0, addr => 8 },
4939         spice => { bus => 0, addr => 9 },
4940         virtio0 => { bus => 0, addr => 10 },
4941         virtio1 => { bus => 0, addr => 11 },
4942         virtio2 => { bus => 0, addr => 12 },
4943         virtio3 => { bus => 0, addr => 13 },
4944         virtio4 => { bus => 0, addr => 14 },
4945         virtio5 => { bus => 0, addr => 15 },
4946         hostpci0 => { bus => 0, addr => 16 },
4947         hostpci1 => { bus => 0, addr => 17 },
4948         net0 => { bus => 0, addr => 18 },
4949         net1 => { bus => 0, addr => 19 },
4950         net2 => { bus => 0, addr => 20 },
4951         net3 => { bus => 0, addr => 21 },
4952         net4 => { bus => 0, addr => 22 },
4953         net5 => { bus => 0, addr => 23 },
4954         vga1 => { bus => 0, addr => 24 },
4955         vga2 => { bus => 0, addr => 25 },
4956         vga3 => { bus => 0, addr => 26 },
4957         hostpci2 => { bus => 0, addr => 27 },
4958         hostpci3 => { bus => 0, addr => 28 },
4959         #addr29 : usb-host (pve-usb.cfg)
4960         'pci.1' => { bus => 0, addr => 30 },
4961         'pci.2' => { bus => 0, addr => 31 },
4962         'net6' => { bus => 1, addr => 1 },
4963         'net7' => { bus => 1, addr => 2 },
4964         'net8' => { bus => 1, addr => 3 },
4965         'net9' => { bus => 1, addr => 4 },
4966         'net10' => { bus => 1, addr => 5 },
4967         'net11' => { bus => 1, addr => 6 },
4968         'net12' => { bus => 1, addr => 7 },
4969         'net13' => { bus => 1, addr => 8 },
4970         'net14' => { bus => 1, addr => 9 },
4971         'net15' => { bus => 1, addr => 10 },
4972         'net16' => { bus => 1, addr => 11 },
4973         'net17' => { bus => 1, addr => 12 },
4974         'net18' => { bus => 1, addr => 13 },
4975         'net19' => { bus => 1, addr => 14 },
4976         'net20' => { bus => 1, addr => 15 },
4977         'net21' => { bus => 1, addr => 16 },
4978         'net22' => { bus => 1, addr => 17 },
4979         'net23' => { bus => 1, addr => 18 },
4980         'net24' => { bus => 1, addr => 19 },
4981         'net25' => { bus => 1, addr => 20 },
4982         'net26' => { bus => 1, addr => 21 },
4983         'net27' => { bus => 1, addr => 22 },
4984         'net28' => { bus => 1, addr => 23 },
4985         'net29' => { bus => 1, addr => 24 },
4986         'net30' => { bus => 1, addr => 25 },
4987         'net31' => { bus => 1, addr => 26 },
4988         'xhci' => { bus => 1, addr => 27 },
4989         'virtio6' => { bus => 2, addr => 1 },
4990         'virtio7' => { bus => 2, addr => 2 },
4991         'virtio8' => { bus => 2, addr => 3 },
4992         'virtio9' => { bus => 2, addr => 4 },
4993         'virtio10' => { bus => 2, addr => 5 },
4994         'virtio11' => { bus => 2, addr => 6 },
4995         'virtio12' => { bus => 2, addr => 7 },
4996         'virtio13' => { bus => 2, addr => 8 },
4997         'virtio14' => { bus => 2, addr => 9 },
4998         'virtio15' => { bus => 2, addr => 10 },
4999         'virtioscsi0' => { bus => 3, addr => 1 },
5000         'virtioscsi1' => { bus => 3, addr => 2 },
5001         'virtioscsi2' => { bus => 3, addr => 3 },
5002         'virtioscsi3' => { bus => 3, addr => 4 },
5003         'virtioscsi4' => { bus => 3, addr => 5 },
5004         'virtioscsi5' => { bus => 3, addr => 6 },
5005         'virtioscsi6' => { bus => 3, addr => 7 },
5006         'virtioscsi7' => { bus => 3, addr => 8 },
5007         'virtioscsi8' => { bus => 3, addr => 9 },
5008         'virtioscsi9' => { bus => 3, addr => 10 },
5009         'virtioscsi10' => { bus => 3, addr => 11 },
5010         'virtioscsi11' => { bus => 3, addr => 12 },
5011         'virtioscsi12' => { bus => 3, addr => 13 },
5012         'virtioscsi13' => { bus => 3, addr => 14 },
5013         'virtioscsi14' => { bus => 3, addr => 15 },
5014         'virtioscsi15' => { bus => 3, addr => 16 },
5015         'virtioscsi16' => { bus => 3, addr => 17 },
5016         'virtioscsi17' => { bus => 3, addr => 18 },
5017         'virtioscsi18' => { bus => 3, addr => 19 },
5018         'virtioscsi19' => { bus => 3, addr => 20 },
5019         'virtioscsi20' => { bus => 3, addr => 21 },
5020         'virtioscsi21' => { bus => 3, addr => 22 },
5021         'virtioscsi22' => { bus => 3, addr => 23 },
5022         'virtioscsi23' => { bus => 3, addr => 24 },
5023         'virtioscsi24' => { bus => 3, addr => 25 },
5024         'virtioscsi25' => { bus => 3, addr => 26 },
5025         'virtioscsi26' => { bus => 3, addr => 27 },
5026         'virtioscsi27' => { bus => 3, addr => 28 },
5027         'virtioscsi28' => { bus => 3, addr => 29 },
5028         'virtioscsi29' => { bus => 3, addr => 30 },
5029         'virtioscsi30' => { bus => 3, addr => 31 },
5030
5031     };
5032
5033     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5034            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5035            my $bus = $devices->{$id}->{bus};
5036            $res = ",bus=pci.$bus,addr=$addr";
5037            $bridges->{$bus} = 1 if $bridges;
5038     }
5039     return $res;
5040
5041 }
5042
5043 sub print_pcie_addr {
5044     my ($id) = @_;
5045
5046     my $res = '';
5047     my $devices = {
5048         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5049         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5050         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5051         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5052     };
5053
5054     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5055            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5056            my $bus = $devices->{$id}->{bus};
5057            $res = ",bus=$bus,addr=$addr";
5058     }
5059     return $res;
5060
5061 }
5062
5063 # vzdump restore implementaion
5064
5065 sub tar_archive_read_firstfile {
5066     my $archive = shift;
5067
5068     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5069
5070     # try to detect archive type first
5071     my $pid = open (TMP, "tar tf '$archive'|") ||
5072         die "unable to open file '$archive'\n";
5073     my $firstfile = <TMP>;
5074     kill 15, $pid;
5075     close TMP;
5076
5077     die "ERROR: archive contaions no data\n" if !$firstfile;
5078     chomp $firstfile;
5079
5080     return $firstfile;
5081 }
5082
5083 sub tar_restore_cleanup {
5084     my ($storecfg, $statfile) = @_;
5085
5086     print STDERR "starting cleanup\n";
5087
5088     if (my $fd = IO::File->new($statfile, "r")) {
5089         while (defined(my $line = <$fd>)) {
5090             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5091                 my $volid = $2;
5092                 eval {
5093                     if ($volid =~ m|^/|) {
5094                         unlink $volid || die 'unlink failed\n';
5095                     } else {
5096                         PVE::Storage::vdisk_free($storecfg, $volid);
5097                     }
5098                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5099                 };
5100                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5101             } else {
5102                 print STDERR "unable to parse line in statfile - $line";
5103             }
5104         }
5105         $fd->close();
5106     }
5107 }
5108
5109 sub restore_archive {
5110     my ($archive, $vmid, $user, $opts) = @_;
5111
5112     my $format = $opts->{format};
5113     my $comp;
5114
5115     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5116         $format = 'tar' if !$format;
5117         $comp = 'gzip';
5118     } elsif ($archive =~ m/\.tar$/) {
5119         $format = 'tar' if !$format;
5120     } elsif ($archive =~ m/.tar.lzo$/) {
5121         $format = 'tar' if !$format;
5122         $comp = 'lzop';
5123     } elsif ($archive =~ m/\.vma$/) {
5124         $format = 'vma' if !$format;
5125     } elsif ($archive =~ m/\.vma\.gz$/) {
5126         $format = 'vma' if !$format;
5127         $comp = 'gzip';
5128     } elsif ($archive =~ m/\.vma\.lzo$/) {
5129         $format = 'vma' if !$format;
5130         $comp = 'lzop';
5131     } else {
5132         $format = 'vma' if !$format; # default
5133     }
5134
5135     # try to detect archive format
5136     if ($format eq 'tar') {
5137         return restore_tar_archive($archive, $vmid, $user, $opts);
5138     } else {
5139         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5140     }
5141 }
5142
5143 sub restore_update_config_line {
5144     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5145
5146     return if $line =~ m/^\#qmdump\#/;
5147     return if $line =~ m/^\#vzdump\#/;
5148     return if $line =~ m/^lock:/;
5149     return if $line =~ m/^unused\d+:/;
5150     return if $line =~ m/^parent:/;
5151     return if $line =~ m/^template:/; # restored VM is never a template
5152
5153     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5154         # try to convert old 1.X settings
5155         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5156         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5157             my ($model, $macaddr) = split(/\=/, $devconfig);
5158             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5159             my $net = {
5160                 model => $model,
5161                 bridge => "vmbr$ind",
5162                 macaddr => $macaddr,
5163             };
5164             my $netstr = print_net($net);
5165
5166             print $outfd "net$cookie->{netcount}: $netstr\n";
5167             $cookie->{netcount}++;
5168         }
5169     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5170         my ($id, $netstr) = ($1, $2);
5171         my $net = parse_net($netstr);
5172         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5173         $netstr = print_net($net);
5174         print $outfd "$id: $netstr\n";
5175     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5176         my $virtdev = $1;
5177         my $value = $3;
5178         my $di = parse_drive($virtdev, $value);
5179         if (defined($di->{backup}) && !$di->{backup}) {
5180             print $outfd "#$line";
5181         } elsif ($map->{$virtdev}) {
5182             delete $di->{format}; # format can change on restore
5183             $di->{file} = $map->{$virtdev};
5184             $value = print_drive($vmid, $di);
5185             print $outfd "$virtdev: $value\n";
5186         } else {
5187             print $outfd $line;
5188         }
5189     } else {
5190         print $outfd $line;
5191     }
5192 }
5193
5194 sub scan_volids {
5195     my ($cfg, $vmid) = @_;
5196
5197     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5198
5199     my $volid_hash = {};
5200     foreach my $storeid (keys %$info) {
5201         foreach my $item (@{$info->{$storeid}}) {
5202             next if !($item->{volid} && $item->{size});
5203             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5204             $volid_hash->{$item->{volid}} = $item;
5205         }
5206     }
5207
5208     return $volid_hash;
5209 }
5210
5211 sub is_volume_in_use {
5212     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5213
5214     my $path = PVE::Storage::path($storecfg, $volid);
5215
5216     my $scan_config = sub {
5217         my ($cref, $snapname) = @_;
5218
5219         foreach my $key (keys %$cref) {
5220             my $value = $cref->{$key};
5221             if (is_valid_drivename($key)) {
5222                 next if $skip_drive && $key eq $skip_drive;
5223                 my $drive = parse_drive($key, $value);
5224                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5225                 return 1 if $volid eq $drive->{file};
5226                 if ($drive->{file} =~ m!^/!) {
5227                     return 1 if $drive->{file} eq $path;
5228                 } else {
5229                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5230                     next if !$storeid;
5231                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5232                     next if !$scfg;
5233                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5234                 }
5235             }
5236         }
5237
5238         return 0;
5239     };
5240
5241     return 1 if &$scan_config($conf);
5242
5243     undef $skip_drive;
5244
5245     foreach my $snapname (keys %{$conf->{snapshots}}) {
5246         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5247     }
5248
5249     return 0;
5250 }
5251
5252 sub update_disksize {
5253     my ($vmid, $conf, $volid_hash) = @_;
5254
5255     my $changes;
5256
5257     my $used = {};
5258
5259     # Note: it is allowed to define multiple storages with same path (alias), so
5260     # we need to check both 'volid' and real 'path' (two different volid can point
5261     # to the same path).
5262
5263     my $usedpath = {};
5264
5265     # update size info
5266     foreach my $opt (keys %$conf) {
5267         if (is_valid_drivename($opt)) {
5268             my $drive = parse_drive($opt, $conf->{$opt});
5269             my $volid = $drive->{file};
5270             next if !$volid;
5271
5272             $used->{$volid} = 1;
5273             if ($volid_hash->{$volid} &&
5274                 (my $path = $volid_hash->{$volid}->{path})) {
5275                 $usedpath->{$path} = 1;
5276             }
5277
5278             next if drive_is_cdrom($drive);
5279             next if !$volid_hash->{$volid};
5280
5281             $drive->{size} = $volid_hash->{$volid}->{size};
5282             my $new = print_drive($vmid, $drive);
5283             if ($new ne $conf->{$opt}) {
5284                 $changes = 1;
5285                 $conf->{$opt} = $new;
5286             }
5287         }
5288     }
5289
5290     # remove 'unusedX' entry if volume is used
5291     foreach my $opt (keys %$conf) {
5292         next if $opt !~ m/^unused\d+$/;
5293         my $volid = $conf->{$opt};
5294         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5295         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5296             $changes = 1;
5297             delete $conf->{$opt};
5298         }
5299     }
5300
5301     foreach my $volid (sort keys %$volid_hash) {
5302         next if $volid =~ m/vm-$vmid-state-/;
5303         next if $used->{$volid};
5304         my $path = $volid_hash->{$volid}->{path};
5305         next if !$path; # just to be sure
5306         next if $usedpath->{$path};
5307         $changes = 1;
5308         add_unused_volume($conf, $volid);
5309         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5310     }
5311
5312     return $changes;
5313 }
5314
5315 sub rescan {
5316     my ($vmid, $nolock) = @_;
5317
5318     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5319
5320     my $volid_hash = scan_volids($cfg, $vmid);
5321
5322     my $updatefn =  sub {
5323         my ($vmid) = @_;
5324
5325         my $conf = PVE::QemuConfig->load_config($vmid);
5326
5327         PVE::QemuConfig->check_lock($conf);
5328
5329         my $vm_volids = {};
5330         foreach my $volid (keys %$volid_hash) {
5331             my $info = $volid_hash->{$volid};
5332             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5333         }
5334
5335         my $changes = update_disksize($vmid, $conf, $vm_volids);
5336
5337         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5338     };
5339
5340     if (defined($vmid)) {
5341         if ($nolock) {
5342             &$updatefn($vmid);
5343         } else {
5344             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5345         }
5346     } else {
5347         my $vmlist = config_list();
5348         foreach my $vmid (keys %$vmlist) {
5349             if ($nolock) {
5350                 &$updatefn($vmid);
5351             } else {
5352                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5353             }
5354         }
5355     }
5356 }
5357
5358 sub restore_vma_archive {
5359     my ($archive, $vmid, $user, $opts, $comp) = @_;
5360
5361     my $input = $archive eq '-' ? "<&STDIN" : undef;
5362     my $readfrom = $archive;
5363
5364     my $uncomp = '';
5365     if ($comp) {
5366         $readfrom = '-';
5367         my $qarchive = PVE::Tools::shellquote($archive);
5368         if ($comp eq 'gzip') {
5369             $uncomp = "zcat $qarchive|";
5370         } elsif ($comp eq 'lzop') {
5371             $uncomp = "lzop -d -c $qarchive|";
5372         } else {
5373             die "unknown compression method '$comp'\n";
5374         }
5375
5376     }
5377
5378     my $tmpdir = "/var/tmp/vzdumptmp$$";
5379     rmtree $tmpdir;
5380
5381     # disable interrupts (always do cleanups)
5382     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5383         warn "got interrupt - ignored\n";
5384     };
5385
5386     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5387     POSIX::mkfifo($mapfifo, 0600);
5388     my $fifofh;
5389
5390     my $openfifo = sub {
5391         open($fifofh, '>', $mapfifo) || die $!;
5392     };
5393
5394     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5395
5396     my $oldtimeout;
5397     my $timeout = 5;
5398
5399     my $devinfo = {};
5400
5401     my $rpcenv = PVE::RPCEnvironment::get();
5402
5403     my $conffile = PVE::QemuConfig->config_file($vmid);
5404     my $tmpfn = "$conffile.$$.tmp";
5405
5406     # Note: $oldconf is undef if VM does not exists
5407     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5408     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5409
5410     my $print_devmap = sub {
5411         my $virtdev_hash = {};
5412
5413         my $cfgfn = "$tmpdir/qemu-server.conf";
5414
5415         # we can read the config - that is already extracted
5416         my $fh = IO::File->new($cfgfn, "r") ||
5417             "unable to read qemu-server.conf - $!\n";
5418
5419         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5420         if (-f $fwcfgfn) {
5421             my $pve_firewall_dir = '/etc/pve/firewall';
5422             mkdir $pve_firewall_dir; # make sure the dir exists
5423             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5424         }
5425
5426         while (defined(my $line = <$fh>)) {
5427             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5428                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5429                 die "archive does not contain data for drive '$virtdev'\n"
5430                     if !$devinfo->{$devname};
5431                 if (defined($opts->{storage})) {
5432                     $storeid = $opts->{storage} || 'local';
5433                 } elsif (!$storeid) {
5434                     $storeid = 'local';
5435                 }
5436                 $format = 'raw' if !$format;
5437                 $devinfo->{$devname}->{devname} = $devname;
5438                 $devinfo->{$devname}->{virtdev} = $virtdev;
5439                 $devinfo->{$devname}->{format} = $format;
5440                 $devinfo->{$devname}->{storeid} = $storeid;
5441
5442                 # check permission on storage
5443                 my $pool = $opts->{pool}; # todo: do we need that?
5444                 if ($user ne 'root@pam') {
5445                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5446                 }
5447
5448                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5449             }
5450         }
5451
5452         foreach my $devname (keys %$devinfo) {
5453             die "found no device mapping information for device '$devname'\n"
5454                 if !$devinfo->{$devname}->{virtdev};
5455         }
5456
5457         my $cfg = cfs_read_file('storage.cfg');
5458
5459         # create empty/temp config
5460         if ($oldconf) {
5461             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5462             foreach_drive($oldconf, sub {
5463                 my ($ds, $drive) = @_;
5464
5465                 return if drive_is_cdrom($drive);
5466
5467                 my $volid = $drive->{file};
5468
5469                 return if !$volid || $volid =~ m|^/|;
5470
5471                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5472                 return if !$path || !$owner || ($owner != $vmid);
5473
5474                 # Note: only delete disk we want to restore
5475                 # other volumes will become unused
5476                 if ($virtdev_hash->{$ds}) {
5477                     PVE::Storage::vdisk_free($cfg, $volid);
5478                 }
5479             });
5480
5481             # delete vmstate files
5482             # since after the restore we have no snapshots anymore
5483             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5484                 my $snap = $oldconf->{snapshots}->{$snapname};
5485                 if ($snap->{vmstate}) {
5486                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5487                     if (my $err = $@) {
5488                         warn $err;
5489                     }
5490                 }
5491             }
5492         }
5493
5494         my $map = {};
5495         foreach my $virtdev (sort keys %$virtdev_hash) {
5496             my $d = $virtdev_hash->{$virtdev};
5497             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5498             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5499
5500             # test if requested format is supported
5501             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5502             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5503             $d->{format} = $defFormat if !$supported;
5504
5505             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5506                                                   $d->{format}, undef, $alloc_size);
5507             print STDERR "new volume ID is '$volid'\n";
5508             $d->{volid} = $volid;
5509             my $path = PVE::Storage::path($cfg, $volid);
5510
5511             PVE::Storage::activate_volumes($cfg,[$volid]);
5512
5513             my $write_zeros = 1;
5514             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5515                 $write_zeros = 0;
5516             }
5517
5518             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5519
5520             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5521             $map->{$virtdev} = $volid;
5522         }
5523
5524         $fh->seek(0, 0) || die "seek failed - $!\n";
5525
5526         my $outfd = new IO::File ($tmpfn, "w") ||
5527             die "unable to write config for VM $vmid\n";
5528
5529         my $cookie = { netcount => 0 };
5530         while (defined(my $line = <$fh>)) {
5531             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5532         }
5533
5534         $fh->close();
5535         $outfd->close();
5536     };
5537
5538     eval {
5539         # enable interrupts
5540         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5541             die "interrupted by signal\n";
5542         };
5543         local $SIG{ALRM} = sub { die "got timeout\n"; };
5544
5545         $oldtimeout = alarm($timeout);
5546
5547         my $parser = sub {
5548             my $line = shift;
5549
5550             print "$line\n";
5551
5552             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5553                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5554                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5555             } elsif ($line =~ m/^CTIME: /) {
5556                 # we correctly received the vma config, so we can disable
5557                 # the timeout now for disk allocation (set to 10 minutes, so
5558                 # that we always timeout if something goes wrong)
5559                 alarm(600);
5560                 &$print_devmap();
5561                 print $fifofh "done\n";
5562                 my $tmp = $oldtimeout || 0;
5563                 $oldtimeout = undef;
5564                 alarm($tmp);
5565                 close($fifofh);
5566             }
5567         };
5568
5569         print "restore vma archive: $cmd\n";
5570         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5571     };
5572     my $err = $@;
5573
5574     alarm($oldtimeout) if $oldtimeout;
5575
5576     my $vollist = [];
5577     foreach my $devname (keys %$devinfo) {
5578         my $volid = $devinfo->{$devname}->{volid};
5579         push @$vollist, $volid if $volid;
5580     }
5581
5582     my $cfg = cfs_read_file('storage.cfg');
5583     PVE::Storage::deactivate_volumes($cfg, $vollist);
5584
5585     unlink $mapfifo;
5586
5587     if ($err) {
5588         rmtree $tmpdir;
5589         unlink $tmpfn;
5590
5591         foreach my $devname (keys %$devinfo) {
5592             my $volid = $devinfo->{$devname}->{volid};
5593             next if !$volid;
5594             eval {
5595                 if ($volid =~ m|^/|) {
5596                     unlink $volid || die 'unlink failed\n';
5597                 } else {
5598                     PVE::Storage::vdisk_free($cfg, $volid);
5599                 }
5600                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5601             };
5602             print STDERR "unable to cleanup '$volid' - $@" if $@;
5603         }
5604         die $err;
5605     }
5606
5607     rmtree $tmpdir;
5608
5609     rename($tmpfn, $conffile) ||
5610         die "unable to commit configuration file '$conffile'\n";
5611
5612     PVE::Cluster::cfs_update(); # make sure we read new file
5613
5614     eval { rescan($vmid, 1); };
5615     warn $@ if $@;
5616 }
5617
5618 sub restore_tar_archive {
5619     my ($archive, $vmid, $user, $opts) = @_;
5620
5621     if ($archive ne '-') {
5622         my $firstfile = tar_archive_read_firstfile($archive);
5623         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5624             if $firstfile ne 'qemu-server.conf';
5625     }
5626
5627     my $storecfg = cfs_read_file('storage.cfg');
5628
5629     # destroy existing data - keep empty config
5630     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5631     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5632
5633     my $tocmd = "/usr/lib/qemu-server/qmextract";
5634
5635     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5636     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5637     $tocmd .= ' --prealloc' if $opts->{prealloc};
5638     $tocmd .= ' --info' if $opts->{info};
5639
5640     # tar option "xf" does not autodetect compression when read from STDIN,
5641     # so we pipe to zcat
5642     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5643         PVE::Tools::shellquote("--to-command=$tocmd");
5644
5645     my $tmpdir = "/var/tmp/vzdumptmp$$";
5646     mkpath $tmpdir;
5647
5648     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5649     local $ENV{VZDUMP_VMID} = $vmid;
5650     local $ENV{VZDUMP_USER} = $user;
5651
5652     my $conffile = PVE::QemuConfig->config_file($vmid);
5653     my $tmpfn = "$conffile.$$.tmp";
5654
5655     # disable interrupts (always do cleanups)
5656     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5657         print STDERR "got interrupt - ignored\n";
5658     };
5659
5660     eval {
5661         # enable interrupts
5662         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5663             die "interrupted by signal\n";
5664         };
5665
5666         if ($archive eq '-') {
5667             print "extracting archive from STDIN\n";
5668             run_command($cmd, input => "<&STDIN");
5669         } else {
5670             print "extracting archive '$archive'\n";
5671             run_command($cmd);
5672         }
5673
5674         return if $opts->{info};
5675
5676         # read new mapping
5677         my $map = {};
5678         my $statfile = "$tmpdir/qmrestore.stat";
5679         if (my $fd = IO::File->new($statfile, "r")) {
5680             while (defined (my $line = <$fd>)) {
5681                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5682                     $map->{$1} = $2 if $1;
5683                 } else {
5684                     print STDERR "unable to parse line in statfile - $line\n";
5685                 }
5686             }
5687             $fd->close();
5688         }
5689
5690         my $confsrc = "$tmpdir/qemu-server.conf";
5691
5692         my $srcfd = new IO::File($confsrc, "r") ||
5693             die "unable to open file '$confsrc'\n";
5694
5695         my $outfd = new IO::File ($tmpfn, "w") ||
5696             die "unable to write config for VM $vmid\n";
5697
5698         my $cookie = { netcount => 0 };
5699         while (defined (my $line = <$srcfd>)) {
5700             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5701         }
5702
5703         $srcfd->close();
5704         $outfd->close();
5705     };
5706     my $err = $@;
5707
5708     if ($err) {
5709
5710         unlink $tmpfn;
5711
5712         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5713
5714         die $err;
5715     }
5716
5717     rmtree $tmpdir;
5718
5719     rename $tmpfn, $conffile ||
5720         die "unable to commit configuration file '$conffile'\n";
5721
5722     PVE::Cluster::cfs_update(); # make sure we read new file
5723
5724     eval { rescan($vmid, 1); };
5725     warn $@ if $@;
5726 };
5727
5728
5729 # Internal snapshots
5730
5731 # NOTE: Snapshot create/delete involves several non-atomic
5732 # action, and can take a long time.
5733 # So we try to avoid locking the file and use 'lock' variable
5734 # inside the config file instead.
5735
5736 my $snapshot_copy_config = sub {
5737     my ($source, $dest) = @_;
5738
5739     foreach my $k (keys %$source) {
5740         next if $k eq 'snapshots';
5741         next if $k eq 'snapstate';
5742         next if $k eq 'snaptime';
5743         next if $k eq 'vmstate';
5744         next if $k eq 'lock';
5745         next if $k eq 'digest';
5746         next if $k eq 'description';
5747         next if $k =~ m/^unused\d+$/;
5748
5749         $dest->{$k} = $source->{$k};
5750     }
5751 };
5752
5753 my $snapshot_apply_config = sub {
5754     my ($conf, $snap) = @_;
5755
5756     # copy snapshot list
5757     my $newconf = {
5758         snapshots => $conf->{snapshots},
5759     };
5760
5761     # keep description and list of unused disks
5762     foreach my $k (keys %$conf) {
5763         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5764         $newconf->{$k} = $conf->{$k};
5765     }
5766
5767     &$snapshot_copy_config($snap, $newconf);
5768
5769     return $newconf;
5770 };
5771
5772 sub foreach_writable_storage {
5773     my ($conf, $func) = @_;
5774
5775     my $sidhash = {};
5776
5777     foreach my $ds (keys %$conf) {
5778         next if !is_valid_drivename($ds);
5779
5780         my $drive = parse_drive($ds, $conf->{$ds});
5781         next if !$drive;
5782         next if drive_is_cdrom($drive);
5783
5784         my $volid = $drive->{file};
5785
5786         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5787         $sidhash->{$sid} = $sid if $sid;
5788     }
5789
5790     foreach my $sid (sort keys %$sidhash) {
5791         &$func($sid);
5792     }
5793 }
5794
5795 my $alloc_vmstate_volid = sub {
5796     my ($storecfg, $vmid, $conf, $snapname) = @_;
5797
5798     # Note: we try to be smart when selecting a $target storage
5799
5800     my $target;
5801
5802     # search shared storage first
5803     foreach_writable_storage($conf, sub {
5804         my ($sid) = @_;
5805         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5806         return if !$scfg->{shared};
5807
5808         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5809     });
5810
5811     if (!$target) {
5812         # now search local storage
5813         foreach_writable_storage($conf, sub {
5814             my ($sid) = @_;
5815             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5816             return if $scfg->{shared};
5817
5818             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5819         });
5820     }
5821
5822     $target = 'local' if !$target;
5823
5824     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5825     # we abort live save after $conf->{memory}, so we need at max twice that space
5826     my $size = $conf->{memory}*2 + $driver_state_size;
5827
5828     my $name = "vm-$vmid-state-$snapname";
5829     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5830     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5831     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5832
5833     return $volid;
5834 };
5835
5836 sub snapshot_save_vmstate {
5837     my ($vmid, $conf, $snapname, $storecfg) = @_;
5838
5839     my $snap = $conf->{snapshots}->{$snapname};
5840
5841     $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5842     # always overwrite machine if we save vmstate. This makes sure we
5843     # can restore it later using correct machine type
5844     $snap->{machine} = get_current_qemu_machine($vmid);
5845 }
5846
5847 sub snapshot_prepare {
5848     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5849
5850     my $snap;
5851
5852     my $updatefn =  sub {
5853
5854         my $conf = PVE::QemuConfig->load_config($vmid);
5855
5856         die "you can't take a snapshot if it's a template\n"
5857             if PVE::QemuConfig->is_template($conf);
5858
5859         PVE::QemuConfig->check_lock($conf);
5860
5861         $conf->{lock} = 'snapshot';
5862
5863         die "snapshot name '$snapname' already used\n"
5864             if defined($conf->{snapshots}->{$snapname});
5865
5866         my $storecfg = PVE::Storage::config();
5867         die "snapshot feature is not available\n"
5868             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
5869
5870         $snap = $conf->{snapshots}->{$snapname} = {};
5871
5872         if ($save_vmstate && check_running($vmid)) {
5873             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
5874         }
5875
5876         &$snapshot_copy_config($conf, $snap);
5877
5878         $snap->{snapstate} = "prepare";
5879         $snap->{snaptime} = time();
5880         $snap->{description} = $comment if $comment;
5881
5882         PVE::QemuConfig->write_config($vmid, $conf);
5883     };
5884
5885     PVE::QemuConfig->lock_config($vmid, $updatefn);
5886
5887     return $snap;
5888 }
5889
5890 sub snapshot_commit {
5891     my ($vmid, $snapname) = @_;
5892
5893     my $updatefn = sub {
5894
5895         my $conf = PVE::QemuConfig->load_config($vmid);
5896
5897         die "missing snapshot lock\n"
5898             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5899
5900         my $snap = $conf->{snapshots}->{$snapname};
5901
5902         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5903
5904         die "wrong snapshot state\n"
5905             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5906
5907         delete $snap->{snapstate};
5908         delete $conf->{lock};
5909
5910         $conf->{parent} = $snapname;
5911
5912         PVE::QemuConfig->write_config($vmid, $conf);
5913     };
5914
5915     PVE::QemuConfig->lock_config($vmid, $updatefn);
5916 }
5917
5918 sub snapshot_rollback {
5919     my ($vmid, $snapname) = @_;
5920
5921     my $prepare = 1;
5922
5923     my $storecfg = PVE::Storage::config();
5924
5925     my $conf = PVE::QemuConfig->load_config($vmid);
5926
5927     my $get_snapshot_config = sub {
5928
5929         die "you can't rollback if vm is a template\n" if PVE::QemuConfig->is_template($conf);
5930
5931         my $res = $conf->{snapshots}->{$snapname};
5932
5933         die "snapshot '$snapname' does not exist\n" if !defined($res);
5934
5935         return $res;
5936     };
5937
5938     my $snap = &$get_snapshot_config();
5939
5940     foreach_drive($snap, sub {
5941         my ($ds, $drive) = @_;
5942
5943         return if drive_is_cdrom($drive);
5944
5945         my $volid = $drive->{file};
5946
5947         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5948     });
5949
5950     my $updatefn = sub {
5951
5952         $conf = PVE::QemuConfig->load_config($vmid);
5953
5954         $snap = &$get_snapshot_config();
5955
5956         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5957             if $snap->{snapstate};
5958
5959         if ($prepare) {
5960             PVE::QemuConfig->check_lock($conf);
5961             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5962         }
5963
5964         die "unable to rollback vm $vmid: vm is running\n"
5965             if check_running($vmid);
5966
5967         if ($prepare) {
5968             $conf->{lock} = 'rollback';
5969         } else {
5970             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5971             delete $conf->{lock};
5972         }
5973
5974         my $forcemachine;
5975
5976         if (!$prepare) {
5977             my $has_machine_config = defined($conf->{machine});
5978
5979             # copy snapshot config to current config
5980             $conf = &$snapshot_apply_config($conf, $snap);
5981             $conf->{parent} = $snapname;
5982
5983             # Note: old code did not store 'machine', so we try to be smart
5984             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5985             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5986             # we remove the 'machine' configuration if not explicitly specified
5987             # in the original config.
5988             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5989         }
5990
5991         PVE::QemuConfig->write_config($vmid, $conf);
5992
5993         if (!$prepare && $snap->{vmstate}) {
5994             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5995             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5996         }
5997     };
5998
5999     PVE::QemuConfig->lock_config($vmid, $updatefn);
6000
6001     foreach_drive($snap, sub {
6002         my ($ds, $drive) = @_;
6003
6004         return if drive_is_cdrom($drive);
6005
6006         my $volid = $drive->{file};
6007         my $device = "drive-$ds";
6008
6009         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6010     });
6011
6012     $prepare = 0;
6013     PVE::QemuConfig->lock_config($vmid, $updatefn);
6014 }
6015
6016 my $savevm_wait = sub {
6017     my ($vmid) = @_;
6018
6019     for(;;) {
6020         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6021         if (!$stat->{status}) {
6022             die "savevm not active\n";
6023         } elsif ($stat->{status} eq 'active') {
6024             sleep(1);
6025             next;
6026         } elsif ($stat->{status} eq 'completed') {
6027             last;
6028         } else {
6029             die "query-savevm returned status '$stat->{status}'\n";
6030         }
6031     }
6032 };
6033
6034 sub do_snapshots_with_qemu {
6035     my ($storecfg, $volid) = @_;
6036
6037     my $storage_name = PVE::Storage::parse_volume_id($volid);
6038
6039     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6040         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6041         return 1;
6042     }
6043
6044     if ($volid =~ m/\.(qcow2|qed)$/){
6045         return 1;
6046     }
6047
6048     return undef;
6049 }
6050
6051 sub qga_check_running {
6052     my ($vmid) = @_;
6053
6054     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6055     if ($@) {
6056         warn "Qemu Guest Agent are not running - $@";
6057         return 0;
6058     }
6059     return 1;
6060 }
6061
6062 sub check_freeze_needed {
6063     my ($vmid, $config, $save_vmstate) = @_;
6064
6065     my $running = check_running($vmid);
6066     if ($save_vmstate) {
6067         return ($running, $running && $config->{agent} && qga_check_running($vmid));
6068     } else {
6069         return ($running, 0);
6070     }
6071 }
6072
6073 sub snapshot_create {
6074     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6075
6076     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6077
6078     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6079
6080     my $config = PVE::QemuConfig->load_config($vmid);
6081
6082     my ($running, $freezefs) = check_freeze_needed($vmid, $config, $snap->{vmstate});
6083
6084     my $drivehash = {};
6085
6086     eval {
6087         if ($freezefs) {
6088             eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6089             warn "guest-fsfreeze-freeze problems - $@" if $@;
6090         }
6091
6092         # create internal snapshots of all drives
6093
6094         my $storecfg = PVE::Storage::config();
6095
6096         if ($running) {
6097             if ($snap->{vmstate}) {
6098                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6099                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6100                 &$savevm_wait($vmid);
6101             } else {
6102                 vm_mon_cmd($vmid, "savevm-start");
6103             }
6104         };
6105
6106         foreach_drive($snap, sub {
6107             my ($ds, $drive) = @_;
6108
6109             return if drive_is_cdrom($drive);
6110
6111             my $volid = $drive->{file};
6112             my $device = "drive-$ds";
6113
6114             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6115             $drivehash->{$ds} = 1;
6116        });
6117     };
6118     my $err = $@;
6119
6120     if ($running) {
6121         eval { vm_mon_cmd($vmid, "savevm-end")  };
6122         warn $@ if $@;
6123
6124         if ($freezefs) {
6125             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6126             warn "guest-fsfreeze-thaw problems - $@" if $@;
6127         }
6128
6129         # savevm-end is async, we need to wait
6130         for (;;) {
6131             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6132             if (!$stat->{bytes}) {
6133                 last;
6134             } else {
6135                 print "savevm not yet finished\n";
6136                 sleep(1);
6137                 next;
6138             }
6139         }
6140     }
6141
6142     if ($err) {
6143         warn "snapshot create failed: starting cleanup\n";
6144         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
6145         warn $@ if $@;
6146         die $err;
6147     }
6148
6149     snapshot_commit($vmid, $snapname);
6150 }
6151
6152 # Note: $drivehash is only set when called from snapshot_create.
6153 sub snapshot_delete {
6154     my ($vmid, $snapname, $force, $drivehash) = @_;
6155
6156     my $prepare = 1;
6157
6158     my $snap;
6159     my $unused = [];
6160
6161     my $unlink_parent = sub {
6162         my ($confref, $new_parent) = @_;
6163
6164         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6165             if ($new_parent) {
6166                 $confref->{parent} = $new_parent;
6167             } else {
6168                 delete $confref->{parent};
6169             }
6170         }
6171     };
6172
6173     my $updatefn =  sub {
6174         my ($remove_drive) = @_;
6175
6176         my $conf = PVE::QemuConfig->load_config($vmid);
6177
6178         if (!$drivehash) {
6179             PVE::QemuConfig->check_lock($conf);
6180             die "you can't delete a snapshot if vm is a template\n"
6181                 if PVE::QemuConfig->is_template($conf);
6182         }
6183
6184         $snap = $conf->{snapshots}->{$snapname};
6185
6186         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6187
6188         # remove parent refs
6189         if (!$prepare) {
6190             &$unlink_parent($conf, $snap->{parent});
6191             foreach my $sn (keys %{$conf->{snapshots}}) {
6192                 next if $sn eq $snapname;
6193                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6194             }
6195         }
6196
6197         if ($remove_drive) {
6198             if ($remove_drive eq 'vmstate') {
6199                 delete $snap->{$remove_drive};
6200             } else {
6201                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6202                 my $volid = $drive->{file};
6203                 delete $snap->{$remove_drive};
6204                 add_unused_volume($conf, $volid);
6205             }
6206         }
6207
6208         if ($prepare) {
6209             $snap->{snapstate} = 'delete';
6210         } else {
6211             delete $conf->{snapshots}->{$snapname};
6212             delete $conf->{lock} if $drivehash;
6213             foreach my $volid (@$unused) {
6214                 add_unused_volume($conf, $volid);
6215             }
6216         }
6217
6218         PVE::QemuConfig->write_config($vmid, $conf);
6219     };
6220
6221     PVE::QemuConfig->lock_config($vmid, $updatefn);
6222
6223     # now remove vmstate file
6224
6225     my $storecfg = PVE::Storage::config();
6226
6227     if ($snap->{vmstate}) {
6228         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6229         if (my $err = $@) {
6230             die $err if !$force;
6231             warn $err;
6232         }
6233         # save changes (remove vmstate from snapshot)
6234         PVE::QemuConfig->lock_config($vmid, $updatefn, 'vmstate') if !$force;
6235     };
6236
6237     # now remove all internal snapshots
6238     foreach_drive($snap, sub {
6239         my ($ds, $drive) = @_;
6240
6241         return if drive_is_cdrom($drive);
6242
6243         my $volid = $drive->{file};
6244         my $device = "drive-$ds";
6245
6246         if (!$drivehash || $drivehash->{$ds}) {
6247             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6248             if (my $err = $@) {
6249                 die $err if !$force;
6250                 warn $err;
6251             }
6252         }
6253
6254         # save changes (remove drive fron snapshot)
6255         PVE::QemuConfig->lock_config($vmid, $updatefn, $ds) if !$force;
6256         push @$unused, $volid;
6257     });
6258
6259     # now cleanup config
6260     $prepare = 0;
6261     PVE::QemuConfig->lock_config($vmid, $updatefn);
6262 }
6263
6264 sub has_feature {
6265     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
6266
6267     my $err;
6268     foreach_drive($conf, sub {
6269         my ($ds, $drive) = @_;
6270
6271         return if drive_is_cdrom($drive);
6272         return if $backup_only && !$drive->{backup};
6273         my $volid = $drive->{file};
6274         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6275     });
6276
6277     return $err ? 0 : 1;
6278 }
6279
6280 sub template_create {
6281     my ($vmid, $conf, $disk) = @_;
6282
6283     my $storecfg = PVE::Storage::config();
6284
6285     foreach_drive($conf, sub {
6286         my ($ds, $drive) = @_;
6287
6288         return if drive_is_cdrom($drive);
6289         return if $disk && $ds ne $disk;
6290
6291         my $volid = $drive->{file};
6292         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6293
6294         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6295         $drive->{file} = $voliddst;
6296         $conf->{$ds} = print_drive($vmid, $drive);
6297         PVE::QemuConfig->write_config($vmid, $conf);
6298     });
6299 }
6300
6301 sub qemu_img_convert {
6302     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6303
6304     my $storecfg = PVE::Storage::config();
6305     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6306     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6307
6308     if ($src_storeid && $dst_storeid) {
6309
6310         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6311
6312         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6313         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6314
6315         my $src_format = qemu_img_format($src_scfg, $src_volname);
6316         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6317
6318         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6319         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6320
6321         my $cmd = [];
6322         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6323         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6324         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6325
6326         my $parser = sub {
6327             my $line = shift;
6328             if($line =~ m/\((\S+)\/100\%\)/){
6329                 my $percent = $1;
6330                 my $transferred = int($size * $percent / 100);
6331                 my $remaining = $size - $transferred;
6332
6333                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6334             }
6335
6336         };
6337
6338         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6339         my $err = $@;
6340         die "copy failed: $err" if $err;
6341     }
6342 }
6343
6344 sub qemu_img_format {
6345     my ($scfg, $volname) = @_;
6346
6347     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6348         return $1;
6349     } else {
6350         return "raw";
6351     }
6352 }
6353
6354 sub qemu_drive_mirror {
6355     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6356
6357     my $storecfg = PVE::Storage::config();
6358     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6359
6360     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6361
6362     my $format = qemu_img_format($dst_scfg, $dst_volname);
6363
6364     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6365
6366     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6367     $opts->{format} = $format if $format;
6368
6369     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6370
6371     eval {
6372     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6373         while (1) {
6374             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6375             my $stat = @$stats[0];
6376             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6377             die "error job is not mirroring" if $stat->{type} ne "mirror";
6378
6379             my $busy = $stat->{busy};
6380             my $ready = $stat->{ready};
6381
6382             if (my $total = $stat->{len}) {
6383                 my $transferred = $stat->{offset} || 0;
6384                 my $remaining = $total - $transferred;
6385                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6386
6387                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6388             }
6389
6390
6391             if ($stat->{ready} eq 'true') {
6392
6393                 last if $vmiddst != $vmid;
6394
6395                 # try to switch the disk if source and destination are on the same guest
6396                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6397                 last if !$@;
6398                 die $@ if $@ !~ m/cannot be completed/;
6399             }
6400             sleep 1;
6401         }
6402
6403
6404     };
6405     my $err = $@;
6406
6407     my $cancel_job = sub {
6408         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6409         while (1) {
6410             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6411             my $stat = @$stats[0];
6412             last if !$stat;
6413             sleep 1;
6414         }
6415     };
6416
6417     if ($err) {
6418         eval { &$cancel_job(); };
6419         die "mirroring error: $err";
6420     }
6421
6422     if ($vmiddst != $vmid) {
6423         # if we clone a disk for a new target vm, we don't switch the disk
6424         &$cancel_job(); # so we call block-job-cancel
6425     }
6426 }
6427
6428 sub clone_disk {
6429     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6430         $newvmid, $storage, $format, $full, $newvollist) = @_;
6431
6432     my $newvolid;
6433
6434     if (!$full) {
6435         print "create linked clone of drive $drivename ($drive->{file})\n";
6436         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6437         push @$newvollist, $newvolid;
6438     } else {
6439         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6440         $storeid = $storage if $storage;
6441
6442         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6443         if (!$format) {
6444             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6445             $format = qemu_img_format($scfg, $volname);
6446         }
6447
6448         # test if requested format is supported - else use default
6449         my $supported = grep { $_ eq $format } @$validFormats;
6450         $format = $defFormat if !$supported;
6451
6452         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6453
6454         print "create full clone of drive $drivename ($drive->{file})\n";
6455         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6456         push @$newvollist, $newvolid;
6457
6458         PVE::Storage::activate_volumes($storecfg, $newvollist);
6459
6460         if (!$running || $snapname) {
6461             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6462         } else {
6463             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6464         }
6465     }
6466
6467     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6468
6469     my $disk = $drive;
6470     $disk->{format} = undef;
6471     $disk->{file} = $newvolid;
6472     $disk->{size} = $size;
6473
6474     return $disk;
6475 }
6476
6477 # this only works if VM is running
6478 sub get_current_qemu_machine {
6479     my ($vmid) = @_;
6480
6481     my $cmd = { execute => 'query-machines', arguments => {} };
6482     my $res = vm_qmp_command($vmid, $cmd);
6483
6484     my ($current, $default);
6485     foreach my $e (@$res) {
6486         $default = $e->{name} if $e->{'is-default'};
6487         $current = $e->{name} if $e->{'is-current'};
6488     }
6489
6490     # fallback to the default machine if current is not supported by qemu
6491     return $current || $default || 'pc';
6492 }
6493
6494 sub qemu_machine_feature_enabled {
6495     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6496
6497     my $current_major;
6498     my $current_minor;
6499
6500     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6501
6502         $current_major = $3;
6503         $current_minor = $4;
6504
6505     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6506
6507         $current_major = $1;
6508         $current_minor = $2;
6509     }
6510
6511     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6512
6513
6514 }
6515
6516 sub qemu_machine_pxe {
6517     my ($vmid, $conf, $machine) = @_;
6518
6519     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6520
6521     foreach my $opt (keys %$conf) {
6522         next if $opt !~ m/^net(\d+)$/;
6523         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6524         next if !$net;
6525         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6526         return $machine.".pxe" if $romfile =~ m/pxe/;
6527         last;
6528     }
6529
6530     return $machine;
6531 }
6532
6533 sub qemu_use_old_bios_files {
6534     my ($machine_type) = @_;
6535
6536     return if !$machine_type;
6537
6538     my $use_old_bios_files = undef;
6539
6540     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6541         $machine_type = $1;
6542         $use_old_bios_files = 1;
6543     } else {
6544         my $kvmver = kvm_user_version();
6545         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6546         # load new efi bios files on migration. So this hack is required to allow
6547         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6548         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6549         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6550     }
6551
6552     return ($use_old_bios_files, $machine_type);
6553 }
6554
6555 sub lspci {
6556
6557     my $devices = {};
6558
6559     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6560             my (undef, $id, $function) = @_;
6561             my $res = { id => $id, function => $function};
6562             push @{$devices->{$id}}, $res;
6563     });
6564
6565     return $devices;
6566 }
6567
6568 sub vm_iothreads_list {
6569     my ($vmid) = @_;
6570
6571     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6572
6573     my $iothreads = {};
6574     foreach my $iothread (@$res) {
6575         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6576     }
6577
6578     return $iothreads;
6579 }
6580
6581 sub scsihw_infos {
6582     my ($conf, $drive) = @_;
6583
6584     my $maxdev = 0;
6585
6586     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6587         $maxdev = 7;
6588     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6589         $maxdev = 1;
6590     } else {
6591         $maxdev = 256;
6592     }
6593
6594     my $controller = int($drive->{index} / $maxdev);
6595     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6596
6597     return ($maxdev, $controller, $controller_prefix);
6598 }
6599
6600 # bash completion helper
6601
6602 sub complete_backup_archives {
6603     my ($cmdname, $pname, $cvalue) = @_;
6604
6605     my $cfg = PVE::Storage::config();
6606
6607     my $storeid;
6608
6609     if ($cvalue =~ m/^([^:]+):/) {
6610         $storeid = $1;
6611     }
6612
6613     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6614
6615     my $res = [];
6616     foreach my $id (keys %$data) {
6617         foreach my $item (@{$data->{$id}}) {
6618             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6619             push @$res, $item->{volid} if defined($item->{volid});
6620         }
6621     }
6622
6623     return $res;
6624 }
6625
6626 my $complete_vmid_full = sub {
6627     my ($running) = @_;
6628
6629     my $idlist = vmstatus();
6630
6631     my $res = [];
6632
6633     foreach my $id (keys %$idlist) {
6634         my $d = $idlist->{$id};
6635         if (defined($running)) {
6636             next if $d->{template};
6637             next if $running && $d->{status} ne 'running';
6638             next if !$running && $d->{status} eq 'running';
6639         }
6640         push @$res, $id;
6641
6642     }
6643     return $res;
6644 };
6645
6646 sub complete_vmid {
6647     return &$complete_vmid_full();
6648 }
6649
6650 sub complete_vmid_stopped {
6651     return &$complete_vmid_full(0);
6652 }
6653
6654 sub complete_vmid_running {
6655     return &$complete_vmid_full(1);
6656 }
6657
6658 sub complete_storage {
6659
6660     my $cfg = PVE::Storage::config();
6661     my $ids = $cfg->{ids};
6662
6663     my $res = [];
6664     foreach my $sid (keys %$ids) {
6665         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6666         next if !$ids->{$sid}->{content}->{images};
6667         push @$res, $sid;
6668     }
6669
6670     return $res;
6671 }
6672
6673 1;