PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbdesc = {
 731     optional => 1,
 732     type => 'string', format => 'pve-qm-usb-device',
 733     typetext => 'host=HOSTUSBDEVICE [,usb3=yes|no]|spice',
 734     description => <<EODESCR,
 735 Configure an USB device (n is 0 to 4). This can be used to
 736 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 737
 738 'bus-port(.port)*' (decimal numbers) or
 739 'vendor_id:product_id' (hexadeciaml numbers)
 740
 741 You can use the 'lsusb -t' command to list existing usb devices.
 742
 743 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 744
 745 The value 'spice' can be used to add a usb redirection devices for spice.
 746
 747 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work with spice redirection).
 748
 749 EODESCR
 750 };
 751 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 752
 753 my $hostpcidesc = {
 754         optional => 1,
 755         type => 'string', format => 'pve-qm-hostpci',
 756         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 757         description => <<EODESCR,
 758 Map host pci devices. HOSTPCIDEVICE syntax is:
 759
 760 'bus:dev.func' (hexadecimal numbers)
 761
 762 You can us the 'lspci' command to list existing pci devices.
 763
 764 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 765
 766 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 767
 768 Experimental: user reported problems with this option.
 769 EODESCR
 770 };
 771 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 772
 773 my $serialdesc = {
 774         optional => 1,
 775         type => 'string',
 776         pattern => '(/dev/.+|socket)',
 777         description =>  <<EODESCR,
 778 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 779
 780 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 781
 782 Experimental: user reported problems with this option.
 783 EODESCR
 784 };
 785
 786 my $paralleldesc= {
 787         optional => 1,
 788         type => 'string',
 789         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 790         description =>  <<EODESCR,
 791 Map host parallel devices (n is 0 to 2).
 792
 793 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 794
 795 Experimental: user reported problems with this option.
 796 EODESCR
 797 };
 798
 799 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 800     $confdesc->{"parallel$i"} = $paralleldesc;
 801 }
 802
 803 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 804     $confdesc->{"serial$i"} = $serialdesc;
 805 }
 806
 807 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 808     $confdesc->{"hostpci$i"} = $hostpcidesc;
 809 }
 810
 811 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 812     $drivename_hash->{"ide$i"} = 1;
 813     $confdesc->{"ide$i"} = $idedesc;
 814 }
 815
 816 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 817     $drivename_hash->{"sata$i"} = 1;
 818     $confdesc->{"sata$i"} = $satadesc;
 819 }
 820
 821 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 822     $drivename_hash->{"scsi$i"} = 1;
 823     $confdesc->{"scsi$i"} = $scsidesc ;
 824 }
 825
 826 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 827     $drivename_hash->{"virtio$i"} = 1;
 828     $confdesc->{"virtio$i"} = $virtiodesc;
 829 }
 830
 831 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 832     $confdesc->{"usb$i"} = $usbdesc;
 833 }
 834
 835 my $unuseddesc = {
 836     optional => 1,
 837     type => 'string', format => 'pve-volume-id',
 838     description => "Reference to unused volumes.",
 839 };
 840
 841 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 842     $confdesc->{"unused$i"} = $unuseddesc;
 843 }
 844
 845 my $kvm_api_version = 0;
 846
 847 sub kvm_version {
 848
 849     return $kvm_api_version if $kvm_api_version;
 850
 851     my $fh = IO::File->new("</dev/kvm") ||
 852         return 0;
 853
 854     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 855         $kvm_api_version = $v;
 856     }
 857
 858     $fh->close();
 859
 860     return  $kvm_api_version;
 861 }
 862
 863 my $kvm_user_version;
 864
 865 sub kvm_user_version {
 866
 867     return $kvm_user_version if $kvm_user_version;
 868
 869     $kvm_user_version = 'unknown';
 870
 871     my $tmp = `kvm -help 2>/dev/null`;
 872
 873     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 874         $kvm_user_version = $2;
 875     }
 876
 877     return $kvm_user_version;
 878
 879 }
 880
 881 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 882
 883 sub disknames {
 884     # order is important - used to autoselect boot disk
 885     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 886             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 887             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 888             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 889 }
 890
 891 sub valid_drivename {
 892     my $dev = shift;
 893
 894     return defined($drivename_hash->{$dev});
 895 }
 896
 897 sub option_exists {
 898     my $key = shift;
 899     return defined($confdesc->{$key});
 900 }
 901
 902 sub nic_models {
 903     return $nic_model_list;
 904 }
 905
 906 sub os_list_description {
 907
 908     return {
 909         other => 'Other',
 910         wxp => 'Windows XP',
 911         w2k => 'Windows 2000',
 912         w2k3 =>, 'Windows 2003',
 913         w2k8 => 'Windows 2008',
 914         wvista => 'Windows Vista',
 915         win7 => 'Windows 7',
 916         win8 => 'Windows 8/2012',
 917         l24 => 'Linux 2.4',
 918         l26 => 'Linux 2.6',
 919     };
 920 }
 921
 922 my $cdrom_path;
 923
 924 sub get_cdrom_path {
 925
 926     return  $cdrom_path if $cdrom_path;
 927
 928     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 929     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 930     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 931 }
 932
 933 sub get_iso_path {
 934     my ($storecfg, $vmid, $cdrom) = @_;
 935
 936     if ($cdrom eq 'cdrom') {
 937         return get_cdrom_path();
 938     } elsif ($cdrom eq 'none') {
 939         return '';
 940     } elsif ($cdrom =~ m|^/|) {
 941         return $cdrom;
 942     } else {
 943         return PVE::Storage::path($storecfg, $cdrom);
 944     }
 945 }
 946
 947 # try to convert old style file names to volume IDs
 948 sub filename_to_volume_id {
 949     my ($vmid, $file, $media) = @_;
 950
 951     if (!($file eq 'none' || $file eq 'cdrom' ||
 952           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 953
 954         return undef if $file =~ m|/|;
 955
 956         if ($media && $media eq 'cdrom') {
 957             $file = "local:iso/$file";
 958         } else {
 959             $file = "local:$vmid/$file";
 960         }
 961     }
 962
 963     return $file;
 964 }
 965
 966 sub verify_media_type {
 967     my ($opt, $vtype, $media) = @_;
 968
 969     return if !$media;
 970
 971     my $etype;
 972     if ($media eq 'disk') {
 973         $etype = 'images';
 974     } elsif ($media eq 'cdrom') {
 975         $etype = 'iso';
 976     } else {
 977         die "internal error";
 978     }
 979
 980     return if ($vtype eq $etype);
 981
 982     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 983 }
 984
 985 sub cleanup_drive_path {
 986     my ($opt, $storecfg, $drive) = @_;
 987
 988     # try to convert filesystem paths to volume IDs
 989
 990     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 991         ($drive->{file} !~ m|^/dev/.+|) &&
 992         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 993         ($drive->{file} !~ m/^\d+$/)) {
 994         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 995         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 996         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 997         verify_media_type($opt, $vtype, $drive->{media});
 998         $drive->{file} = $volid;
 999     }
1000
1001     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1002 }
1003
1004 sub create_conf_nolock {
1005     my ($vmid, $settings) = @_;
1006
1007     my $filename = config_file($vmid);
1008
1009     die "configuration file '$filename' already exists\n" if -f $filename;
1010
1011     my $defaults = load_defaults();
1012
1013     $settings->{name} = "vm$vmid" if !$settings->{name};
1014     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
1015
1016     my $data = '';
1017     foreach my $opt (keys %$settings) {
1018         next if !$confdesc->{$opt};
1019
1020         my $value = $settings->{$opt};
1021         next if !$value;
1022
1023         $data .= "$opt: $value\n";
1024     }
1025
1026     PVE::Tools::file_set_contents($filename, $data);
1027 }
1028
1029 sub parse_hotplug_features {
1030     my ($data) = @_;
1031
1032     my $res = {};
1033
1034     return $res if $data eq '0';
1035
1036     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1037
1038     foreach my $feature (PVE::Tools::split_list($data)) {
1039         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1040             $res->{$1} = 1;
1041         } else {
1042             warn "ignoring unknown hotplug feature '$feature'\n";
1043         }
1044     }
1045     return $res;
1046 }
1047
1048 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1049 sub pve_verify_hotplug_features {
1050     my ($value, $noerr) = @_;
1051
1052     return $value if parse_hotplug_features($value);
1053
1054     return undef if $noerr;
1055
1056     die "unable to parse hotplug option\n";
1057 }
1058
1059 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1060 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1061 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1062 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1063 #        [,iothread=on][,serial=serial][,model=model]
1064
1065 sub parse_drive {
1066     my ($key, $data) = @_;
1067
1068     my ($interface, $index);
1069
1070     if ($key =~ m/^([^\d]+)(\d+)$/) {
1071         $interface = $1;
1072         $index = $2;
1073     } else {
1074         return undef;
1075     }
1076
1077     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1078                                      : $confdesc->{$key}->{format};
1079     if (!$desc) {
1080         warn "invalid drive key: $key\n";
1081         return undef;
1082     }
1083     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1084     return undef if !$res;
1085     $res->{interface} = $interface;
1086     $res->{index} = $index;
1087
1088     my $error = 0;
1089     foreach my $opt (qw(bps bps_rd bps_wr)) {
1090         if (my $bps = defined(delete $res->{$opt})) {
1091             if (defined($res->{"m$opt"})) {
1092                 warn "both $opt and m$opt specified\n";
1093                 ++$error;
1094                 next;
1095             }
1096             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1097         }
1098     }
1099     return undef if $error;
1100
1101     return undef if $res->{mbps_rd} && $res->{mbps};
1102     return undef if $res->{mbps_wr} && $res->{mbps};
1103     return undef if $res->{iops_rd} && $res->{iops};
1104     return undef if $res->{iops_wr} && $res->{iops};
1105
1106     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1107         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1108         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1109         return undef if $res->{interface} eq 'virtio';
1110     }
1111
1112     if (my $size = $res->{size}) {
1113         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1114     }
1115
1116     return $res;
1117 }
1118
1119 sub print_drive {
1120     my ($vmid, $drive) = @_;
1121     my $data = { %$drive };
1122     delete $data->{$_} for qw(index interface);
1123     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1124 }
1125
1126 sub scsi_inquiry {
1127     my($fh, $noerr) = @_;
1128
1129     my $SG_IO = 0x2285;
1130     my $SG_GET_VERSION_NUM = 0x2282;
1131
1132     my $versionbuf = "\x00" x 8;
1133     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1134     if (!$ret) {
1135         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1136         return undef;
1137     }
1138     my $version = unpack("I", $versionbuf);
1139     if ($version < 30000) {
1140         die "scsi generic interface too old\n"  if !$noerr;
1141         return undef;
1142     }
1143
1144     my $buf = "\x00" x 36;
1145     my $sensebuf = "\x00" x 8;
1146     my $cmd = pack("C x3 C x1", 0x12, 36);
1147
1148     # see /usr/include/scsi/sg.h
1149     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1150
1151     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1152                       length($sensebuf), 0, length($buf), $buf,
1153                       $cmd, $sensebuf, 6000);
1154
1155     $ret = ioctl($fh, $SG_IO, $packet);
1156     if (!$ret) {
1157         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1158         return undef;
1159     }
1160
1161     my @res = unpack($sg_io_hdr_t, $packet);
1162     if ($res[17] || $res[18]) {
1163         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1164         return undef;
1165     }
1166
1167     my $res = {};
1168     (my $byte0, my $byte1, $res->{vendor},
1169      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1170
1171     $res->{removable} = $byte1 & 128 ? 1 : 0;
1172     $res->{type} = $byte0 & 31;
1173
1174     return $res;
1175 }
1176
1177 sub path_is_scsi {
1178     my ($path) = @_;
1179
1180     my $fh = IO::File->new("+<$path") || return undef;
1181     my $res = scsi_inquiry($fh, 1);
1182     close($fh);
1183
1184     return $res;
1185 }
1186
1187 sub machine_type_is_q35 {
1188     my ($conf) = @_;
1189
1190     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1191 }
1192
1193 sub print_tabletdevice_full {
1194     my ($conf) = @_;
1195
1196     my $q35 = machine_type_is_q35($conf);
1197
1198     # we use uhci for old VMs because tablet driver was buggy in older qemu
1199     my $usbbus = $q35 ? "ehci" : "uhci";
1200
1201     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1202 }
1203
1204 sub print_drivedevice_full {
1205     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1206
1207     my $device = '';
1208     my $maxdev = 0;
1209
1210     if ($drive->{interface} eq 'virtio') {
1211         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1212         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1213         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1214     } elsif ($drive->{interface} eq 'scsi') {
1215
1216         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1217         my $unit = $drive->{index} % $maxdev;
1218         my $devicetype = 'hd';
1219         my $path = '';
1220         if (drive_is_cdrom($drive)) {
1221               $devicetype = 'cd';
1222         } else {
1223               if ($drive->{file} =~ m|^/|) {
1224                   $path = $drive->{file};
1225               } else {
1226                   $path = PVE::Storage::path($storecfg, $drive->{file});
1227               }
1228
1229               if($path =~ m/^iscsi\:\/\//){
1230                   $devicetype = 'generic';
1231               } else {
1232                   if (my $info = path_is_scsi($path)) {
1233                       if ($info->{type} == 0) {
1234                           $devicetype = 'block';
1235                       } elsif ($info->{type} == 1) { # tape
1236                           $devicetype = 'generic';
1237                       }
1238                   }
1239               }
1240          }
1241
1242         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1243             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1244         } else {
1245             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1246         }
1247
1248     } elsif ($drive->{interface} eq 'ide'){
1249         $maxdev = 2;
1250         my $controller = int($drive->{index} / $maxdev);
1251         my $unit = $drive->{index} % $maxdev;
1252         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1253
1254         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1255         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1256             $device .= ",model=$model";
1257         }
1258     } elsif ($drive->{interface} eq 'sata'){
1259         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1260         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1261         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1262     } elsif ($drive->{interface} eq 'usb') {
1263         die "implement me";
1264         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1265     } else {
1266         die "unsupported interface type";
1267     }
1268
1269     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1270
1271     return $device;
1272 }
1273
1274 sub get_initiator_name {
1275     my $initiator;
1276
1277     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1278     while (defined(my $line = <$fh>)) {
1279         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1280         $initiator = $1;
1281         last;
1282     }
1283     $fh->close();
1284
1285     return $initiator;
1286 }
1287
1288 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1289 sub print_drive_full {
1290     my ($storecfg, $vmid, $drive) = @_;
1291
1292     my $path;
1293     my $volid = $drive->{file};
1294     my $format;
1295
1296     if (drive_is_cdrom($drive)) {
1297         $path = get_iso_path($storecfg, $vmid, $volid);
1298     } else {
1299         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1300         if ($storeid) {
1301             $path = PVE::Storage::path($storecfg, $volid);
1302             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1303             $format = qemu_img_format($scfg, $volname);
1304         } else {
1305             $path = $volid;
1306             $format = "raw";
1307         }
1308    }
1309
1310     my $opts = '';
1311     foreach my $o (@qemu_drive_options) {
1312         next if $o eq 'bootindex';
1313         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1314     }
1315
1316     $opts .= ",format=$format" if $format && !$drive->{format};
1317
1318     foreach my $o (qw(bps bps_rd bps_wr)) {
1319         my $v = $drive->{"m$o"};
1320         $opts .= ",$o=" . int($v*1024*1024) if $v;
1321     }
1322
1323     my $cache_direct = 0;
1324
1325     if (my $cache = $drive->{cache}) {
1326         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1327     } elsif (!drive_is_cdrom($drive)) {
1328         $opts .= ",cache=none";
1329         $cache_direct = 1;
1330     }
1331
1332     # aio native works only with O_DIRECT
1333     if (!$drive->{aio}) {
1334         if($cache_direct) {
1335             $opts .= ",aio=native";
1336         } else {
1337             $opts .= ",aio=threads";
1338         }
1339     }
1340
1341     if (!drive_is_cdrom($drive)) {
1342         my $detectzeroes;
1343         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1344             $detectzeroes = 'off';
1345         } elsif ($drive->{discard}) {
1346             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1347         } else {
1348             # This used to be our default with discard not being specified:
1349             $detectzeroes = 'on';
1350         }
1351         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1352     }
1353
1354     my $pathinfo = $path ? "file=$path," : '';
1355
1356     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1357 }
1358
1359 sub print_netdevice_full {
1360     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1361
1362     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1363
1364     my $device = $net->{model};
1365     if ($net->{model} eq 'virtio') {
1366          $device = 'virtio-net-pci';
1367      };
1368
1369     my $pciaddr = print_pci_addr("$netid", $bridges);
1370     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1371     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1372         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1373         my $vectors = $net->{queues} * 2 + 2;
1374         $tmpstr .= ",vectors=$vectors,mq=on";
1375     }
1376     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1377
1378     if ($use_old_bios_files) {
1379         my $romfile;
1380         if ($device eq 'virtio-net-pci') {
1381             $romfile = 'pxe-virtio.rom';
1382         } elsif ($device eq 'e1000') {
1383             $romfile = 'pxe-e1000.rom';
1384         } elsif ($device eq 'ne2k') {
1385             $romfile = 'pxe-ne2k_pci.rom';
1386         } elsif ($device eq 'pcnet') {
1387             $romfile = 'pxe-pcnet.rom';
1388         } elsif ($device eq 'rtl8139') {
1389             $romfile = 'pxe-rtl8139.rom';
1390         }
1391         $tmpstr .= ",romfile=$romfile" if $romfile;
1392     }
1393
1394     return $tmpstr;
1395 }
1396
1397 sub print_netdev_full {
1398     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1399
1400     my $i = '';
1401     if ($netid =~ m/^net(\d+)$/) {
1402         $i = int($1);
1403     }
1404
1405     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1406
1407     my $ifname = "tap${vmid}i$i";
1408
1409     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1410     die "interface name '$ifname' is too long (max 15 character)\n"
1411         if length($ifname) >= 16;
1412
1413     my $vhostparam = '';
1414     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1415
1416     my $vmname = $conf->{name} || "vm$vmid";
1417
1418     my $netdev = "";
1419     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1420
1421     if ($net->{bridge}) {
1422         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1423     } else {
1424         $netdev = "type=user,id=$netid,hostname=$vmname";
1425     }
1426
1427     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1428
1429     return $netdev;
1430 }
1431
1432 sub drive_is_cdrom {
1433     my ($drive) = @_;
1434
1435     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1436
1437 }
1438
1439 sub parse_numa {
1440     my ($data) = @_;
1441
1442     my $res = {};
1443
1444     foreach my $kvp (split(/,/, $data)) {
1445
1446         if ($kvp =~ m/^memory=(\S+)$/) {
1447             $res->{memory} = $1;
1448         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1449             $res->{policy} = $1;
1450         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1451             $res->{cpus}->{start} = $1;
1452             $res->{cpus}->{end} = $3;
1453         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1454             $res->{hostnodes}->{start} = $1;
1455             $res->{hostnodes}->{end} = $3;
1456         } else {
1457             return undef;
1458         }
1459     }
1460
1461     return $res;
1462 }
1463
1464 sub parse_hostpci {
1465     my ($value) = @_;
1466
1467     return undef if !$value;
1468
1469
1470     my @list = split(/,/, $value);
1471     my $found;
1472
1473     my $res = {};
1474     foreach my $kv (@list) {
1475
1476         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1477             $found = 1;
1478             if(defined($4)){
1479                 push @{$res->{pciid}}, { id => $2 , function => $4};
1480
1481             }else{
1482                 my $pcidevices = lspci($2);
1483                 $res->{pciid} = $pcidevices->{$2};
1484             }
1485         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1486             $res->{rombar} = $1;
1487         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1488             $res->{'x-vga'} = $1;
1489         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1490             $res->{pcie} = 1 if $1 == 1;
1491         } else {
1492             warn "unknown hostpci setting '$kv'\n";
1493         }
1494     }
1495
1496     return undef if !$found;
1497
1498     return $res;
1499 }
1500
1501 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1502 sub parse_net {
1503     my ($data) = @_;
1504
1505     my $res = {};
1506
1507     foreach my $kvp (split(/,/, $data)) {
1508
1509         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1510             my $model = lc($1);
1511             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1512             $res->{model} = $model;
1513             $res->{macaddr} = $mac;
1514         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1515             $res->{bridge} = $1;
1516         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1517             $res->{queues} = $1;
1518         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1519             $res->{rate} = $1;
1520         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1521             $res->{tag} = $1;
1522         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1523             $res->{trunks} = $1;
1524         } elsif ($kvp =~ m/^firewall=([01])$/) {
1525             $res->{firewall} = $1;
1526         } elsif ($kvp =~ m/^link_down=([01])$/) {
1527             $res->{link_down} = $1;
1528         } else {
1529             return undef;
1530         }
1531
1532     }
1533
1534     return undef if !$res->{model};
1535
1536     return $res;
1537 }
1538
1539 sub print_net {
1540     my $net = shift;
1541
1542     my $res = "$net->{model}";
1543     $res .= "=$net->{macaddr}" if $net->{macaddr};
1544     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1545     $res .= ",rate=$net->{rate}" if $net->{rate};
1546     $res .= ",tag=$net->{tag}" if $net->{tag};
1547     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1548     $res .= ",firewall=1" if $net->{firewall};
1549     $res .= ",link_down=1" if $net->{link_down};
1550     $res .= ",queues=$net->{queues}" if $net->{queues};
1551
1552     return $res;
1553 }
1554
1555 sub add_random_macs {
1556     my ($settings) = @_;
1557
1558     foreach my $opt (keys %$settings) {
1559         next if $opt !~ m/^net(\d+)$/;
1560         my $net = parse_net($settings->{$opt});
1561         next if !$net;
1562         $settings->{$opt} = print_net($net);
1563     }
1564 }
1565
1566 sub add_unused_volume {
1567     my ($config, $volid) = @_;
1568
1569     my $key;
1570     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1571         my $test = "unused$ind";
1572         if (my $vid = $config->{$test}) {
1573             return if $vid eq $volid; # do not add duplicates
1574         } else {
1575             $key = $test;
1576         }
1577     }
1578
1579     die "To many unused volume - please delete them first.\n" if !$key;
1580
1581     $config->{$key} = $volid;
1582
1583     return $key;
1584 }
1585
1586 sub vm_is_volid_owner {
1587     my ($storecfg, $vmid, $volid) = @_;
1588
1589     if ($volid !~  m|^/|) {
1590         my ($path, $owner);
1591         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1592         if ($owner && ($owner == $vmid)) {
1593             return 1;
1594         }
1595     }
1596
1597     return undef;
1598 }
1599
1600 sub split_flagged_list {
1601     my $text = shift || '';
1602     $text =~ s/[,;]/ /g;
1603     $text =~ s/^\s+//;
1604     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1605 }
1606
1607 sub join_flagged_list {
1608     my ($how, $lst) = @_;
1609     join $how, map { $lst->{$_} . $_ } keys %$lst;
1610 }
1611
1612 sub vmconfig_delete_pending_option {
1613     my ($conf, $key, $force) = @_;
1614
1615     delete $conf->{pending}->{$key};
1616     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1617     $pending_delete_hash->{$key} = $force ? '!' : '';
1618     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1619 }
1620
1621 sub vmconfig_undelete_pending_option {
1622     my ($conf, $key) = @_;
1623
1624     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1625     delete $pending_delete_hash->{$key};
1626
1627     if (%$pending_delete_hash) {
1628         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1629     } else {
1630         delete $conf->{pending}->{delete};
1631     }
1632 }
1633
1634 sub vmconfig_register_unused_drive {
1635     my ($storecfg, $vmid, $conf, $drive) = @_;
1636
1637     if (!drive_is_cdrom($drive)) {
1638         my $volid = $drive->{file};
1639         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1640             add_unused_volume($conf, $volid, $vmid);
1641         }
1642     }
1643 }
1644
1645 sub vmconfig_cleanup_pending {
1646     my ($conf) = @_;
1647
1648     # remove pending changes when nothing changed
1649     my $changes;
1650     foreach my $opt (keys %{$conf->{pending}}) {
1651         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1652             $changes = 1;
1653             delete $conf->{pending}->{$opt};
1654         }
1655     }
1656
1657     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1658     my $pending_delete_hash = {};
1659     while (my ($opt, $force) = each %$current_delete_hash) {
1660         if (defined($conf->{$opt})) {
1661             $pending_delete_hash->{$opt} = $force;
1662         } else {
1663             $changes = 1;
1664         }
1665     }
1666
1667     if (%$pending_delete_hash) {
1668         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1669     } else {
1670         delete $conf->{pending}->{delete};
1671     }
1672
1673     return $changes;
1674 }
1675
1676 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1677 my $smbios1_desc = {
1678     uuid => {
1679         type => 'string',
1680         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1681         format_description => 'UUID',
1682         optional => 1,
1683     },
1684     version => {
1685         type => 'string',
1686         pattern => '\S+',
1687         format_description => 'str',
1688         optional => 1,
1689     },
1690     serial => {
1691         type => 'string',
1692         pattern => '\S+',
1693         format_description => 'str',
1694         optional => 1,
1695     },
1696     manufacturer => {
1697         type => 'string',
1698         pattern => '\S+',
1699         format_description => 'name',
1700         optional => 1,
1701     },
1702     product => {
1703         type => 'string',
1704         pattern => '\S+',
1705         format_description => 'name',
1706         optional => 1,
1707     },
1708     sku => {
1709         type => 'string',
1710         pattern => '\S+',
1711         format_description => 'str',
1712         optional => 1,
1713     },
1714     family => {
1715         type => 'string',
1716         pattern => '\S+',
1717         format_description => 'str',
1718         optional => 1,
1719     },
1720 };
1721
1722 sub parse_smbios1 {
1723     my ($data) = @_;
1724
1725     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1726     warn $@ if $@;
1727     return $res;
1728 }
1729
1730 sub print_smbios1 {
1731     my ($smbios1) = @_;
1732     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1733 }
1734
1735 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1736
1737 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1738 sub verify_bootdisk {
1739     my ($value, $noerr) = @_;
1740
1741     return $value if valid_drivename($value);
1742
1743     return undef if $noerr;
1744
1745     die "invalid boot disk '$value'\n";
1746 }
1747
1748 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1749 sub verify_numa {
1750     my ($value, $noerr) = @_;
1751
1752     return $value if parse_numa($value);
1753
1754     return undef if $noerr;
1755
1756     die "unable to parse numa options\n";
1757 }
1758
1759 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1760 sub verify_net {
1761     my ($value, $noerr) = @_;
1762
1763     return $value if parse_net($value);
1764
1765     return undef if $noerr;
1766
1767     die "unable to parse network options\n";
1768 }
1769
1770 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1771 sub verify_hostpci {
1772     my ($value, $noerr) = @_;
1773
1774     return $value if parse_hostpci($value);
1775
1776     return undef if $noerr;
1777
1778     die "unable to parse pci id\n";
1779 }
1780
1781 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1782 sub verify_watchdog {
1783     my ($value, $noerr) = @_;
1784
1785     return $value if parse_watchdog($value);
1786
1787     return undef if $noerr;
1788
1789     die "unable to parse watchdog options\n";
1790 }
1791
1792 sub parse_watchdog {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my $res = {};
1798
1799     foreach my $p (split(/,/, $value)) {
1800         next if $p =~ m/^\s*$/;
1801
1802         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1803             $res->{model} = $2;
1804         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1805             $res->{action} = $2;
1806         } else {
1807             return undef;
1808         }
1809     }
1810
1811     return $res;
1812 }
1813
1814 sub parse_usb_device {
1815     my ($value) = @_;
1816
1817     return undef if !$value;
1818
1819     my @dl = split(/,/, $value);
1820     my $found;
1821
1822     my $res = {};
1823     foreach my $v (@dl) {
1824         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1825             $found = 1;
1826             $res->{vendorid} = $2;
1827             $res->{productid} = $4;
1828         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1829             $found = 1;
1830             $res->{hostbus} = $1;
1831             $res->{hostport} = $2;
1832         } elsif ($v =~ m/^spice$/) {
1833             $found = 1;
1834             $res->{spice} = 1;
1835         } elsif ($v =~ m/^usb3=yes$/) {
1836             $res->{usb3} = 1;
1837         } else {
1838             return undef;
1839         }
1840     }
1841     return undef if !$found;
1842
1843     return $res;
1844 }
1845
1846 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1847 sub verify_usb_device {
1848     my ($value, $noerr) = @_;
1849
1850     return $value if parse_usb_device($value);
1851
1852     return undef if $noerr;
1853
1854     die "unable to parse usb device\n";
1855 }
1856
1857 # add JSON properties for create and set function
1858 sub json_config_properties {
1859     my $prop = shift;
1860
1861     foreach my $opt (keys %$confdesc) {
1862         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1863         $prop->{$opt} = $confdesc->{$opt};
1864     }
1865
1866     return $prop;
1867 }
1868
1869 sub check_type {
1870     my ($key, $value) = @_;
1871
1872     die "unknown setting '$key'\n" if !$confdesc->{$key};
1873
1874     my $type = $confdesc->{$key}->{type};
1875
1876     if (!defined($value)) {
1877         die "got undefined value\n";
1878     }
1879
1880     if ($value =~ m/[\n\r]/) {
1881         die "property contains a line feed\n";
1882     }
1883
1884     if ($type eq 'boolean') {
1885         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1886         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1887         die "type check ('boolean') failed - got '$value'\n";
1888     } elsif ($type eq 'integer') {
1889         return int($1) if $value =~ m/^(\d+)$/;
1890         die "type check ('integer') failed - got '$value'\n";
1891     } elsif ($type eq 'number') {
1892         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1893         die "type check ('number') failed - got '$value'\n";
1894     } elsif ($type eq 'string') {
1895         if (my $fmt = $confdesc->{$key}->{format}) {
1896             if ($fmt eq 'pve-qm-drive') {
1897                 # special case - we need to pass $key to parse_drive()
1898                 my $drive = parse_drive($key, $value);
1899                 return $value if $drive;
1900                 die "unable to parse drive options\n";
1901             }
1902             PVE::JSONSchema::check_format($fmt, $value);
1903             return $value;
1904         }
1905         $value =~ s/^\"(.*)\"$/$1/;
1906         return $value;
1907     } else {
1908         die "internal error"
1909     }
1910 }
1911
1912 sub lock_config_full {
1913     my ($vmid, $timeout, $code, @param) = @_;
1914
1915     my $filename = config_file_lock($vmid);
1916
1917     my $res = lock_file($filename, $timeout, $code, @param);
1918
1919     die $@ if $@;
1920
1921     return $res;
1922 }
1923
1924 sub lock_config_mode {
1925     my ($vmid, $timeout, $shared, $code, @param) = @_;
1926
1927     my $filename = config_file_lock($vmid);
1928
1929     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1930
1931     die $@ if $@;
1932
1933     return $res;
1934 }
1935
1936 sub lock_config {
1937     my ($vmid, $code, @param) = @_;
1938
1939     return lock_config_full($vmid, 10, $code, @param);
1940 }
1941
1942 sub cfs_config_path {
1943     my ($vmid, $node) = @_;
1944
1945     $node = $nodename if !$node;
1946     return "nodes/$node/qemu-server/$vmid.conf";
1947 }
1948
1949 sub check_iommu_support{
1950     #fixme : need to check IOMMU support
1951     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1952
1953     my $iommu=1;
1954     return $iommu;
1955
1956 }
1957
1958 sub config_file {
1959     my ($vmid, $node) = @_;
1960
1961     my $cfspath = cfs_config_path($vmid, $node);
1962     return "/etc/pve/$cfspath";
1963 }
1964
1965 sub config_file_lock {
1966     my ($vmid) = @_;
1967
1968     return "$lock_dir/lock-$vmid.conf";
1969 }
1970
1971 sub touch_config {
1972     my ($vmid) = @_;
1973
1974     my $conf = config_file($vmid);
1975     utime undef, undef, $conf;
1976 }
1977
1978 sub destroy_vm {
1979     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1980
1981     my $conffile = config_file($vmid);
1982
1983     my $conf = load_config($vmid);
1984
1985     check_lock($conf) if !$skiplock;
1986
1987     # only remove disks owned by this VM
1988     foreach_drive($conf, sub {
1989         my ($ds, $drive) = @_;
1990
1991         return if drive_is_cdrom($drive);
1992
1993         my $volid = $drive->{file};
1994
1995         return if !$volid || $volid =~ m|^/|;
1996
1997         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1998         return if !$path || !$owner || ($owner != $vmid);
1999
2000         PVE::Storage::vdisk_free($storecfg, $volid);
2001     });
2002
2003     if ($keep_empty_config) {
2004         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2005     } else {
2006         unlink $conffile;
2007     }
2008
2009     # also remove unused disk
2010     eval {
2011         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2012
2013         eval {
2014             PVE::Storage::foreach_volid($dl, sub {
2015                 my ($volid, $sid, $volname, $d) = @_;
2016                 PVE::Storage::vdisk_free($storecfg, $volid);
2017             });
2018         };
2019         warn $@ if $@;
2020
2021     };
2022     warn $@ if $@;
2023 }
2024
2025 sub load_config {
2026     my ($vmid, $node) = @_;
2027
2028     my $cfspath = cfs_config_path($vmid, $node);
2029
2030     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2031
2032     die "no such VM ('$vmid')\n" if !defined($conf);
2033
2034     return $conf;
2035 }
2036
2037 sub parse_vm_config {
2038     my ($filename, $raw) = @_;
2039
2040     return undef if !defined($raw);
2041
2042     my $res = {
2043         digest => Digest::SHA::sha1_hex($raw),
2044         snapshots => {},
2045         pending => {},
2046     };
2047
2048     $filename =~ m|/qemu-server/(\d+)\.conf$|
2049         || die "got strange filename '$filename'";
2050
2051     my $vmid = $1;
2052
2053     my $conf = $res;
2054     my $descr;
2055     my $section = '';
2056
2057     my @lines = split(/\n/, $raw);
2058     foreach my $line (@lines) {
2059         next if $line =~ m/^\s*$/;
2060
2061         if ($line =~ m/^\[PENDING\]\s*$/i) {
2062             $section = 'pending';
2063             if (defined($descr)) {
2064                 $descr =~ s/\s+$//;
2065                 $conf->{description} = $descr;
2066             }
2067             $descr = undef;
2068             $conf = $res->{$section} = {};
2069             next;
2070
2071         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2072             $section = $1;
2073             if (defined($descr)) {
2074                 $descr =~ s/\s+$//;
2075                 $conf->{description} = $descr;
2076             }
2077             $descr = undef;
2078             $conf = $res->{snapshots}->{$section} = {};
2079             next;
2080         }
2081
2082         if ($line =~ m/^\#(.*)\s*$/) {
2083             $descr = '' if !defined($descr);
2084             $descr .= PVE::Tools::decode_text($1) . "\n";
2085             next;
2086         }
2087
2088         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2089             $descr = '' if !defined($descr);
2090             $descr .= PVE::Tools::decode_text($2);
2091         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2092             $conf->{snapstate} = $1;
2093         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2094             my $key = $1;
2095             my $value = $2;
2096             $conf->{$key} = $value;
2097         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2098             my $value = $1;
2099             if ($section eq 'pending') {
2100                 $conf->{delete} = $value; # we parse this later
2101             } else {
2102                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2103             }
2104         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2105             my $key = $1;
2106             my $value = $2;
2107             eval { $value = check_type($key, $value); };
2108             if ($@) {
2109                 warn "vm $vmid - unable to parse value of '$key' - $@";
2110             } else {
2111                 my $fmt = $confdesc->{$key}->{format};
2112                 if ($fmt && $fmt eq 'pve-qm-drive') {
2113                     my $v = parse_drive($key, $value);
2114                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2115                         $v->{file} = $volid;
2116                         $value = print_drive($vmid, $v);
2117                     } else {
2118                         warn "vm $vmid - unable to parse value of '$key'\n";
2119                         next;
2120                     }
2121                 }
2122
2123                 if ($key eq 'cdrom') {
2124                     $conf->{ide2} = $value;
2125                 } else {
2126                     $conf->{$key} = $value;
2127                 }
2128             }
2129         }
2130     }
2131
2132     if (defined($descr)) {
2133         $descr =~ s/\s+$//;
2134         $conf->{description} = $descr;
2135     }
2136     delete $res->{snapstate}; # just to be sure
2137
2138     return $res;
2139 }
2140
2141 sub write_vm_config {
2142     my ($filename, $conf) = @_;
2143
2144     delete $conf->{snapstate}; # just to be sure
2145
2146     if ($conf->{cdrom}) {
2147         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2148         $conf->{ide2} = $conf->{cdrom};
2149         delete $conf->{cdrom};
2150     }
2151
2152     # we do not use 'smp' any longer
2153     if ($conf->{sockets}) {
2154         delete $conf->{smp};
2155     } elsif ($conf->{smp}) {
2156         $conf->{sockets} = $conf->{smp};
2157         delete $conf->{cores};
2158         delete $conf->{smp};
2159     }
2160
2161     my $used_volids = {};
2162
2163     my $cleanup_config = sub {
2164         my ($cref, $pending, $snapname) = @_;
2165
2166         foreach my $key (keys %$cref) {
2167             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2168                 $key eq 'snapstate' || $key eq 'pending';
2169             my $value = $cref->{$key};
2170             if ($key eq 'delete') {
2171                 die "propertry 'delete' is only allowed in [PENDING]\n"
2172                     if !$pending;
2173                 # fixme: check syntax?
2174                 next;
2175             }
2176             eval { $value = check_type($key, $value); };
2177             die "unable to parse value of '$key' - $@" if $@;
2178
2179             $cref->{$key} = $value;
2180
2181             if (!$snapname && valid_drivename($key)) {
2182                 my $drive = parse_drive($key, $value);
2183                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2184             }
2185         }
2186     };
2187
2188     &$cleanup_config($conf);
2189
2190     &$cleanup_config($conf->{pending}, 1);
2191
2192     foreach my $snapname (keys %{$conf->{snapshots}}) {
2193         die "internal error" if $snapname eq 'pending';
2194         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2195     }
2196
2197     # remove 'unusedX' settings if we re-add a volume
2198     foreach my $key (keys %$conf) {
2199         my $value = $conf->{$key};
2200         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2201             delete $conf->{$key};
2202         }
2203     }
2204
2205     my $generate_raw_config = sub {
2206         my ($conf, $pending) = @_;
2207
2208         my $raw = '';
2209
2210         # add description as comment to top of file
2211         if (defined(my $descr = $conf->{description})) {
2212             if ($descr) {
2213                 foreach my $cl (split(/\n/, $descr)) {
2214                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2215                 }
2216             } else {
2217                 $raw .= "#\n" if $pending;
2218             }
2219         }
2220
2221         foreach my $key (sort keys %$conf) {
2222             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2223             $raw .= "$key: $conf->{$key}\n";
2224         }
2225         return $raw;
2226     };
2227
2228     my $raw = &$generate_raw_config($conf);
2229
2230     if (scalar(keys %{$conf->{pending}})){
2231         $raw .= "\n[PENDING]\n";
2232         $raw .= &$generate_raw_config($conf->{pending}, 1);
2233     }
2234
2235     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2236         $raw .= "\n[$snapname]\n";
2237         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2238     }
2239
2240     return $raw;
2241 }
2242
2243 sub update_config_nolock {
2244     my ($vmid, $conf, $skiplock) = @_;
2245
2246     check_lock($conf) if !$skiplock;
2247
2248     my $cfspath = cfs_config_path($vmid);
2249
2250     PVE::Cluster::cfs_write_file($cfspath, $conf);
2251 }
2252
2253 sub update_config {
2254     my ($vmid, $conf, $skiplock) = @_;
2255
2256     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2257 }
2258
2259 sub load_defaults {
2260
2261     my $res = {};
2262
2263     # we use static defaults from our JSON schema configuration
2264     foreach my $key (keys %$confdesc) {
2265         if (defined(my $default = $confdesc->{$key}->{default})) {
2266             $res->{$key} = $default;
2267         }
2268     }
2269
2270     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2271     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2272
2273     return $res;
2274 }
2275
2276 sub config_list {
2277     my $vmlist = PVE::Cluster::get_vmlist();
2278     my $res = {};
2279     return $res if !$vmlist || !$vmlist->{ids};
2280     my $ids = $vmlist->{ids};
2281
2282     foreach my $vmid (keys %$ids) {
2283         my $d = $ids->{$vmid};
2284         next if !$d->{node} || $d->{node} ne $nodename;
2285         next if !$d->{type} || $d->{type} ne 'qemu';
2286         $res->{$vmid}->{exists} = 1;
2287     }
2288     return $res;
2289 }
2290
2291 # test if VM uses local resources (to prevent migration)
2292 sub check_local_resources {
2293     my ($conf, $noerr) = @_;
2294
2295     my $loc_res = 0;
2296
2297     $loc_res = 1 if $conf->{hostusb}; # old syntax
2298     $loc_res = 1 if $conf->{hostpci}; # old syntax
2299
2300     foreach my $k (keys %$conf) {
2301         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2302         # sockets are safe: they will recreated be on the target side post-migrate
2303         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2304         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2305     }
2306
2307     die "VM uses local resources\n" if $loc_res && !$noerr;
2308
2309     return $loc_res;
2310 }
2311
2312 # check if used storages are available on all nodes (use by migrate)
2313 sub check_storage_availability {
2314     my ($storecfg, $conf, $node) = @_;
2315
2316     foreach_drive($conf, sub {
2317         my ($ds, $drive) = @_;
2318
2319         my $volid = $drive->{file};
2320         return if !$volid;
2321
2322         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2323         return if !$sid;
2324
2325         # check if storage is available on both nodes
2326         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2327         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2328    });
2329 }
2330
2331 # list nodes where all VM images are available (used by has_feature API)
2332 sub shared_nodes {
2333     my ($conf, $storecfg) = @_;
2334
2335     my $nodelist = PVE::Cluster::get_nodelist();
2336     my $nodehash = { map { $_ => 1 } @$nodelist };
2337     my $nodename = PVE::INotify::nodename();
2338
2339     foreach_drive($conf, sub {
2340         my ($ds, $drive) = @_;
2341
2342         my $volid = $drive->{file};
2343         return if !$volid;
2344
2345         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2346         if ($storeid) {
2347             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2348             if ($scfg->{disable}) {
2349                 $nodehash = {};
2350             } elsif (my $avail = $scfg->{nodes}) {
2351                 foreach my $node (keys %$nodehash) {
2352                     delete $nodehash->{$node} if !$avail->{$node};
2353                 }
2354             } elsif (!$scfg->{shared}) {
2355                 foreach my $node (keys %$nodehash) {
2356                     delete $nodehash->{$node} if $node ne $nodename
2357                 }
2358             }
2359         }
2360     });
2361
2362     return $nodehash
2363 }
2364
2365 sub check_lock {
2366     my ($conf) = @_;
2367
2368     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2369 }
2370
2371 sub check_cmdline {
2372     my ($pidfile, $pid) = @_;
2373
2374     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2375     if (defined($fh)) {
2376         my $line = <$fh>;
2377         $fh->close;
2378         return undef if !$line;
2379         my @param = split(/\0/, $line);
2380
2381         my $cmd = $param[0];
2382         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2383
2384         for (my $i = 0; $i < scalar (@param); $i++) {
2385             my $p = $param[$i];
2386             next if !$p;
2387             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2388                 my $p = $param[$i+1];
2389                 return 1 if $p && ($p eq $pidfile);
2390                 return undef;
2391             }
2392         }
2393     }
2394     return undef;
2395 }
2396
2397 sub check_running {
2398     my ($vmid, $nocheck, $node) = @_;
2399
2400     my $filename = config_file($vmid, $node);
2401
2402     die "unable to find configuration file for VM $vmid - no such machine\n"
2403         if !$nocheck && ! -f $filename;
2404
2405     my $pidfile = pidfile_name($vmid);
2406
2407     if (my $fd = IO::File->new("<$pidfile")) {
2408         my $st = stat($fd);
2409         my $line = <$fd>;
2410         close($fd);
2411
2412         my $mtime = $st->mtime;
2413         if ($mtime > time()) {
2414             warn "file '$filename' modified in future\n";
2415         }
2416
2417         if ($line =~ m/^(\d+)$/) {
2418             my $pid = $1;
2419             if (check_cmdline($pidfile, $pid)) {
2420                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2421                     return $pid;
2422                 }
2423             }
2424         }
2425     }
2426
2427     return undef;
2428 }
2429
2430 sub vzlist {
2431
2432     my $vzlist = config_list();
2433
2434     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2435
2436     while (defined(my $de = $fd->read)) {
2437         next if $de !~ m/^(\d+)\.pid$/;
2438         my $vmid = $1;
2439         next if !defined($vzlist->{$vmid});
2440         if (my $pid = check_running($vmid)) {
2441             $vzlist->{$vmid}->{pid} = $pid;
2442         }
2443     }
2444
2445     return $vzlist;
2446 }
2447
2448 sub disksize {
2449     my ($storecfg, $conf) = @_;
2450
2451     my $bootdisk = $conf->{bootdisk};
2452     return undef if !$bootdisk;
2453     return undef if !valid_drivename($bootdisk);
2454
2455     return undef if !$conf->{$bootdisk};
2456
2457     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2458     return undef if !defined($drive);
2459
2460     return undef if drive_is_cdrom($drive);
2461
2462     my $volid = $drive->{file};
2463     return undef if !$volid;
2464
2465     return $drive->{size};
2466 }
2467
2468 my $last_proc_pid_stat;
2469
2470 # get VM status information
2471 # This must be fast and should not block ($full == false)
2472 # We only query KVM using QMP if $full == true (this can be slow)
2473 sub vmstatus {
2474     my ($opt_vmid, $full) = @_;
2475
2476     my $res = {};
2477
2478     my $storecfg = PVE::Storage::config();
2479
2480     my $list = vzlist();
2481     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2482
2483     my $cpucount = $cpuinfo->{cpus} || 1;
2484
2485     foreach my $vmid (keys %$list) {
2486         next if $opt_vmid && ($vmid ne $opt_vmid);
2487
2488         my $cfspath = cfs_config_path($vmid);
2489         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2490
2491         my $d = {};
2492         $d->{pid} = $list->{$vmid}->{pid};
2493
2494         # fixme: better status?
2495         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2496
2497         my $size = disksize($storecfg, $conf);
2498         if (defined($size)) {
2499             $d->{disk} = 0; # no info available
2500             $d->{maxdisk} = $size;
2501         } else {
2502             $d->{disk} = 0;
2503             $d->{maxdisk} = 0;
2504         }
2505
2506         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2507         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2508         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2509
2510         $d->{name} = $conf->{name} || "VM $vmid";
2511         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2512
2513         if ($conf->{balloon}) {
2514             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2515             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2516         }
2517
2518         $d->{uptime} = 0;
2519         $d->{cpu} = 0;
2520         $d->{mem} = 0;
2521
2522         $d->{netout} = 0;
2523         $d->{netin} = 0;
2524
2525         $d->{diskread} = 0;
2526         $d->{diskwrite} = 0;
2527
2528         $d->{template} = is_template($conf);
2529
2530         $res->{$vmid} = $d;
2531     }
2532
2533     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2534     foreach my $dev (keys %$netdev) {
2535         next if $dev !~ m/^tap([1-9]\d*)i/;
2536         my $vmid = $1;
2537         my $d = $res->{$vmid};
2538         next if !$d;
2539
2540         $d->{netout} += $netdev->{$dev}->{receive};
2541         $d->{netin} += $netdev->{$dev}->{transmit};
2542
2543         if ($full) {
2544             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2545             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2546         }
2547
2548     }
2549
2550     my $ctime = gettimeofday;
2551
2552     foreach my $vmid (keys %$list) {
2553
2554         my $d = $res->{$vmid};
2555         my $pid = $d->{pid};
2556         next if !$pid;
2557
2558         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2559         next if !$pstat; # not running
2560
2561         my $used = $pstat->{utime} + $pstat->{stime};
2562
2563         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2564
2565         if ($pstat->{vsize}) {
2566             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2567         }
2568
2569         my $old = $last_proc_pid_stat->{$pid};
2570         if (!$old) {
2571             $last_proc_pid_stat->{$pid} = {
2572                 time => $ctime,
2573                 used => $used,
2574                 cpu => 0,
2575             };
2576             next;
2577         }
2578
2579         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2580
2581         if ($dtime > 1000) {
2582             my $dutime = $used -  $old->{used};
2583
2584             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2585             $last_proc_pid_stat->{$pid} = {
2586                 time => $ctime,
2587                 used => $used,
2588                 cpu => $d->{cpu},
2589             };
2590         } else {
2591             $d->{cpu} = $old->{cpu};
2592         }
2593     }
2594
2595     return $res if !$full;
2596
2597     my $qmpclient = PVE::QMPClient->new();
2598
2599     my $ballooncb = sub {
2600         my ($vmid, $resp) = @_;
2601
2602         my $info = $resp->{'return'};
2603         return if !$info->{max_mem};
2604
2605         my $d = $res->{$vmid};
2606
2607         # use memory assigned to VM
2608         $d->{maxmem} = $info->{max_mem};
2609         $d->{balloon} = $info->{actual};
2610
2611         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2612             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2613             $d->{freemem} = $info->{free_mem};
2614         }
2615
2616         $d->{ballooninfo} = $info;
2617     };
2618
2619     my $blockstatscb = sub {
2620         my ($vmid, $resp) = @_;
2621         my $data = $resp->{'return'} || [];
2622         my $totalrdbytes = 0;
2623         my $totalwrbytes = 0;
2624
2625         for my $blockstat (@$data) {
2626             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2627             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2628
2629             $blockstat->{device} =~ s/drive-//;
2630             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2631         }
2632         $res->{$vmid}->{diskread} = $totalrdbytes;
2633         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2634     };
2635
2636     my $statuscb = sub {
2637         my ($vmid, $resp) = @_;
2638
2639         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2640         # this fails if ballon driver is not loaded, so this must be
2641         # the last commnand (following command are aborted if this fails).
2642         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2643
2644         my $status = 'unknown';
2645         if (!defined($status = $resp->{'return'}->{status})) {
2646             warn "unable to get VM status\n";
2647             return;
2648         }
2649
2650         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2651     };
2652
2653     foreach my $vmid (keys %$list) {
2654         next if $opt_vmid && ($vmid ne $opt_vmid);
2655         next if !$res->{$vmid}->{pid}; # not running
2656         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2657     }
2658
2659     $qmpclient->queue_execute(undef, 1);
2660
2661     foreach my $vmid (keys %$list) {
2662         next if $opt_vmid && ($vmid ne $opt_vmid);
2663         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2664     }
2665
2666     return $res;
2667 }
2668
2669 sub foreach_dimm {
2670     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2671
2672     my $dimm_id = 0;
2673     my $current_size = 1024;
2674     my $dimm_size = 512;
2675     return if $current_size == $memory;
2676
2677     for (my $j = 0; $j < 8; $j++) {
2678         for (my $i = 0; $i < 32; $i++) {
2679             my $name = "dimm${dimm_id}";
2680             $dimm_id++;
2681             my $numanode = $i % $sockets;
2682             $current_size += $dimm_size;
2683             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2684             return  $current_size if $current_size >= $memory;
2685         }
2686         $dimm_size *= 2;
2687     }
2688 }
2689
2690 sub foreach_reverse_dimm {
2691     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2692
2693     my $dimm_id = 253;
2694     my $current_size = 4177920;
2695     my $dimm_size = 65536;
2696     return if $current_size == $memory;
2697
2698     for (my $j = 0; $j < 8; $j++) {
2699         for (my $i = 0; $i < 32; $i++) {
2700             my $name = "dimm${dimm_id}";
2701             $dimm_id--;
2702             my $numanode = $i % $sockets;
2703             $current_size -= $dimm_size;
2704             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2705             return  $current_size if $current_size <= $memory;
2706         }
2707         $dimm_size /= 2;
2708     }
2709 }
2710
2711 sub foreach_drive {
2712     my ($conf, $func) = @_;
2713
2714     foreach my $ds (keys %$conf) {
2715         next if !valid_drivename($ds);
2716
2717         my $drive = parse_drive($ds, $conf->{$ds});
2718         next if !$drive;
2719
2720         &$func($ds, $drive);
2721     }
2722 }
2723
2724 sub foreach_volid {
2725     my ($conf, $func) = @_;
2726
2727     my $volhash = {};
2728
2729     my $test_volid = sub {
2730         my ($volid, $is_cdrom) = @_;
2731
2732         return if !$volid;
2733
2734         $volhash->{$volid} = $is_cdrom || 0;
2735     };
2736
2737     foreach_drive($conf, sub {
2738         my ($ds, $drive) = @_;
2739         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2740     });
2741
2742     foreach my $snapname (keys %{$conf->{snapshots}}) {
2743         my $snap = $conf->{snapshots}->{$snapname};
2744         &$test_volid($snap->{vmstate}, 0);
2745         foreach_drive($snap, sub {
2746             my ($ds, $drive) = @_;
2747             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2748         });
2749     }
2750
2751     foreach my $volid (keys %$volhash) {
2752         &$func($volid, $volhash->{$volid});
2753     }
2754 }
2755
2756 sub vga_conf_has_spice {
2757     my ($vga) = @_;
2758
2759     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2760
2761     return $1 || 1;
2762 }
2763
2764 sub config_to_command {
2765     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2766
2767     my $cmd = [];
2768     my $globalFlags = [];
2769     my $machineFlags = [];
2770     my $rtcFlags = [];
2771     my $cpuFlags = [];
2772     my $devices = [];
2773     my $pciaddr = '';
2774     my $bridges = {};
2775     my $kvmver = kvm_user_version();
2776     my $vernum = 0; # unknown
2777     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2778         $vernum = $1*1000000+$2*1000;
2779     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2780         $vernum = $1*1000000+$2*1000+$3;
2781     }
2782
2783     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2784
2785     my $have_ovz = -f '/proc/vz/vestat';
2786
2787     my $q35 = machine_type_is_q35($conf);
2788     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2789     my $machine_type = $forcemachine || $conf->{machine};
2790     my $use_old_bios_files = undef;
2791     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2792
2793     my $cpuunits = defined($conf->{cpuunits}) ?
2794             $conf->{cpuunits} : $defaults->{cpuunits};
2795
2796     push @$cmd, '/usr/bin/systemd-run';
2797     push @$cmd, '--scope';
2798     push @$cmd, '--slice', "qemu";
2799     push @$cmd, '--unit', $vmid;
2800     # set KillMode=none, so that systemd don't kill those scopes
2801     # at shutdown (pve-manager service should stop the VMs instead)
2802     push @$cmd, '-p', "KillMode=none";
2803     push @$cmd, '-p', "CPUShares=$cpuunits";
2804     if ($conf->{cpulimit}) {
2805         my $cpulimit = int($conf->{cpulimit} * 100);
2806         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2807     }
2808
2809     push @$cmd, '/usr/bin/kvm';
2810
2811     push @$cmd, '-id', $vmid;
2812
2813     my $use_virtio = 0;
2814
2815     my $qmpsocket = qmp_socket($vmid);
2816     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2817     push @$cmd, '-mon', "chardev=qmp,mode=control";
2818
2819     my $socket = vnc_socket($vmid);
2820     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2821
2822     push @$cmd, '-pidfile' , pidfile_name($vmid);
2823
2824     push @$cmd, '-daemonize';
2825
2826     if ($conf->{smbios1}) {
2827         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2828     }
2829
2830     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2831         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2832         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2833         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2834         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2835         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2836         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2837     }
2838
2839     if ($q35) {
2840         # the q35 chipset support native usb2, so we enable usb controller
2841         # by default for this machine type
2842         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2843     } else {
2844         $pciaddr = print_pci_addr("piix3", $bridges);
2845         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2846
2847         my $use_usb2 = 0;
2848         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2849             next if !$conf->{"usb$i"};
2850             my $d = parse_usb_device($conf->{"usb$i"});
2851             next if $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2852             $use_usb2 = 1;
2853         }
2854         # include usb device config
2855         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2856     }
2857
2858     # add usb3 controller if needed
2859
2860     my $use_usb3 = 0;
2861     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2862         next if !$conf->{"usb$i"};
2863         my $d = parse_usb_device($conf->{"usb$i"});
2864         next if !$d->{usb3};
2865         $use_usb3 = 1;
2866     }
2867
2868     $pciaddr = print_pci_addr("xhci", $bridges);
2869     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2870
2871     my $vga = $conf->{vga};
2872
2873     my $qxlnum = vga_conf_has_spice($vga);
2874     $vga = 'qxl' if $qxlnum;
2875
2876     if (!$vga) {
2877         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2878                                 $conf->{ostype} eq 'win7' ||
2879                                 $conf->{ostype} eq 'w2k8')) {
2880             $vga = 'std';
2881         } else {
2882             $vga = 'cirrus';
2883         }
2884     }
2885
2886     # enable absolute mouse coordinates (needed by vnc)
2887     my $tablet;
2888     if (defined($conf->{tablet})) {
2889         $tablet = $conf->{tablet};
2890     } else {
2891         $tablet = $defaults->{tablet};
2892         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2893         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2894     }
2895
2896     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2897
2898     my $kvm_off = 0;
2899     my $nohyperv;
2900     # host pci devices
2901     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2902         my $d = parse_hostpci($conf->{"hostpci$i"});
2903         next if !$d;
2904
2905         my $pcie = $d->{pcie};
2906         if($pcie){
2907             die "q35 machine model is not enabled" if !$q35;
2908             $pciaddr = print_pcie_addr("hostpci$i");
2909         }else{
2910             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2911         }
2912
2913         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2914         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2915         if ($xvga && $xvga ne '') {
2916             $kvm_off = 1;
2917             $vga = 'none';
2918             $nohyperv = 1;
2919             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2920                 $xvga = "";
2921             }
2922         }
2923         my $pcidevices = $d->{pciid};
2924         my $multifunction = 1 if @$pcidevices > 1;
2925
2926         my $j=0;
2927         foreach my $pcidevice (@$pcidevices) {
2928
2929             my $id = "hostpci$i";
2930             $id .= ".$j" if $multifunction;
2931             my $addr = $pciaddr;
2932             $addr .= ".$j" if $multifunction;
2933             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2934
2935             if($j == 0){
2936                 $devicestr .= "$rombar$xvga";
2937                 $devicestr .= ",multifunction=on" if $multifunction;
2938             }
2939
2940             push @$devices, '-device', $devicestr;
2941             $j++;
2942         }
2943     }
2944
2945     # usb devices
2946     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2947         my $d = parse_usb_device($conf->{"usb$i"});
2948         next if !$d;
2949
2950         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2951         my $usbbus = '';
2952         if ($d->{usb3}) {
2953             $usbbus = ',bus=xhci.0';
2954         }
2955
2956         if ($d->{vendorid} && $d->{productid}) {
2957             push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2958         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2959             push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2960         } elsif ($d->{spice}) {
2961             # usb redir support for spice, currently no usb3
2962             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2963             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2964         }
2965     }
2966
2967     # serial devices
2968     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2969         if (my $path = $conf->{"serial$i"}) {
2970             if ($path eq 'socket') {
2971                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2972                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2973                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2974             } else {
2975                 die "no such serial device\n" if ! -c $path;
2976                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2977                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2978             }
2979         }
2980     }
2981
2982     # parallel devices
2983     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2984         if (my $path = $conf->{"parallel$i"}) {
2985             die "no such parallel device\n" if ! -c $path;
2986             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2987             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2988             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2989         }
2990     }
2991
2992     my $vmname = $conf->{name} || "vm$vmid";
2993
2994     push @$cmd, '-name', $vmname;
2995
2996     my $sockets = 1;
2997     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2998     $sockets = $conf->{sockets} if  $conf->{sockets};
2999
3000     my $cores = $conf->{cores} || 1;
3001
3002     my $maxcpus = $sockets * $cores;
3003
3004     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3005
3006     my $allowed_vcpus = $cpuinfo->{cpus};
3007
3008     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3009         if ($allowed_vcpus < $maxcpus);
3010
3011     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3012
3013     push @$cmd, '-nodefaults';
3014
3015     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3016
3017     my $bootindex_hash = {};
3018     my $i = 1;
3019     foreach my $o (split(//, $bootorder)) {
3020         $bootindex_hash->{$o} = $i*100;
3021         $i++;
3022     }
3023
3024     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3025
3026     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3027
3028     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3029
3030     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3031
3032     # time drift fix
3033     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3034
3035     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3036     my $useLocaltime = $conf->{localtime};
3037
3038     if (my $ost = $conf->{ostype}) {
3039         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3040
3041         if ($ost =~ m/^w/) { # windows
3042             $useLocaltime = 1 if !defined($conf->{localtime});
3043
3044             # use time drift fix when acpi is enabled
3045             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3046                 $tdf = 1 if !defined($conf->{tdf});
3047             }
3048         }
3049
3050         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
3051             $ost eq 'wvista') {
3052             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3053             push @$cmd, '-no-hpet';
3054             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3055                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
3056                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
3057                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
3058
3059             } else {
3060                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
3061             }
3062         }
3063
3064         if ($ost eq 'win7' || $ost eq 'win8') {
3065             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
3066         }
3067     }
3068
3069     push @$rtcFlags, 'driftfix=slew' if $tdf;
3070
3071     if ($nokvm) {
3072         push @$machineFlags, 'accel=tcg';
3073     } else {
3074         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3075     }
3076
3077     if ($machine_type) {
3078         push @$machineFlags, "type=${machine_type}";
3079     }
3080
3081     if ($conf->{startdate}) {
3082         push @$rtcFlags, "base=$conf->{startdate}";
3083     } elsif ($useLocaltime) {
3084         push @$rtcFlags, 'base=localtime';
3085     }
3086
3087     my $cpu = $nokvm ? "qemu64" : "kvm64";
3088     if (my $cputype = $conf->{cpu}) {
3089         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3090             or die "Cannot parse cpu description: $cputype\n";
3091         $cpu = $cpuconf->{cputype};
3092         $kvm_off = 1 if $cpuconf->{hidden};
3093     }
3094
3095     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3096
3097     push @$cpuFlags , '-x2apic'
3098         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3099
3100     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3101
3102     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3103
3104     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3105
3106         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3107         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3108     }
3109
3110     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3111
3112     push @$cpuFlags, 'kvm=off' if $kvm_off;
3113
3114     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3115
3116     push @$cmd, '-cpu', $cpu;
3117
3118     my $memory = $conf->{memory} || $defaults->{memory};
3119     my $static_memory = 0;
3120     my $dimm_memory = 0;
3121
3122     if ($hotplug_features->{memory}) {
3123         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3124         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3125         $static_memory = $STATICMEM;
3126         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3127         $dimm_memory = $memory - $static_memory;
3128         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3129
3130     } else {
3131
3132         $static_memory = $memory;
3133         push @$cmd, '-m', $static_memory;
3134     }
3135
3136     if ($conf->{numa}) {
3137
3138         my $numa_totalmemory = undef;
3139         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3140             next if !$conf->{"numa$i"};
3141             my $numa = parse_numa($conf->{"numa$i"});
3142             next if !$numa;
3143             # memory
3144             die "missing numa node$i memory value\n" if !$numa->{memory};
3145             my $numa_memory = $numa->{memory};
3146             $numa_totalmemory += $numa_memory;
3147             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3148
3149             # cpus
3150             my $cpus_start = $numa->{cpus}->{start};
3151             die "missing numa node$i cpus\n" if !defined($cpus_start);
3152             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3153             my $cpus = $cpus_start;
3154             if (defined($cpus_end)) {
3155                 $cpus .= "-$cpus_end";
3156                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3157             }
3158
3159             # hostnodes
3160             my $hostnodes_start = $numa->{hostnodes}->{start};
3161             if (defined($hostnodes_start)) {
3162                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3163                 my $hostnodes = $hostnodes_start;
3164                 if (defined($hostnodes_end)) {
3165                     $hostnodes .= "-$hostnodes_end";
3166                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3167                 }
3168
3169                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3170                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3171                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3172                 }
3173
3174                 # policy
3175                 my $policy = $numa->{policy};
3176                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3177                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3178             }
3179
3180             push @$cmd, '-object', $numa_object;
3181             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3182         }
3183
3184         die "total memory for NUMA nodes must be equal to vm static memory\n"
3185             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3186
3187         #if no custom tology, we split memory and cores across numa nodes
3188         if(!$numa_totalmemory) {
3189
3190             my $numa_memory = ($static_memory / $sockets) . "M";
3191
3192             for (my $i = 0; $i < $sockets; $i++)  {
3193
3194                 my $cpustart = ($cores * $i);
3195                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3196                 my $cpus = $cpustart;
3197                 $cpus .= "-$cpuend" if $cpuend;
3198
3199                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3200                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3201             }
3202         }
3203     }
3204
3205     if ($hotplug_features->{memory}) {
3206         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3207             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3208             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3209             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3210
3211             #if dimm_memory is not aligned to dimm map
3212             if($current_size > $memory) {
3213                  $conf->{memory} = $current_size;
3214                  update_config_nolock($vmid, $conf, 1);
3215             }
3216         });
3217     }
3218
3219     push @$cmd, '-S' if $conf->{freeze};
3220
3221     # set keyboard layout
3222     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3223     push @$cmd, '-k', $kb if $kb;
3224
3225     # enable sound
3226     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3227     #push @$cmd, '-soundhw', 'es1370';
3228     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3229
3230     if($conf->{agent}) {
3231         my $qgasocket = qmp_socket($vmid, 1);
3232         my $pciaddr = print_pci_addr("qga0", $bridges);
3233         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3234         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3235         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3236     }
3237
3238     my $spice_port;
3239
3240     if ($qxlnum) {
3241         if ($qxlnum > 1) {
3242             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3243                 for(my $i = 1; $i < $qxlnum; $i++){
3244                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3245                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3246                 }
3247             } else {
3248                 # assume other OS works like Linux
3249                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3250                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3251             }
3252         }
3253
3254         my $pciaddr = print_pci_addr("spice", $bridges);
3255
3256         my $nodename = PVE::INotify::nodename();
3257         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3258         $spice_port = PVE::Tools::next_spice_port($pfamily);
3259
3260         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3261
3262         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3263         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3264         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3265     }
3266
3267     # enable balloon by default, unless explicitly disabled
3268     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3269         $pciaddr = print_pci_addr("balloon0", $bridges);
3270         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3271     }
3272
3273     if ($conf->{watchdog}) {
3274         my $wdopts = parse_watchdog($conf->{watchdog});
3275         $pciaddr = print_pci_addr("watchdog", $bridges);
3276         my $watchdog = $wdopts->{model} || 'i6300esb';
3277         push @$devices, '-device', "$watchdog$pciaddr";
3278         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3279     }
3280
3281     my $vollist = [];
3282     my $scsicontroller = {};
3283     my $ahcicontroller = {};
3284     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3285
3286     # Add iscsi initiator name if available
3287     if (my $initiator = get_initiator_name()) {
3288         push @$devices, '-iscsi', "initiator-name=$initiator";
3289     }
3290
3291     foreach_drive($conf, sub {
3292         my ($ds, $drive) = @_;
3293
3294         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3295             push @$vollist, $drive->{file};
3296         }
3297
3298         $use_virtio = 1 if $ds =~ m/^virtio/;
3299
3300         if (drive_is_cdrom ($drive)) {
3301             if ($bootindex_hash->{d}) {
3302                 $drive->{bootindex} = $bootindex_hash->{d};
3303                 $bootindex_hash->{d} += 1;
3304             }
3305         } else {
3306             if ($bootindex_hash->{c}) {
3307                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3308                 $bootindex_hash->{c} += 1;
3309             }
3310         }
3311
3312         if($drive->{interface} eq 'virtio'){
3313            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3314         }
3315
3316         if ($drive->{interface} eq 'scsi') {
3317
3318             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3319
3320             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3321             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3322
3323             my $iothread = '';
3324             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3325                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3326                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3327             }
3328
3329             my $queues = '';
3330             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3331                 $queues = ",num_queues=$drive->{queues}";
3332             }
3333
3334             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3335             $scsicontroller->{$controller}=1;
3336         }
3337
3338         if ($drive->{interface} eq 'sata') {
3339            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3340            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3341            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3342            $ahcicontroller->{$controller}=1;
3343         }
3344
3345         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3346         push @$devices, '-drive',$drive_cmd;
3347         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3348     });
3349
3350     for (my $i = 0; $i < $MAX_NETS; $i++) {
3351          next if !$conf->{"net$i"};
3352          my $d = parse_net($conf->{"net$i"});
3353          next if !$d;
3354
3355          $use_virtio = 1 if $d->{model} eq 'virtio';
3356
3357          if ($bootindex_hash->{n}) {
3358             $d->{bootindex} = $bootindex_hash->{n};
3359             $bootindex_hash->{n} += 1;
3360          }
3361
3362          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3363          push @$devices, '-netdev', $netdevfull;
3364
3365          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3366          push @$devices, '-device', $netdevicefull;
3367     }
3368
3369     if (!$q35) {
3370         # add pci bridges
3371         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3372            $bridges->{1} = 1;
3373            $bridges->{2} = 1;
3374         }
3375
3376         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3377
3378         while (my ($k, $v) = each %$bridges) {
3379             $pciaddr = print_pci_addr("pci.$k");
3380             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3381         }
3382     }
3383
3384     # add custom args
3385     if ($conf->{args}) {
3386         my $aa = PVE::Tools::split_args($conf->{args});
3387         push @$cmd, @$aa;
3388     }
3389
3390     push @$cmd, @$devices;
3391     push @$cmd, '-rtc', join(',', @$rtcFlags)
3392         if scalar(@$rtcFlags);
3393     push @$cmd, '-machine', join(',', @$machineFlags)
3394         if scalar(@$machineFlags);
3395     push @$cmd, '-global', join(',', @$globalFlags)
3396         if scalar(@$globalFlags);
3397
3398     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3399 }
3400
3401 sub vnc_socket {
3402     my ($vmid) = @_;
3403     return "${var_run_tmpdir}/$vmid.vnc";
3404 }
3405
3406 sub spice_port {
3407     my ($vmid) = @_;
3408
3409     my $res = vm_mon_cmd($vmid, 'query-spice');
3410
3411     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3412 }
3413
3414 sub qmp_socket {
3415     my ($vmid, $qga) = @_;
3416     my $sockettype = $qga ? 'qga' : 'qmp';
3417     return "${var_run_tmpdir}/$vmid.$sockettype";
3418 }
3419
3420 sub pidfile_name {
3421     my ($vmid) = @_;
3422     return "${var_run_tmpdir}/$vmid.pid";
3423 }
3424
3425 sub vm_devices_list {
3426     my ($vmid) = @_;
3427
3428     my $res = vm_mon_cmd($vmid, 'query-pci');
3429     my $devices = {};
3430     foreach my $pcibus (@$res) {
3431         foreach my $device (@{$pcibus->{devices}}) {
3432             next if !$device->{'qdev_id'};
3433             if ($device->{'pci_bridge'}) {
3434                 $devices->{$device->{'qdev_id'}} = 1;
3435                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3436                     next if !$bridge_device->{'qdev_id'};
3437                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3438                     $devices->{$device->{'qdev_id'}}++;
3439                 }
3440             } else {
3441                 $devices->{$device->{'qdev_id'}} = 1;
3442             }
3443         }
3444     }
3445
3446     my $resblock = vm_mon_cmd($vmid, 'query-block');
3447     foreach my $block (@$resblock) {
3448         if($block->{device} =~ m/^drive-(\S+)/){
3449                 $devices->{$1} = 1;
3450         }
3451     }
3452
3453     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3454     foreach my $mice (@$resmice) {
3455         if ($mice->{name} eq 'QEMU HID Tablet') {
3456             $devices->{tablet} = 1;
3457             last;
3458         }
3459     }
3460
3461     return $devices;
3462 }
3463
3464 sub vm_deviceplug {
3465     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3466
3467     my $q35 = machine_type_is_q35($conf);
3468
3469     my $devices_list = vm_devices_list($vmid);
3470     return 1 if defined($devices_list->{$deviceid});
3471
3472     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3473
3474     if ($deviceid eq 'tablet') {
3475
3476         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3477
3478     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3479
3480         qemu_iothread_add($vmid, $deviceid, $device);
3481
3482         qemu_driveadd($storecfg, $vmid, $device);
3483         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3484
3485         qemu_deviceadd($vmid, $devicefull);
3486         eval { qemu_deviceaddverify($vmid, $deviceid); };
3487         if (my $err = $@) {
3488             eval { qemu_drivedel($vmid, $deviceid); };
3489             warn $@ if $@;
3490             die $err;
3491         }
3492
3493     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3494
3495
3496         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3497         my $pciaddr = print_pci_addr($deviceid);
3498         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3499
3500         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3501
3502         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3503             qemu_iothread_add($vmid, $deviceid, $device);
3504             $devicefull .= ",iothread=iothread-$deviceid";
3505         }
3506
3507         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3508             $devicefull .= ",num_queues=$device->{queues}";
3509         }
3510
3511         qemu_deviceadd($vmid, $devicefull);
3512         qemu_deviceaddverify($vmid, $deviceid);
3513
3514     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3515
3516         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3517         qemu_driveadd($storecfg, $vmid, $device);
3518
3519         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3520         eval { qemu_deviceadd($vmid, $devicefull); };
3521         if (my $err = $@) {
3522             eval { qemu_drivedel($vmid, $deviceid); };
3523             warn $@ if $@;
3524             die $err;
3525         }
3526
3527     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3528
3529         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3530
3531         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3532         my $use_old_bios_files = undef;
3533         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3534
3535         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3536         qemu_deviceadd($vmid, $netdevicefull);
3537         eval { qemu_deviceaddverify($vmid, $deviceid); };
3538         if (my $err = $@) {
3539             eval { qemu_netdevdel($vmid, $deviceid); };
3540             warn $@ if $@;
3541             die $err;
3542         }
3543
3544     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3545
3546         my $bridgeid = $2;
3547         my $pciaddr = print_pci_addr($deviceid);
3548         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3549
3550         qemu_deviceadd($vmid, $devicefull);
3551         qemu_deviceaddverify($vmid, $deviceid);
3552
3553     } else {
3554         die "can't hotplug device '$deviceid'\n";
3555     }
3556
3557     return 1;
3558 }
3559
3560 # fixme: this should raise exceptions on error!
3561 sub vm_deviceunplug {
3562     my ($vmid, $conf, $deviceid) = @_;
3563
3564     my $devices_list = vm_devices_list($vmid);
3565     return 1 if !defined($devices_list->{$deviceid});
3566
3567     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3568
3569     if ($deviceid eq 'tablet') {
3570
3571         qemu_devicedel($vmid, $deviceid);
3572
3573     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3574
3575         qemu_devicedel($vmid, $deviceid);
3576         qemu_devicedelverify($vmid, $deviceid);
3577         qemu_drivedel($vmid, $deviceid);
3578         qemu_iothread_del($conf, $vmid, $deviceid);
3579
3580     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3581
3582         qemu_devicedel($vmid, $deviceid);
3583         qemu_devicedelverify($vmid, $deviceid);
3584         qemu_iothread_del($conf, $vmid, $deviceid);
3585
3586     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3587
3588         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3589         my $device = parse_drive($deviceid, $conf->{$deviceid});
3590         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3591
3592         qemu_devicedel($vmid, $deviceid);
3593         qemu_drivedel($vmid, $deviceid);
3594         qemu_deletescsihw($conf, $vmid, $deviceid);
3595
3596     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3597
3598         qemu_devicedel($vmid, $deviceid);
3599         qemu_devicedelverify($vmid, $deviceid);
3600         qemu_netdevdel($vmid, $deviceid);
3601
3602     } else {
3603         die "can't unplug device '$deviceid'\n";
3604     }
3605
3606     return 1;
3607 }
3608
3609 sub qemu_deviceadd {
3610     my ($vmid, $devicefull) = @_;
3611
3612     $devicefull = "driver=".$devicefull;
3613     my %options =  split(/[=,]/, $devicefull);
3614
3615     vm_mon_cmd($vmid, "device_add" , %options);
3616 }
3617
3618 sub qemu_devicedel {
3619     my ($vmid, $deviceid) = @_;
3620
3621     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3622 }
3623
3624 sub qemu_iothread_add {
3625     my($vmid, $deviceid, $device) = @_;
3626
3627     if ($device->{iothread}) {
3628         my $iothreads = vm_iothreads_list($vmid);
3629         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3630     }
3631 }
3632
3633 sub qemu_iothread_del {
3634     my($conf, $vmid, $deviceid) = @_;
3635
3636     my $device = parse_drive($deviceid, $conf->{$deviceid});
3637     if ($device->{iothread}) {
3638         my $iothreads = vm_iothreads_list($vmid);
3639         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3640     }
3641 }
3642
3643 sub qemu_objectadd {
3644     my($vmid, $objectid, $qomtype) = @_;
3645
3646     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3647
3648     return 1;
3649 }
3650
3651 sub qemu_objectdel {
3652     my($vmid, $objectid) = @_;
3653
3654     vm_mon_cmd($vmid, "object-del", id => $objectid);
3655
3656     return 1;
3657 }
3658
3659 sub qemu_driveadd {
3660     my ($storecfg, $vmid, $device) = @_;
3661
3662     my $drive = print_drive_full($storecfg, $vmid, $device);
3663     $drive =~ s/\\/\\\\/g;
3664     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3665
3666     # If the command succeeds qemu prints: "OK"
3667     return 1 if $ret =~ m/OK/s;
3668
3669     die "adding drive failed: $ret\n";
3670 }
3671
3672 sub qemu_drivedel {
3673     my($vmid, $deviceid) = @_;
3674
3675     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3676     $ret =~ s/^\s+//;
3677
3678     return 1 if $ret eq "";
3679
3680     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3681     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3682
3683     die "deleting drive $deviceid failed : $ret\n";
3684 }
3685
3686 sub qemu_deviceaddverify {
3687     my ($vmid, $deviceid) = @_;
3688
3689     for (my $i = 0; $i <= 5; $i++) {
3690          my $devices_list = vm_devices_list($vmid);
3691          return 1 if defined($devices_list->{$deviceid});
3692          sleep 1;
3693     }
3694
3695     die "error on hotplug device '$deviceid'\n";
3696 }
3697
3698
3699 sub qemu_devicedelverify {
3700     my ($vmid, $deviceid) = @_;
3701
3702     # need to verify that the device is correctly removed as device_del
3703     # is async and empty return is not reliable
3704
3705     for (my $i = 0; $i <= 5; $i++) {
3706          my $devices_list = vm_devices_list($vmid);
3707          return 1 if !defined($devices_list->{$deviceid});
3708          sleep 1;
3709     }
3710
3711     die "error on hot-unplugging device '$deviceid'\n";
3712 }
3713
3714 sub qemu_findorcreatescsihw {
3715     my ($storecfg, $conf, $vmid, $device) = @_;
3716
3717     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3718
3719     my $scsihwid="$controller_prefix$controller";
3720     my $devices_list = vm_devices_list($vmid);
3721
3722     if(!defined($devices_list->{$scsihwid})) {
3723         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3724     }
3725
3726     return 1;
3727 }
3728
3729 sub qemu_deletescsihw {
3730     my ($conf, $vmid, $opt) = @_;
3731
3732     my $device = parse_drive($opt, $conf->{$opt});
3733
3734     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3735         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3736         return 1;
3737     }
3738
3739     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3740
3741     my $devices_list = vm_devices_list($vmid);
3742     foreach my $opt (keys %{$devices_list}) {
3743         if (PVE::QemuServer::valid_drivename($opt)) {
3744             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3745             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3746                 return 1;
3747             }
3748         }
3749     }
3750
3751     my $scsihwid="scsihw$controller";
3752
3753     vm_deviceunplug($vmid, $conf, $scsihwid);
3754
3755     return 1;
3756 }
3757
3758 sub qemu_add_pci_bridge {
3759     my ($storecfg, $conf, $vmid, $device) = @_;
3760
3761     my $bridges = {};
3762
3763     my $bridgeid;
3764
3765     print_pci_addr($device, $bridges);
3766
3767     while (my ($k, $v) = each %$bridges) {
3768         $bridgeid = $k;
3769     }
3770     return 1 if !defined($bridgeid) || $bridgeid < 1;
3771
3772     my $bridge = "pci.$bridgeid";
3773     my $devices_list = vm_devices_list($vmid);
3774
3775     if (!defined($devices_list->{$bridge})) {
3776         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3777     }
3778
3779     return 1;
3780 }
3781
3782 sub qemu_set_link_status {
3783     my ($vmid, $device, $up) = @_;
3784
3785     vm_mon_cmd($vmid, "set_link", name => $device,
3786                up => $up ? JSON::true : JSON::false);
3787 }
3788
3789 sub qemu_netdevadd {
3790     my ($vmid, $conf, $device, $deviceid) = @_;
3791
3792     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3793     my %options =  split(/[=,]/, $netdev);
3794
3795     vm_mon_cmd($vmid, "netdev_add",  %options);
3796     return 1;
3797 }
3798
3799 sub qemu_netdevdel {
3800     my ($vmid, $deviceid) = @_;
3801
3802     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3803 }
3804
3805 sub qemu_cpu_hotplug {
3806     my ($vmid, $conf, $vcpus) = @_;
3807
3808     my $sockets = 1;
3809     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3810     $sockets = $conf->{sockets} if  $conf->{sockets};
3811     my $cores = $conf->{cores} || 1;
3812     my $maxcpus = $sockets * $cores;
3813
3814     $vcpus = $maxcpus if !$vcpus;
3815
3816     die "you can't add more vcpus than maxcpus\n"
3817         if $vcpus > $maxcpus;
3818
3819     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3820     die "online cpu unplug is not yet possible\n"
3821         if $vcpus < $currentvcpus;
3822
3823     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3824     die "vcpus in running vm is different than configuration\n"
3825         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3826
3827     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3828         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3829     }
3830 }
3831
3832 sub qemu_memory_hotplug {
3833     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3834
3835     return $value if !check_running($vmid);
3836
3837     my $memory = $conf->{memory} || $defaults->{memory};
3838     $value = $defaults->{memory} if !$value;
3839     return $value if $value == $memory;
3840
3841     my $static_memory = $STATICMEM;
3842     my $dimm_memory = $memory - $static_memory;
3843
3844     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3845     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3846
3847
3848     my $sockets = 1;
3849     $sockets = $conf->{sockets} if $conf->{sockets};
3850
3851     if($value > $memory) {
3852
3853         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3854             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3855
3856                 return if $current_size <= $conf->{memory};
3857
3858                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3859                 if (my $err = $@) {
3860                     eval { qemu_objectdel($vmid, "mem-$name"); };
3861                     die $err;
3862                 }
3863
3864                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3865                 if (my $err = $@) {
3866                     eval { qemu_objectdel($vmid, "mem-$name"); };
3867                     die $err;
3868                 }
3869                 #update conf after each succesful module hotplug
3870                 $conf->{memory} = $current_size;
3871                 update_config_nolock($vmid, $conf, 1);
3872         });
3873
3874     } else {
3875
3876         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3877             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3878
3879                 return if $current_size >= $conf->{memory};
3880                 print "try to unplug memory dimm $name\n";
3881
3882                 my $retry = 0;
3883                 while (1) {
3884                     eval { qemu_devicedel($vmid, $name) };
3885                     sleep 3;
3886                     my $dimm_list = qemu_dimm_list($vmid);
3887                     last if !$dimm_list->{$name};
3888                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3889                     $retry++;
3890                 }
3891
3892                 #update conf after each succesful module unplug
3893                 $conf->{memory} = $current_size;
3894
3895                 eval { qemu_objectdel($vmid, "mem-$name"); };
3896                 update_config_nolock($vmid, $conf, 1);
3897         });
3898     }
3899 }
3900
3901 sub qemu_dimm_list {
3902     my ($vmid) = @_;
3903
3904     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3905     my $dimms = {};
3906
3907     foreach my $dimm (@$dimmarray) {
3908
3909         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3910         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3911         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3912         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3913         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3914     }
3915     return $dimms;
3916 }
3917
3918 sub qemu_block_set_io_throttle {
3919     my ($vmid, $deviceid,
3920         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3921         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3922
3923     return if !check_running($vmid) ;
3924
3925     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3926         bps => int($bps),
3927         bps_rd => int($bps_rd),
3928         bps_wr => int($bps_wr),
3929         iops => int($iops),
3930         iops_rd => int($iops_rd),
3931         iops_wr => int($iops_wr),
3932         bps_max => int($bps_max),
3933         bps_rd_max => int($bps_rd_max),
3934         bps_wr_max => int($bps_wr_max),
3935         iops_max => int($iops_max),
3936         iops_rd_max => int($iops_rd_max),
3937         iops_wr_max => int($iops_wr_max)
3938     );
3939
3940 }
3941
3942 # old code, only used to shutdown old VM after update
3943 sub __read_avail {
3944     my ($fh, $timeout) = @_;
3945
3946     my $sel = new IO::Select;
3947     $sel->add($fh);
3948
3949     my $res = '';
3950     my $buf;
3951
3952     my @ready;
3953     while (scalar (@ready = $sel->can_read($timeout))) {
3954         my $count;
3955         if ($count = $fh->sysread($buf, 8192)) {
3956             if ($buf =~ /^(.*)\(qemu\) $/s) {
3957                 $res .= $1;
3958                 last;
3959             } else {
3960                 $res .= $buf;
3961             }
3962         } else {
3963             if (!defined($count)) {
3964                 die "$!\n";
3965             }
3966             last;
3967         }
3968     }
3969
3970     die "monitor read timeout\n" if !scalar(@ready);
3971
3972     return $res;
3973 }
3974
3975 # old code, only used to shutdown old VM after update
3976 sub vm_monitor_command {
3977     my ($vmid, $cmdstr, $nocheck) = @_;
3978
3979     my $res;
3980
3981     eval {
3982         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3983
3984         my $sname = "${var_run_tmpdir}/$vmid.mon";
3985
3986         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3987             die "unable to connect to VM $vmid socket - $!\n";
3988
3989         my $timeout = 3;
3990
3991         # hack: migrate sometime blocks the monitor (when migrate_downtime
3992         # is set)
3993         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3994             $timeout = 60*60; # 1 hour
3995         }
3996
3997         # read banner;
3998         my $data = __read_avail($sock, $timeout);
3999
4000         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4001             die "got unexpected qemu monitor banner\n";
4002         }
4003
4004         my $sel = new IO::Select;
4005         $sel->add($sock);
4006
4007         if (!scalar(my @ready = $sel->can_write($timeout))) {
4008             die "monitor write error - timeout";
4009         }
4010
4011         my $fullcmd = "$cmdstr\r";
4012
4013         # syslog('info', "VM $vmid monitor command: $cmdstr");
4014
4015         my $b;
4016         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4017             die "monitor write error - $!";
4018         }
4019
4020         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4021
4022         $timeout = 20;
4023
4024         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4025             $timeout = 60*60; # 1 hour
4026         } elsif ($cmdstr =~ m/^(eject|change)/) {
4027             $timeout = 60; # note: cdrom mount command is slow
4028         }
4029         if ($res = __read_avail($sock, $timeout)) {
4030
4031             my @lines = split("\r?\n", $res);
4032
4033             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4034
4035             $res = join("\n", @lines);
4036             $res .= "\n";
4037         }
4038     };
4039
4040     my $err = $@;
4041
4042     if ($err) {
4043         syslog("err", "VM $vmid monitor command failed - $err");
4044         die $err;
4045     }
4046
4047     return $res;
4048 }
4049
4050 sub qemu_block_resize {
4051     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4052
4053     my $running = check_running($vmid);
4054
4055     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4056
4057     return if !$running;
4058
4059     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4060
4061 }
4062
4063 sub qemu_volume_snapshot {
4064     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4065
4066     my $running = check_running($vmid);
4067
4068     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4069         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4070     } else {
4071         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4072     }
4073 }
4074
4075 sub qemu_volume_snapshot_delete {
4076     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4077
4078     my $running = check_running($vmid);
4079
4080     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4081
4082     return if !$running;
4083
4084     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4085 }
4086
4087 sub set_migration_caps {
4088     my ($vmid) = @_;
4089
4090     my $cap_ref = [];
4091
4092     my $enabled_cap = {
4093         "auto-converge" => 1,
4094         "xbzrle" => 1,
4095         "x-rdma-pin-all" => 0,
4096         "zero-blocks" => 0,
4097         "compress" => 0
4098     };
4099
4100     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4101
4102     for my $supported_capability (@$supported_capabilities) {
4103         push @$cap_ref, {
4104             capability => $supported_capability->{capability},
4105             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4106         };
4107     }
4108
4109     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4110 }
4111
4112 my $fast_plug_option = {
4113     'lock' => 1,
4114     'name' => 1,
4115     'onboot' => 1,
4116     'shares' => 1,
4117     'startup' => 1,
4118     'description' => 1,
4119 };
4120
4121 # hotplug changes in [PENDING]
4122 # $selection hash can be used to only apply specified options, for
4123 # example: { cores => 1 } (only apply changed 'cores')
4124 # $errors ref is used to return error messages
4125 sub vmconfig_hotplug_pending {
4126     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4127
4128     my $defaults = load_defaults();
4129
4130     # commit values which do not have any impact on running VM first
4131     # Note: those option cannot raise errors, we we do not care about
4132     # $selection and always apply them.
4133
4134     my $add_error = sub {
4135         my ($opt, $msg) = @_;
4136         $errors->{$opt} = "hotplug problem - $msg";
4137     };
4138
4139     my $changes = 0;
4140     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4141         if ($fast_plug_option->{$opt}) {
4142             $conf->{$opt} = $conf->{pending}->{$opt};
4143             delete $conf->{pending}->{$opt};
4144             $changes = 1;
4145         }
4146     }
4147
4148     if ($changes) {
4149         update_config_nolock($vmid, $conf, 1);
4150         $conf = load_config($vmid); # update/reload
4151     }
4152
4153     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4154
4155     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4156     while (my ($opt, $force) = each %$pending_delete_hash) {
4157         next if $selection && !$selection->{$opt};
4158         eval {
4159             if ($opt eq 'hotplug') {
4160                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4161             } elsif ($opt eq 'tablet') {
4162                 die "skip\n" if !$hotplug_features->{usb};
4163                 if ($defaults->{tablet}) {
4164                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4165                 } else {
4166                     vm_deviceunplug($vmid, $conf, $opt);
4167                 }
4168             } elsif ($opt eq 'vcpus') {
4169                 die "skip\n" if !$hotplug_features->{cpu};
4170                 qemu_cpu_hotplug($vmid, $conf, undef);
4171             } elsif ($opt eq 'balloon') {
4172                 # enable balloon device is not hotpluggable
4173                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4174             } elsif ($fast_plug_option->{$opt}) {
4175                 # do nothing
4176             } elsif ($opt =~ m/^net(\d+)$/) {
4177                 die "skip\n" if !$hotplug_features->{network};
4178                 vm_deviceunplug($vmid, $conf, $opt);
4179             } elsif (valid_drivename($opt)) {
4180                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4181                 vm_deviceunplug($vmid, $conf, $opt);
4182                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4183             } elsif ($opt =~ m/^memory$/) {
4184                 die "skip\n" if !$hotplug_features->{memory};
4185                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4186             } elsif ($opt eq 'cpuunits') {
4187                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4188             } elsif ($opt eq 'cpulimit') {
4189                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4190             } else {
4191                 die "skip\n";
4192             }
4193         };
4194         if (my $err = $@) {
4195             &$add_error($opt, $err) if $err ne "skip\n";
4196         } else {
4197             # save new config if hotplug was successful
4198             delete $conf->{$opt};
4199             vmconfig_undelete_pending_option($conf, $opt);
4200             update_config_nolock($vmid, $conf, 1);
4201             $conf = load_config($vmid); # update/reload
4202         }
4203     }
4204
4205     foreach my $opt (keys %{$conf->{pending}}) {
4206         next if $selection && !$selection->{$opt};
4207         my $value = $conf->{pending}->{$opt};
4208         eval {
4209             if ($opt eq 'hotplug') {
4210                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4211             } elsif ($opt eq 'tablet') {
4212                 die "skip\n" if !$hotplug_features->{usb};
4213                 if ($value == 1) {
4214                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4215                 } elsif ($value == 0) {
4216                     vm_deviceunplug($vmid, $conf, $opt);
4217                 }
4218             } elsif ($opt eq 'vcpus') {
4219                 die "skip\n" if !$hotplug_features->{cpu};
4220                 qemu_cpu_hotplug($vmid, $conf, $value);
4221             } elsif ($opt eq 'balloon') {
4222                 # enable/disable balloning device is not hotpluggable
4223                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4224                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4225                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4226
4227                 # allow manual ballooning if shares is set to zero
4228                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4229                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4230                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4231                 }
4232             } elsif ($opt =~ m/^net(\d+)$/) {
4233                 # some changes can be done without hotplug
4234                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4235                                     $vmid, $opt, $value);
4236             } elsif (valid_drivename($opt)) {
4237                 # some changes can be done without hotplug
4238                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4239                                      $vmid, $opt, $value, 1);
4240             } elsif ($opt =~ m/^memory$/) { #dimms
4241                 die "skip\n" if !$hotplug_features->{memory};
4242                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4243             } elsif ($opt eq 'cpuunits') {
4244                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4245             } elsif ($opt eq 'cpulimit') {
4246                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4247                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4248             } else {
4249                 die "skip\n";  # skip non-hot-pluggable options
4250             }
4251         };
4252         if (my $err = $@) {
4253             &$add_error($opt, $err) if $err ne "skip\n";
4254         } else {
4255             # save new config if hotplug was successful
4256             $conf->{$opt} = $value;
4257             delete $conf->{pending}->{$opt};
4258             update_config_nolock($vmid, $conf, 1);
4259             $conf = load_config($vmid); # update/reload
4260         }
4261     }
4262 }
4263
4264 sub try_deallocate_drive {
4265     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4266
4267     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4268         my $volid = $drive->{file};
4269         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4270             my $sid = PVE::Storage::parse_volume_id($volid);
4271             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4272
4273             # check if the disk is really unused
4274             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4275                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4276             PVE::Storage::vdisk_free($storecfg, $volid);
4277             return 1;
4278         } else {
4279             # If vm is not owner of this disk remove from config
4280             return 1;
4281         }
4282     }
4283
4284     return undef;
4285 }
4286
4287 sub vmconfig_delete_or_detach_drive {
4288     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4289
4290     my $drive = parse_drive($opt, $conf->{$opt});
4291
4292     my $rpcenv = PVE::RPCEnvironment::get();
4293     my $authuser = $rpcenv->get_user();
4294
4295     if ($force) {
4296         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4297         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4298     } else {
4299         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4300     }
4301 }
4302
4303 sub vmconfig_apply_pending {
4304     my ($vmid, $conf, $storecfg) = @_;
4305
4306     # cold plug
4307
4308     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4309     while (my ($opt, $force) = each %$pending_delete_hash) {
4310         die "internal error" if $opt =~ m/^unused/;
4311         $conf = load_config($vmid); # update/reload
4312         if (!defined($conf->{$opt})) {
4313             vmconfig_undelete_pending_option($conf, $opt);
4314             update_config_nolock($vmid, $conf, 1);
4315         } elsif (valid_drivename($opt)) {
4316             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4317             vmconfig_undelete_pending_option($conf, $opt);
4318             delete $conf->{$opt};
4319             update_config_nolock($vmid, $conf, 1);
4320         } else {
4321             vmconfig_undelete_pending_option($conf, $opt);
4322             delete $conf->{$opt};
4323             update_config_nolock($vmid, $conf, 1);
4324         }
4325     }
4326
4327     $conf = load_config($vmid); # update/reload
4328
4329     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4330         $conf = load_config($vmid); # update/reload
4331
4332         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4333             # skip if nothing changed
4334         } elsif (valid_drivename($opt)) {
4335             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4336                 if defined($conf->{$opt});
4337             $conf->{$opt} = $conf->{pending}->{$opt};
4338         } else {
4339             $conf->{$opt} = $conf->{pending}->{$opt};
4340         }
4341
4342         delete $conf->{pending}->{$opt};
4343         update_config_nolock($vmid, $conf, 1);
4344     }
4345 }
4346
4347 my $safe_num_ne = sub {
4348     my ($a, $b) = @_;
4349
4350     return 0 if !defined($a) && !defined($b);
4351     return 1 if !defined($a);
4352     return 1 if !defined($b);
4353
4354     return $a != $b;
4355 };
4356
4357 my $safe_string_ne = sub {
4358     my ($a, $b) = @_;
4359
4360     return 0 if !defined($a) && !defined($b);
4361     return 1 if !defined($a);
4362     return 1 if !defined($b);
4363
4364     return $a ne $b;
4365 };
4366
4367 sub vmconfig_update_net {
4368     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4369
4370     my $newnet = parse_net($value);
4371
4372     if ($conf->{$opt}) {
4373         my $oldnet = parse_net($conf->{$opt});
4374
4375         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4376             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4377             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4378             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4379
4380             # for non online change, we try to hot-unplug
4381             die "skip\n" if !$hotplug;
4382             vm_deviceunplug($vmid, $conf, $opt);
4383         } else {
4384
4385             die "internal error" if $opt !~ m/net(\d+)/;
4386             my $iface = "tap${vmid}i$1";
4387
4388             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4389                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4390             }
4391
4392             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4393                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4394                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4395                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4396                 PVE::Network::tap_unplug($iface);
4397                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4398             }
4399
4400             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4401                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4402             }
4403
4404             return 1;
4405         }
4406     }
4407
4408     if ($hotplug) {
4409         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4410     } else {
4411         die "skip\n";
4412     }
4413 }
4414
4415 sub vmconfig_update_disk {
4416     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4417
4418     # fixme: do we need force?
4419
4420     my $drive = parse_drive($opt, $value);
4421
4422     if ($conf->{$opt}) {
4423
4424         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4425
4426             my $media = $drive->{media} || 'disk';
4427             my $oldmedia = $old_drive->{media} || 'disk';
4428             die "unable to change media type\n" if $media ne $oldmedia;
4429
4430             if (!drive_is_cdrom($old_drive)) {
4431
4432                 if ($drive->{file} ne $old_drive->{file}) {
4433
4434                     die "skip\n" if !$hotplug;
4435
4436                     # unplug and register as unused
4437                     vm_deviceunplug($vmid, $conf, $opt);
4438                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4439
4440                 } else {
4441                     # update existing disk
4442
4443                     # skip non hotpluggable value
4444                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4445                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4446                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4447                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4448                         die "skip\n";
4449                     }
4450
4451                     # apply throttle
4452                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4453                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4454                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4455                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4456                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4457                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4458                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4459                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4460                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4461                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4462                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4463                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4464
4465                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4466                                                    ($drive->{mbps} || 0)*1024*1024,
4467                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4468                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4469                                                    $drive->{iops} || 0,
4470                                                    $drive->{iops_rd} || 0,
4471                                                    $drive->{iops_wr} || 0,
4472                                                    ($drive->{mbps_max} || 0)*1024*1024,
4473                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4474                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4475                                                    $drive->{iops_max} || 0,
4476                                                    $drive->{iops_rd_max} || 0,
4477                                                    $drive->{iops_wr_max} || 0);
4478
4479                     }
4480
4481                     return 1;
4482                 }
4483
4484             } else { # cdrom
4485
4486                 if ($drive->{file} eq 'none') {
4487                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4488                 } else {
4489                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4490                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4491                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4492                 }
4493
4494                 return 1;
4495             }
4496         }
4497     }
4498
4499     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4500     # hotplug new disks
4501     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4502     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4503 }
4504
4505 sub vm_start {
4506     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4507         $forcemachine, $spice_ticket) = @_;
4508
4509     lock_config($vmid, sub {
4510         my $conf = load_config($vmid, $migratedfrom);
4511
4512         die "you can't start a vm if it's a template\n" if is_template($conf);
4513
4514         check_lock($conf) if !$skiplock;
4515
4516         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4517
4518         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4519             vmconfig_apply_pending($vmid, $conf, $storecfg);
4520             $conf = load_config($vmid); # update/reload
4521         }
4522
4523         my $defaults = load_defaults();
4524
4525         # set environment variable useful inside network script
4526         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4527
4528         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4529
4530         my $migrate_port = 0;
4531         my $migrate_uri;
4532         if ($statefile) {
4533             if ($statefile eq 'tcp') {
4534                 my $localip = "localhost";
4535                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4536                 my $nodename = PVE::INotify::nodename();
4537                 if ($datacenterconf->{migration_unsecure}) {
4538                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4539                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4540                 }
4541                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4542                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4543                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4544                 push @$cmd, '-incoming', $migrate_uri;
4545                 push @$cmd, '-S';
4546             } else {
4547                 push @$cmd, '-loadstate', $statefile;
4548             }
4549         } elsif ($paused) {
4550             push @$cmd, '-S';
4551         }
4552
4553         # host pci devices
4554         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4555           my $d = parse_hostpci($conf->{"hostpci$i"});
4556           next if !$d;
4557           my $pcidevices = $d->{pciid};
4558           foreach my $pcidevice (@$pcidevices) {
4559                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4560
4561                 my $info = pci_device_info("0000:$pciid");
4562                 die "IOMMU not present\n" if !check_iommu_support();
4563                 die "no pci device info for device '$pciid'\n" if !$info;
4564                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4565                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4566           }
4567         }
4568
4569         PVE::Storage::activate_volumes($storecfg, $vollist);
4570
4571         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4572                     umask => 0077); };
4573         my $err = $@;
4574         die "start failed: $err" if $err;
4575
4576         print "migration listens on $migrate_uri\n" if $migrate_uri;
4577
4578         if ($statefile && $statefile ne 'tcp')  {
4579             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4580             warn $@ if $@;
4581         }
4582
4583         if ($migratedfrom) {
4584
4585             eval {
4586                 set_migration_caps($vmid);
4587             };
4588             warn $@ if $@;
4589
4590             if ($spice_port) {
4591                 print "spice listens on port $spice_port\n";
4592                 if ($spice_ticket) {
4593                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4594                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4595                 }
4596             }
4597
4598         } else {
4599
4600             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4601                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4602                     if $conf->{balloon};
4603             }
4604
4605             foreach my $opt (keys %$conf) {
4606                 next if $opt !~  m/^net\d+$/;
4607                 my $nicconf = parse_net($conf->{$opt});
4608                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4609             }
4610         }
4611
4612         vm_mon_cmd_nocheck($vmid, 'qom-set',
4613                     path => "machine/peripheral/balloon0",
4614                     property => "guest-stats-polling-interval",
4615                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4616
4617     });
4618 }
4619
4620 sub vm_mon_cmd {
4621     my ($vmid, $execute, %params) = @_;
4622
4623     my $cmd = { execute => $execute, arguments => \%params };
4624     vm_qmp_command($vmid, $cmd);
4625 }
4626
4627 sub vm_mon_cmd_nocheck {
4628     my ($vmid, $execute, %params) = @_;
4629
4630     my $cmd = { execute => $execute, arguments => \%params };
4631     vm_qmp_command($vmid, $cmd, 1);
4632 }
4633
4634 sub vm_qmp_command {
4635     my ($vmid, $cmd, $nocheck) = @_;
4636
4637     my $res;
4638
4639     my $timeout;
4640     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4641         $timeout = $cmd->{arguments}->{timeout};
4642         delete $cmd->{arguments}->{timeout};
4643     }
4644
4645     eval {
4646         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4647         my $sname = qmp_socket($vmid);
4648         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4649             my $qmpclient = PVE::QMPClient->new();
4650
4651             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4652         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4653             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4654                 if scalar(%{$cmd->{arguments}});
4655             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4656         } else {
4657             die "unable to open monitor socket\n";
4658         }
4659     };
4660     if (my $err = $@) {
4661         syslog("err", "VM $vmid qmp command failed - $err");
4662         die $err;
4663     }
4664
4665     return $res;
4666 }
4667
4668 sub vm_human_monitor_command {
4669     my ($vmid, $cmdline) = @_;
4670
4671     my $res;
4672
4673     my $cmd = {
4674         execute => 'human-monitor-command',
4675         arguments => { 'command-line' => $cmdline},
4676     };
4677
4678     return vm_qmp_command($vmid, $cmd);
4679 }
4680
4681 sub vm_commandline {
4682     my ($storecfg, $vmid) = @_;
4683
4684     my $conf = load_config($vmid);
4685
4686     my $defaults = load_defaults();
4687
4688     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4689
4690     return join(' ', @$cmd);
4691 }
4692
4693 sub vm_reset {
4694     my ($vmid, $skiplock) = @_;
4695
4696     lock_config($vmid, sub {
4697
4698         my $conf = load_config($vmid);
4699
4700         check_lock($conf) if !$skiplock;
4701
4702         vm_mon_cmd($vmid, "system_reset");
4703     });
4704 }
4705
4706 sub get_vm_volumes {
4707     my ($conf) = @_;
4708
4709     my $vollist = [];
4710     foreach_volid($conf, sub {
4711         my ($volid, $is_cdrom) = @_;
4712
4713         return if $volid =~ m|^/|;
4714
4715         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4716         return if !$sid;
4717
4718         push @$vollist, $volid;
4719     });
4720
4721     return $vollist;
4722 }
4723
4724 sub vm_stop_cleanup {
4725     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4726
4727     eval {
4728
4729         if (!$keepActive) {
4730             my $vollist = get_vm_volumes($conf);
4731             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4732         }
4733
4734         foreach my $ext (qw(mon qmp pid vnc qga)) {
4735             unlink "/var/run/qemu-server/${vmid}.$ext";
4736         }
4737
4738         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4739     };
4740     warn $@ if $@; # avoid errors - just warn
4741 }
4742
4743 # Note: use $nockeck to skip tests if VM configuration file exists.
4744 # We need that when migration VMs to other nodes (files already moved)
4745 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4746 sub vm_stop {
4747     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4748
4749     $force = 1 if !defined($force) && !$shutdown;
4750
4751     if ($migratedfrom){
4752         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4753         kill 15, $pid if $pid;
4754         my $conf = load_config($vmid, $migratedfrom);
4755         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4756         return;
4757     }
4758
4759     lock_config($vmid, sub {
4760
4761         my $pid = check_running($vmid, $nocheck);
4762         return if !$pid;
4763
4764         my $conf;
4765         if (!$nocheck) {
4766             $conf = load_config($vmid);
4767             check_lock($conf) if !$skiplock;
4768             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4769                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4770                 $timeout = $opts->{down} if $opts->{down};
4771             }
4772         }
4773
4774         $timeout = 60 if !defined($timeout);
4775
4776         eval {
4777             if ($shutdown) {
4778                 if (defined($conf) && $conf->{agent}) {
4779                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4780                 } else {
4781                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4782                 }
4783             } else {
4784                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4785             }
4786         };
4787         my $err = $@;
4788
4789         if (!$err) {
4790             my $count = 0;
4791             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4792                 $count++;
4793                 sleep 1;
4794             }
4795
4796             if ($count >= $timeout) {
4797                 if ($force) {
4798                     warn "VM still running - terminating now with SIGTERM\n";
4799                     kill 15, $pid;
4800                 } else {
4801                     die "VM quit/powerdown failed - got timeout\n";
4802                 }
4803             } else {
4804                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4805                 return;
4806             }
4807         } else {
4808             if ($force) {
4809                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4810                 kill 15, $pid;
4811             } else {
4812                 die "VM quit/powerdown failed\n";
4813             }
4814         }
4815
4816         # wait again
4817         $timeout = 10;
4818
4819         my $count = 0;
4820         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4821             $count++;
4822             sleep 1;
4823         }
4824
4825         if ($count >= $timeout) {
4826             warn "VM still running - terminating now with SIGKILL\n";
4827             kill 9, $pid;
4828             sleep 1;
4829         }
4830
4831         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4832    });
4833 }
4834
4835 sub vm_suspend {
4836     my ($vmid, $skiplock) = @_;
4837
4838     lock_config($vmid, sub {
4839
4840         my $conf = load_config($vmid);
4841
4842         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4843
4844         vm_mon_cmd($vmid, "stop");
4845     });
4846 }
4847
4848 sub vm_resume {
4849     my ($vmid, $skiplock, $nocheck) = @_;
4850
4851     lock_config($vmid, sub {
4852
4853         if (!$nocheck) {
4854
4855             my $conf = load_config($vmid);
4856
4857             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4858
4859             vm_mon_cmd($vmid, "cont");
4860
4861         } else {
4862             vm_mon_cmd_nocheck($vmid, "cont");
4863         }
4864     });
4865 }
4866
4867 sub vm_sendkey {
4868     my ($vmid, $skiplock, $key) = @_;
4869
4870     lock_config($vmid, sub {
4871
4872         my $conf = load_config($vmid);
4873
4874         # there is no qmp command, so we use the human monitor command
4875         vm_human_monitor_command($vmid, "sendkey $key");
4876     });
4877 }
4878
4879 sub vm_destroy {
4880     my ($storecfg, $vmid, $skiplock) = @_;
4881
4882     lock_config($vmid, sub {
4883
4884         my $conf = load_config($vmid);
4885
4886         if (!check_running($vmid)) {
4887             destroy_vm($storecfg, $vmid, undef, $skiplock);
4888         } else {
4889             die "VM $vmid is running - destroy failed\n";
4890         }
4891     });
4892 }
4893
4894 # pci helpers
4895
4896 sub file_write {
4897     my ($filename, $buf) = @_;
4898
4899     my $fh = IO::File->new($filename, "w");
4900     return undef if !$fh;
4901
4902     my $res = print $fh $buf;
4903
4904     $fh->close();
4905
4906     return $res;
4907 }
4908
4909 sub pci_device_info {
4910     my ($name) = @_;
4911
4912     my $res;
4913
4914     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4915     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4916
4917     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4918     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4919
4920     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4921     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4922
4923     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4924     return undef if !defined($product) || $product !~ s/^0x//;
4925
4926     $res = {
4927         name => $name,
4928         vendor => $vendor,
4929         product => $product,
4930         domain => $domain,
4931         bus => $bus,
4932         slot => $slot,
4933         func => $func,
4934         irq => $irq,
4935         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4936     };
4937
4938     return $res;
4939 }
4940
4941 sub pci_dev_reset {
4942     my ($dev) = @_;
4943
4944     my $name = $dev->{name};
4945
4946     my $fn = "$pcisysfs/devices/$name/reset";
4947
4948     return file_write($fn, "1");
4949 }
4950
4951 sub pci_dev_bind_to_vfio {
4952     my ($dev) = @_;
4953
4954     my $name = $dev->{name};
4955
4956     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4957
4958     if (!-d $vfio_basedir) {
4959         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4960     }
4961     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4962
4963     my $testdir = "$vfio_basedir/$name";
4964     return 1 if -d $testdir;
4965
4966     my $data = "$dev->{vendor} $dev->{product}";
4967     return undef if !file_write("$vfio_basedir/new_id", $data);
4968
4969     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4970     if (!file_write($fn, $name)) {
4971         return undef if -f $fn;
4972     }
4973
4974     $fn = "$vfio_basedir/bind";
4975     if (! -d $testdir) {
4976         return undef if !file_write($fn, $name);
4977     }
4978
4979     return -d $testdir;
4980 }
4981
4982 sub pci_dev_group_bind_to_vfio {
4983     my ($pciid) = @_;
4984
4985     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4986
4987     if (!-d $vfio_basedir) {
4988         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4989     }
4990     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4991
4992     # get IOMMU group devices
4993     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4994       my @devs = grep /^0000:/, readdir($D);
4995     closedir($D);
4996
4997     foreach my $pciid (@devs) {
4998         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4999
5000         # pci bridges, switches or root ports are not supported
5001         # they have a pci_bus subdirectory so skip them
5002         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5003
5004         my $info = pci_device_info($1);
5005         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5006     }
5007
5008     return 1;
5009 }
5010
5011 sub print_pci_addr {
5012     my ($id, $bridges) = @_;
5013
5014     my $res = '';
5015     my $devices = {
5016         piix3 => { bus => 0, addr => 1 },
5017         #addr2 : first videocard
5018         balloon0 => { bus => 0, addr => 3 },
5019         watchdog => { bus => 0, addr => 4 },
5020         scsihw0 => { bus => 0, addr => 5 },
5021         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5022         scsihw1 => { bus => 0, addr => 6 },
5023         ahci0 => { bus => 0, addr => 7 },
5024         qga0 => { bus => 0, addr => 8 },
5025         spice => { bus => 0, addr => 9 },
5026         virtio0 => { bus => 0, addr => 10 },
5027         virtio1 => { bus => 0, addr => 11 },
5028         virtio2 => { bus => 0, addr => 12 },
5029         virtio3 => { bus => 0, addr => 13 },
5030         virtio4 => { bus => 0, addr => 14 },
5031         virtio5 => { bus => 0, addr => 15 },
5032         hostpci0 => { bus => 0, addr => 16 },
5033         hostpci1 => { bus => 0, addr => 17 },
5034         net0 => { bus => 0, addr => 18 },
5035         net1 => { bus => 0, addr => 19 },
5036         net2 => { bus => 0, addr => 20 },
5037         net3 => { bus => 0, addr => 21 },
5038         net4 => { bus => 0, addr => 22 },
5039         net5 => { bus => 0, addr => 23 },
5040         vga1 => { bus => 0, addr => 24 },
5041         vga2 => { bus => 0, addr => 25 },
5042         vga3 => { bus => 0, addr => 26 },
5043         hostpci2 => { bus => 0, addr => 27 },
5044         hostpci3 => { bus => 0, addr => 28 },
5045         #addr29 : usb-host (pve-usb.cfg)
5046         'pci.1' => { bus => 0, addr => 30 },
5047         'pci.2' => { bus => 0, addr => 31 },
5048         'net6' => { bus => 1, addr => 1 },
5049         'net7' => { bus => 1, addr => 2 },
5050         'net8' => { bus => 1, addr => 3 },
5051         'net9' => { bus => 1, addr => 4 },
5052         'net10' => { bus => 1, addr => 5 },
5053         'net11' => { bus => 1, addr => 6 },
5054         'net12' => { bus => 1, addr => 7 },
5055         'net13' => { bus => 1, addr => 8 },
5056         'net14' => { bus => 1, addr => 9 },
5057         'net15' => { bus => 1, addr => 10 },
5058         'net16' => { bus => 1, addr => 11 },
5059         'net17' => { bus => 1, addr => 12 },
5060         'net18' => { bus => 1, addr => 13 },
5061         'net19' => { bus => 1, addr => 14 },
5062         'net20' => { bus => 1, addr => 15 },
5063         'net21' => { bus => 1, addr => 16 },
5064         'net22' => { bus => 1, addr => 17 },
5065         'net23' => { bus => 1, addr => 18 },
5066         'net24' => { bus => 1, addr => 19 },
5067         'net25' => { bus => 1, addr => 20 },
5068         'net26' => { bus => 1, addr => 21 },
5069         'net27' => { bus => 1, addr => 22 },
5070         'net28' => { bus => 1, addr => 23 },
5071         'net29' => { bus => 1, addr => 24 },
5072         'net30' => { bus => 1, addr => 25 },
5073         'net31' => { bus => 1, addr => 26 },
5074         'xhci' => { bus => 1, addr => 27 },
5075         'virtio6' => { bus => 2, addr => 1 },
5076         'virtio7' => { bus => 2, addr => 2 },
5077         'virtio8' => { bus => 2, addr => 3 },
5078         'virtio9' => { bus => 2, addr => 4 },
5079         'virtio10' => { bus => 2, addr => 5 },
5080         'virtio11' => { bus => 2, addr => 6 },
5081         'virtio12' => { bus => 2, addr => 7 },
5082         'virtio13' => { bus => 2, addr => 8 },
5083         'virtio14' => { bus => 2, addr => 9 },
5084         'virtio15' => { bus => 2, addr => 10 },
5085         'virtioscsi0' => { bus => 3, addr => 1 },
5086         'virtioscsi1' => { bus => 3, addr => 2 },
5087         'virtioscsi2' => { bus => 3, addr => 3 },
5088         'virtioscsi3' => { bus => 3, addr => 4 },
5089         'virtioscsi4' => { bus => 3, addr => 5 },
5090         'virtioscsi5' => { bus => 3, addr => 6 },
5091         'virtioscsi6' => { bus => 3, addr => 7 },
5092         'virtioscsi7' => { bus => 3, addr => 8 },
5093         'virtioscsi8' => { bus => 3, addr => 9 },
5094         'virtioscsi9' => { bus => 3, addr => 10 },
5095         'virtioscsi10' => { bus => 3, addr => 11 },
5096         'virtioscsi11' => { bus => 3, addr => 12 },
5097         'virtioscsi12' => { bus => 3, addr => 13 },
5098         'virtioscsi13' => { bus => 3, addr => 14 },
5099         'virtioscsi14' => { bus => 3, addr => 15 },
5100         'virtioscsi15' => { bus => 3, addr => 16 },
5101         'virtioscsi16' => { bus => 3, addr => 17 },
5102         'virtioscsi17' => { bus => 3, addr => 18 },
5103         'virtioscsi18' => { bus => 3, addr => 19 },
5104         'virtioscsi19' => { bus => 3, addr => 20 },
5105         'virtioscsi20' => { bus => 3, addr => 21 },
5106         'virtioscsi21' => { bus => 3, addr => 22 },
5107         'virtioscsi22' => { bus => 3, addr => 23 },
5108         'virtioscsi23' => { bus => 3, addr => 24 },
5109         'virtioscsi24' => { bus => 3, addr => 25 },
5110         'virtioscsi25' => { bus => 3, addr => 26 },
5111         'virtioscsi26' => { bus => 3, addr => 27 },
5112         'virtioscsi27' => { bus => 3, addr => 28 },
5113         'virtioscsi28' => { bus => 3, addr => 29 },
5114         'virtioscsi29' => { bus => 3, addr => 30 },
5115         'virtioscsi30' => { bus => 3, addr => 31 },
5116
5117     };
5118
5119     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5120            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5121            my $bus = $devices->{$id}->{bus};
5122            $res = ",bus=pci.$bus,addr=$addr";
5123            $bridges->{$bus} = 1 if $bridges;
5124     }
5125     return $res;
5126
5127 }
5128
5129 sub print_pcie_addr {
5130     my ($id) = @_;
5131
5132     my $res = '';
5133     my $devices = {
5134         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5135         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5136         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5137         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5138     };
5139
5140     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5141            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5142            my $bus = $devices->{$id}->{bus};
5143            $res = ",bus=$bus,addr=$addr";
5144     }
5145     return $res;
5146
5147 }
5148
5149 # vzdump restore implementaion
5150
5151 sub tar_archive_read_firstfile {
5152     my $archive = shift;
5153
5154     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5155
5156     # try to detect archive type first
5157     my $pid = open (TMP, "tar tf '$archive'|") ||
5158         die "unable to open file '$archive'\n";
5159     my $firstfile = <TMP>;
5160     kill 15, $pid;
5161     close TMP;
5162
5163     die "ERROR: archive contaions no data\n" if !$firstfile;
5164     chomp $firstfile;
5165
5166     return $firstfile;
5167 }
5168
5169 sub tar_restore_cleanup {
5170     my ($storecfg, $statfile) = @_;
5171
5172     print STDERR "starting cleanup\n";
5173
5174     if (my $fd = IO::File->new($statfile, "r")) {
5175         while (defined(my $line = <$fd>)) {
5176             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5177                 my $volid = $2;
5178                 eval {
5179                     if ($volid =~ m|^/|) {
5180                         unlink $volid || die 'unlink failed\n';
5181                     } else {
5182                         PVE::Storage::vdisk_free($storecfg, $volid);
5183                     }
5184                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5185                 };
5186                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5187             } else {
5188                 print STDERR "unable to parse line in statfile - $line";
5189             }
5190         }
5191         $fd->close();
5192     }
5193 }
5194
5195 sub restore_archive {
5196     my ($archive, $vmid, $user, $opts) = @_;
5197
5198     my $format = $opts->{format};
5199     my $comp;
5200
5201     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5202         $format = 'tar' if !$format;
5203         $comp = 'gzip';
5204     } elsif ($archive =~ m/\.tar$/) {
5205         $format = 'tar' if !$format;
5206     } elsif ($archive =~ m/.tar.lzo$/) {
5207         $format = 'tar' if !$format;
5208         $comp = 'lzop';
5209     } elsif ($archive =~ m/\.vma$/) {
5210         $format = 'vma' if !$format;
5211     } elsif ($archive =~ m/\.vma\.gz$/) {
5212         $format = 'vma' if !$format;
5213         $comp = 'gzip';
5214     } elsif ($archive =~ m/\.vma\.lzo$/) {
5215         $format = 'vma' if !$format;
5216         $comp = 'lzop';
5217     } else {
5218         $format = 'vma' if !$format; # default
5219     }
5220
5221     # try to detect archive format
5222     if ($format eq 'tar') {
5223         return restore_tar_archive($archive, $vmid, $user, $opts);
5224     } else {
5225         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5226     }
5227 }
5228
5229 sub restore_update_config_line {
5230     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5231
5232     return if $line =~ m/^\#qmdump\#/;
5233     return if $line =~ m/^\#vzdump\#/;
5234     return if $line =~ m/^lock:/;
5235     return if $line =~ m/^unused\d+:/;
5236     return if $line =~ m/^parent:/;
5237     return if $line =~ m/^template:/; # restored VM is never a template
5238
5239     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5240         # try to convert old 1.X settings
5241         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5242         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5243             my ($model, $macaddr) = split(/\=/, $devconfig);
5244             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5245             my $net = {
5246                 model => $model,
5247                 bridge => "vmbr$ind",
5248                 macaddr => $macaddr,
5249             };
5250             my $netstr = print_net($net);
5251
5252             print $outfd "net$cookie->{netcount}: $netstr\n";
5253             $cookie->{netcount}++;
5254         }
5255     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5256         my ($id, $netstr) = ($1, $2);
5257         my $net = parse_net($netstr);
5258         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5259         $netstr = print_net($net);
5260         print $outfd "$id: $netstr\n";
5261     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5262         my $virtdev = $1;
5263         my $value = $3;
5264         if ($line =~ m/backup=no/) {
5265             print $outfd "#$line";
5266         } elsif ($virtdev && $map->{$virtdev}) {
5267             my $di = parse_drive($virtdev, $value);
5268             delete $di->{format}; # format can change on restore
5269             $di->{file} = $map->{$virtdev};
5270             $value = print_drive($vmid, $di);
5271             print $outfd "$virtdev: $value\n";
5272         } else {
5273             print $outfd $line;
5274         }
5275     } else {
5276         print $outfd $line;
5277     }
5278 }
5279
5280 sub scan_volids {
5281     my ($cfg, $vmid) = @_;
5282
5283     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5284
5285     my $volid_hash = {};
5286     foreach my $storeid (keys %$info) {
5287         foreach my $item (@{$info->{$storeid}}) {
5288             next if !($item->{volid} && $item->{size});
5289             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5290             $volid_hash->{$item->{volid}} = $item;
5291         }
5292     }
5293
5294     return $volid_hash;
5295 }
5296
5297 sub is_volume_in_use {
5298     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5299
5300     my $path = PVE::Storage::path($storecfg, $volid);
5301
5302     my $scan_config = sub {
5303         my ($cref, $snapname) = @_;
5304
5305         foreach my $key (keys %$cref) {
5306             my $value = $cref->{$key};
5307             if (valid_drivename($key)) {
5308                 next if $skip_drive && $key eq $skip_drive;
5309                 my $drive = parse_drive($key, $value);
5310                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5311                 return 1 if $volid eq $drive->{file};
5312                 if ($drive->{file} =~ m!^/!) {
5313                     return 1 if $drive->{file} eq $path;
5314                 } else {
5315                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5316                     next if !$storeid;
5317                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5318                     next if !$scfg;
5319                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5320                 }
5321             }
5322         }
5323
5324         return 0;
5325     };
5326
5327     return 1 if &$scan_config($conf);
5328
5329     undef $skip_drive;
5330
5331     foreach my $snapname (keys %{$conf->{snapshots}}) {
5332         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5333     }
5334
5335     return 0;
5336 }
5337
5338 sub update_disksize {
5339     my ($vmid, $conf, $volid_hash) = @_;
5340
5341     my $changes;
5342
5343     my $used = {};
5344
5345     # Note: it is allowed to define multiple storages with same path (alias), so
5346     # we need to check both 'volid' and real 'path' (two different volid can point
5347     # to the same path).
5348
5349     my $usedpath = {};
5350
5351     # update size info
5352     foreach my $opt (keys %$conf) {
5353         if (valid_drivename($opt)) {
5354             my $drive = parse_drive($opt, $conf->{$opt});
5355             my $volid = $drive->{file};
5356             next if !$volid;
5357
5358             $used->{$volid} = 1;
5359             if ($volid_hash->{$volid} &&
5360                 (my $path = $volid_hash->{$volid}->{path})) {
5361                 $usedpath->{$path} = 1;
5362             }
5363
5364             next if drive_is_cdrom($drive);
5365             next if !$volid_hash->{$volid};
5366
5367             $drive->{size} = $volid_hash->{$volid}->{size};
5368             my $new = print_drive($vmid, $drive);
5369             if ($new ne $conf->{$opt}) {
5370                 $changes = 1;
5371                 $conf->{$opt} = $new;
5372             }
5373         }
5374     }
5375
5376     # remove 'unusedX' entry if volume is used
5377     foreach my $opt (keys %$conf) {
5378         next if $opt !~ m/^unused\d+$/;
5379         my $volid = $conf->{$opt};
5380         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5381         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5382             $changes = 1;
5383             delete $conf->{$opt};
5384         }
5385     }
5386
5387     foreach my $volid (sort keys %$volid_hash) {
5388         next if $volid =~ m/vm-$vmid-state-/;
5389         next if $used->{$volid};
5390         my $path = $volid_hash->{$volid}->{path};
5391         next if !$path; # just to be sure
5392         next if $usedpath->{$path};
5393         $changes = 1;
5394         add_unused_volume($conf, $volid);
5395         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5396     }
5397
5398     return $changes;
5399 }
5400
5401 sub rescan {
5402     my ($vmid, $nolock) = @_;
5403
5404     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5405
5406     my $volid_hash = scan_volids($cfg, $vmid);
5407
5408     my $updatefn =  sub {
5409         my ($vmid) = @_;
5410
5411         my $conf = load_config($vmid);
5412
5413         check_lock($conf);
5414
5415         my $vm_volids = {};
5416         foreach my $volid (keys %$volid_hash) {
5417             my $info = $volid_hash->{$volid};
5418             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5419         }
5420
5421         my $changes = update_disksize($vmid, $conf, $vm_volids);
5422
5423         update_config_nolock($vmid, $conf, 1) if $changes;
5424     };
5425
5426     if (defined($vmid)) {
5427         if ($nolock) {
5428             &$updatefn($vmid);
5429         } else {
5430             lock_config($vmid, $updatefn, $vmid);
5431         }
5432     } else {
5433         my $vmlist = config_list();
5434         foreach my $vmid (keys %$vmlist) {
5435             if ($nolock) {
5436                 &$updatefn($vmid);
5437             } else {
5438                 lock_config($vmid, $updatefn, $vmid);
5439             }
5440         }
5441     }
5442 }
5443
5444 sub restore_vma_archive {
5445     my ($archive, $vmid, $user, $opts, $comp) = @_;
5446
5447     my $input = $archive eq '-' ? "<&STDIN" : undef;
5448     my $readfrom = $archive;
5449
5450     my $uncomp = '';
5451     if ($comp) {
5452         $readfrom = '-';
5453         my $qarchive = PVE::Tools::shellquote($archive);
5454         if ($comp eq 'gzip') {
5455             $uncomp = "zcat $qarchive|";
5456         } elsif ($comp eq 'lzop') {
5457             $uncomp = "lzop -d -c $qarchive|";
5458         } else {
5459             die "unknown compression method '$comp'\n";
5460         }
5461
5462     }
5463
5464     my $tmpdir = "/var/tmp/vzdumptmp$$";
5465     rmtree $tmpdir;
5466
5467     # disable interrupts (always do cleanups)
5468     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5469         warn "got interrupt - ignored\n";
5470     };
5471
5472     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5473     POSIX::mkfifo($mapfifo, 0600);
5474     my $fifofh;
5475
5476     my $openfifo = sub {
5477         open($fifofh, '>', $mapfifo) || die $!;
5478     };
5479
5480     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5481
5482     my $oldtimeout;
5483     my $timeout = 5;
5484
5485     my $devinfo = {};
5486
5487     my $rpcenv = PVE::RPCEnvironment::get();
5488
5489     my $conffile = config_file($vmid);
5490     my $tmpfn = "$conffile.$$.tmp";
5491
5492     # Note: $oldconf is undef if VM does not exists
5493     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5494
5495     my $print_devmap = sub {
5496         my $virtdev_hash = {};
5497
5498         my $cfgfn = "$tmpdir/qemu-server.conf";
5499
5500         # we can read the config - that is already extracted
5501         my $fh = IO::File->new($cfgfn, "r") ||
5502             "unable to read qemu-server.conf - $!\n";
5503
5504         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5505         if (-f $fwcfgfn) {
5506             my $pve_firewall_dir = '/etc/pve/firewall';
5507             mkdir $pve_firewall_dir; # make sure the dir exists
5508             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5509         }
5510
5511         while (defined(my $line = <$fh>)) {
5512             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5513                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5514                 die "archive does not contain data for drive '$virtdev'\n"
5515                     if !$devinfo->{$devname};
5516                 if (defined($opts->{storage})) {
5517                     $storeid = $opts->{storage} || 'local';
5518                 } elsif (!$storeid) {
5519                     $storeid = 'local';
5520                 }
5521                 $format = 'raw' if !$format;
5522                 $devinfo->{$devname}->{devname} = $devname;
5523                 $devinfo->{$devname}->{virtdev} = $virtdev;
5524                 $devinfo->{$devname}->{format} = $format;
5525                 $devinfo->{$devname}->{storeid} = $storeid;
5526
5527                 # check permission on storage
5528                 my $pool = $opts->{pool}; # todo: do we need that?
5529                 if ($user ne 'root@pam') {
5530                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5531                 }
5532
5533                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5534             }
5535         }
5536
5537         foreach my $devname (keys %$devinfo) {
5538             die "found no device mapping information for device '$devname'\n"
5539                 if !$devinfo->{$devname}->{virtdev};
5540         }
5541
5542         my $cfg = cfs_read_file('storage.cfg');
5543
5544         # create empty/temp config
5545         if ($oldconf) {
5546             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5547             foreach_drive($oldconf, sub {
5548                 my ($ds, $drive) = @_;
5549
5550                 return if drive_is_cdrom($drive);
5551
5552                 my $volid = $drive->{file};
5553
5554                 return if !$volid || $volid =~ m|^/|;
5555
5556                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5557                 return if !$path || !$owner || ($owner != $vmid);
5558
5559                 # Note: only delete disk we want to restore
5560                 # other volumes will become unused
5561                 if ($virtdev_hash->{$ds}) {
5562                     PVE::Storage::vdisk_free($cfg, $volid);
5563                 }
5564             });
5565         }
5566
5567         my $map = {};
5568         foreach my $virtdev (sort keys %$virtdev_hash) {
5569             my $d = $virtdev_hash->{$virtdev};
5570             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5571             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5572
5573             # test if requested format is supported
5574             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5575             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5576             $d->{format} = $defFormat if !$supported;
5577
5578             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5579                                                   $d->{format}, undef, $alloc_size);
5580             print STDERR "new volume ID is '$volid'\n";
5581             $d->{volid} = $volid;
5582             my $path = PVE::Storage::path($cfg, $volid);
5583
5584             PVE::Storage::activate_volumes($cfg,[$volid]);
5585
5586             my $write_zeros = 1;
5587             # fixme: what other storages types initialize volumes with zero?
5588             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5589                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5590                 $write_zeros = 0;
5591             }
5592
5593             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5594
5595             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5596             $map->{$virtdev} = $volid;
5597         }
5598
5599         $fh->seek(0, 0) || die "seek failed - $!\n";
5600
5601         my $outfd = new IO::File ($tmpfn, "w") ||
5602             die "unable to write config for VM $vmid\n";
5603
5604         my $cookie = { netcount => 0 };
5605         while (defined(my $line = <$fh>)) {
5606             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5607         }
5608
5609         $fh->close();
5610         $outfd->close();
5611     };
5612
5613     eval {
5614         # enable interrupts
5615         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5616             die "interrupted by signal\n";
5617         };
5618         local $SIG{ALRM} = sub { die "got timeout\n"; };
5619
5620         $oldtimeout = alarm($timeout);
5621
5622         my $parser = sub {
5623             my $line = shift;
5624
5625             print "$line\n";
5626
5627             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5628                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5629                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5630             } elsif ($line =~ m/^CTIME: /) {
5631                 # we correctly received the vma config, so we can disable
5632                 # the timeout now for disk allocation (set to 10 minutes, so
5633                 # that we always timeout if something goes wrong)
5634                 alarm(600);
5635                 &$print_devmap();
5636                 print $fifofh "done\n";
5637                 my $tmp = $oldtimeout || 0;
5638                 $oldtimeout = undef;
5639                 alarm($tmp);
5640                 close($fifofh);
5641             }
5642         };
5643
5644         print "restore vma archive: $cmd\n";
5645         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5646     };
5647     my $err = $@;
5648
5649     alarm($oldtimeout) if $oldtimeout;
5650
5651     my $vollist = [];
5652     foreach my $devname (keys %$devinfo) {
5653         my $volid = $devinfo->{$devname}->{volid};
5654         push @$vollist, $volid if $volid;
5655     }
5656
5657     my $cfg = cfs_read_file('storage.cfg');
5658     PVE::Storage::deactivate_volumes($cfg, $vollist);
5659
5660     unlink $mapfifo;
5661
5662     if ($err) {
5663         rmtree $tmpdir;
5664         unlink $tmpfn;
5665
5666         foreach my $devname (keys %$devinfo) {
5667             my $volid = $devinfo->{$devname}->{volid};
5668             next if !$volid;
5669             eval {
5670                 if ($volid =~ m|^/|) {
5671                     unlink $volid || die 'unlink failed\n';
5672                 } else {
5673                     PVE::Storage::vdisk_free($cfg, $volid);
5674                 }
5675                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5676             };
5677             print STDERR "unable to cleanup '$volid' - $@" if $@;
5678         }
5679         die $err;
5680     }
5681
5682     rmtree $tmpdir;
5683
5684     rename($tmpfn, $conffile) ||
5685         die "unable to commit configuration file '$conffile'\n";
5686
5687     PVE::Cluster::cfs_update(); # make sure we read new file
5688
5689     eval { rescan($vmid, 1); };
5690     warn $@ if $@;
5691 }
5692
5693 sub restore_tar_archive {
5694     my ($archive, $vmid, $user, $opts) = @_;
5695
5696     if ($archive ne '-') {
5697         my $firstfile = tar_archive_read_firstfile($archive);
5698         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5699             if $firstfile ne 'qemu-server.conf';
5700     }
5701
5702     my $storecfg = cfs_read_file('storage.cfg');
5703
5704     # destroy existing data - keep empty config
5705     my $vmcfgfn = config_file($vmid);
5706     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5707
5708     my $tocmd = "/usr/lib/qemu-server/qmextract";
5709
5710     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5711     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5712     $tocmd .= ' --prealloc' if $opts->{prealloc};
5713     $tocmd .= ' --info' if $opts->{info};
5714
5715     # tar option "xf" does not autodetect compression when read from STDIN,
5716     # so we pipe to zcat
5717     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5718         PVE::Tools::shellquote("--to-command=$tocmd");
5719
5720     my $tmpdir = "/var/tmp/vzdumptmp$$";
5721     mkpath $tmpdir;
5722
5723     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5724     local $ENV{VZDUMP_VMID} = $vmid;
5725     local $ENV{VZDUMP_USER} = $user;
5726
5727     my $conffile = config_file($vmid);
5728     my $tmpfn = "$conffile.$$.tmp";
5729
5730     # disable interrupts (always do cleanups)
5731     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5732         print STDERR "got interrupt - ignored\n";
5733     };
5734
5735     eval {
5736         # enable interrupts
5737         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5738             die "interrupted by signal\n";
5739         };
5740
5741         if ($archive eq '-') {
5742             print "extracting archive from STDIN\n";
5743             run_command($cmd, input => "<&STDIN");
5744         } else {
5745             print "extracting archive '$archive'\n";
5746             run_command($cmd);
5747         }
5748
5749         return if $opts->{info};
5750
5751         # read new mapping
5752         my $map = {};
5753         my $statfile = "$tmpdir/qmrestore.stat";
5754         if (my $fd = IO::File->new($statfile, "r")) {
5755             while (defined (my $line = <$fd>)) {
5756                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5757                     $map->{$1} = $2 if $1;
5758                 } else {
5759                     print STDERR "unable to parse line in statfile - $line\n";
5760                 }
5761             }
5762             $fd->close();
5763         }
5764
5765         my $confsrc = "$tmpdir/qemu-server.conf";
5766
5767         my $srcfd = new IO::File($confsrc, "r") ||
5768             die "unable to open file '$confsrc'\n";
5769
5770         my $outfd = new IO::File ($tmpfn, "w") ||
5771             die "unable to write config for VM $vmid\n";
5772
5773         my $cookie = { netcount => 0 };
5774         while (defined (my $line = <$srcfd>)) {
5775             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5776         }
5777
5778         $srcfd->close();
5779         $outfd->close();
5780     };
5781     my $err = $@;
5782
5783     if ($err) {
5784
5785         unlink $tmpfn;
5786
5787         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5788
5789         die $err;
5790     }
5791
5792     rmtree $tmpdir;
5793
5794     rename $tmpfn, $conffile ||
5795         die "unable to commit configuration file '$conffile'\n";
5796
5797     PVE::Cluster::cfs_update(); # make sure we read new file
5798
5799     eval { rescan($vmid, 1); };
5800     warn $@ if $@;
5801 };
5802
5803
5804 # Internal snapshots
5805
5806 # NOTE: Snapshot create/delete involves several non-atomic
5807 # action, and can take a long time.
5808 # So we try to avoid locking the file and use 'lock' variable
5809 # inside the config file instead.
5810
5811 my $snapshot_copy_config = sub {
5812     my ($source, $dest) = @_;
5813
5814     foreach my $k (keys %$source) {
5815         next if $k eq 'snapshots';
5816         next if $k eq 'snapstate';
5817         next if $k eq 'snaptime';
5818         next if $k eq 'vmstate';
5819         next if $k eq 'lock';
5820         next if $k eq 'digest';
5821         next if $k eq 'description';
5822         next if $k =~ m/^unused\d+$/;
5823
5824         $dest->{$k} = $source->{$k};
5825     }
5826 };
5827
5828 my $snapshot_apply_config = sub {
5829     my ($conf, $snap) = @_;
5830
5831     # copy snapshot list
5832     my $newconf = {
5833         snapshots => $conf->{snapshots},
5834     };
5835
5836     # keep description and list of unused disks
5837     foreach my $k (keys %$conf) {
5838         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5839         $newconf->{$k} = $conf->{$k};
5840     }
5841
5842     &$snapshot_copy_config($snap, $newconf);
5843
5844     return $newconf;
5845 };
5846
5847 sub foreach_writable_storage {
5848     my ($conf, $func) = @_;
5849
5850     my $sidhash = {};
5851
5852     foreach my $ds (keys %$conf) {
5853         next if !valid_drivename($ds);
5854
5855         my $drive = parse_drive($ds, $conf->{$ds});
5856         next if !$drive;
5857         next if drive_is_cdrom($drive);
5858
5859         my $volid = $drive->{file};
5860
5861         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5862         $sidhash->{$sid} = $sid if $sid;
5863     }
5864
5865     foreach my $sid (sort keys %$sidhash) {
5866         &$func($sid);
5867     }
5868 }
5869
5870 my $alloc_vmstate_volid = sub {
5871     my ($storecfg, $vmid, $conf, $snapname) = @_;
5872
5873     # Note: we try to be smart when selecting a $target storage
5874
5875     my $target;
5876
5877     # search shared storage first
5878     foreach_writable_storage($conf, sub {
5879         my ($sid) = @_;
5880         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5881         return if !$scfg->{shared};
5882
5883         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5884     });
5885
5886     if (!$target) {
5887         # now search local storage
5888         foreach_writable_storage($conf, sub {
5889             my ($sid) = @_;
5890             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5891             return if $scfg->{shared};
5892
5893             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5894         });
5895     }
5896
5897     $target = 'local' if !$target;
5898
5899     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5900     # we abort live save after $conf->{memory}, so we need at max twice that space
5901     my $size = $conf->{memory}*2 + $driver_state_size;
5902
5903     my $name = "vm-$vmid-state-$snapname";
5904     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5905     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5906     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5907
5908     return $volid;
5909 };
5910
5911 my $snapshot_prepare = sub {
5912     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5913
5914     my $snap;
5915
5916     my $updatefn =  sub {
5917
5918         my $conf = load_config($vmid);
5919
5920         die "you can't take a snapshot if it's a template\n"
5921             if is_template($conf);
5922
5923         check_lock($conf);
5924
5925         $conf->{lock} = 'snapshot';
5926
5927         die "snapshot name '$snapname' already used\n"
5928             if defined($conf->{snapshots}->{$snapname});
5929
5930         my $storecfg = PVE::Storage::config();
5931         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5932
5933         $snap = $conf->{snapshots}->{$snapname} = {};
5934
5935         if ($save_vmstate && check_running($vmid)) {
5936             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5937         }
5938
5939         &$snapshot_copy_config($conf, $snap);
5940
5941         $snap->{snapstate} = "prepare";
5942         $snap->{snaptime} = time();
5943         $snap->{description} = $comment if $comment;
5944
5945         # always overwrite machine if we save vmstate. This makes sure we
5946         # can restore it later using correct machine type
5947         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5948
5949         update_config_nolock($vmid, $conf, 1);
5950     };
5951
5952     lock_config($vmid, $updatefn);
5953
5954     return $snap;
5955 };
5956
5957 my $snapshot_commit = sub {
5958     my ($vmid, $snapname) = @_;
5959
5960     my $updatefn = sub {
5961
5962         my $conf = load_config($vmid);
5963
5964         die "missing snapshot lock\n"
5965             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5966
5967         my $has_machine_config = defined($conf->{machine});
5968
5969         my $snap = $conf->{snapshots}->{$snapname};
5970
5971         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5972
5973         die "wrong snapshot state\n"
5974             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5975
5976         delete $snap->{snapstate};
5977         delete $conf->{lock};
5978
5979         my $newconf = &$snapshot_apply_config($conf, $snap);
5980
5981         delete $newconf->{machine} if !$has_machine_config;
5982
5983         $newconf->{parent} = $snapname;
5984
5985         update_config_nolock($vmid, $newconf, 1);
5986     };
5987
5988     lock_config($vmid, $updatefn);
5989 };
5990
5991 sub snapshot_rollback {
5992     my ($vmid, $snapname) = @_;
5993
5994     my $prepare = 1;
5995
5996     my $storecfg = PVE::Storage::config();
5997
5998     my $conf = load_config($vmid);
5999
6000     my $get_snapshot_config = sub {
6001
6002         die "you can't rollback if vm is a template\n" if is_template($conf);
6003
6004         my $res = $conf->{snapshots}->{$snapname};
6005
6006         die "snapshot '$snapname' does not exist\n" if !defined($res);
6007
6008         return $res;
6009     };
6010
6011     my $snap = &$get_snapshot_config();
6012
6013     foreach_drive($snap, sub {
6014         my ($ds, $drive) = @_;
6015
6016         return if drive_is_cdrom($drive);
6017
6018         my $volid = $drive->{file};
6019
6020         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
6021     });
6022
6023     my $updatefn = sub {
6024
6025         $conf = load_config($vmid);
6026
6027         $snap = &$get_snapshot_config();
6028
6029         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6030             if $snap->{snapstate};
6031
6032         if ($prepare) {
6033             check_lock($conf);
6034             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6035         }
6036
6037         die "unable to rollback vm $vmid: vm is running\n"
6038             if check_running($vmid);
6039
6040         if ($prepare) {
6041             $conf->{lock} = 'rollback';
6042         } else {
6043             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6044             delete $conf->{lock};
6045         }
6046
6047         my $forcemachine;
6048
6049         if (!$prepare) {
6050             my $has_machine_config = defined($conf->{machine});
6051
6052             # copy snapshot config to current config
6053             $conf = &$snapshot_apply_config($conf, $snap);
6054             $conf->{parent} = $snapname;
6055
6056             # Note: old code did not store 'machine', so we try to be smart
6057             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6058             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6059             # we remove the 'machine' configuration if not explicitly specified
6060             # in the original config.
6061             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6062         }
6063
6064         update_config_nolock($vmid, $conf, 1);
6065
6066         if (!$prepare && $snap->{vmstate}) {
6067             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6068             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6069         }
6070     };
6071
6072     lock_config($vmid, $updatefn);
6073
6074     foreach_drive($snap, sub {
6075         my ($ds, $drive) = @_;
6076
6077         return if drive_is_cdrom($drive);
6078
6079         my $volid = $drive->{file};
6080         my $device = "drive-$ds";
6081
6082         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6083     });
6084
6085     $prepare = 0;
6086     lock_config($vmid, $updatefn);
6087 }
6088
6089 my $savevm_wait = sub {
6090     my ($vmid) = @_;
6091
6092     for(;;) {
6093         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6094         if (!$stat->{status}) {
6095             die "savevm not active\n";
6096         } elsif ($stat->{status} eq 'active') {
6097             sleep(1);
6098             next;
6099         } elsif ($stat->{status} eq 'completed') {
6100             last;
6101         } else {
6102             die "query-savevm returned status '$stat->{status}'\n";
6103         }
6104     }
6105 };
6106
6107 sub do_snapshots_with_qemu {
6108     my ($storecfg, $volid) = @_;
6109
6110     my $storage_name = PVE::Storage::parse_volume_id($volid);
6111
6112     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6113         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6114         return 1;
6115     }
6116
6117     if ($volid =~ m/\.(qcow2|qed)$/){
6118         return 1;
6119     }
6120
6121     return undef;
6122 }
6123
6124 sub snapshot_create {
6125     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6126
6127     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6128
6129     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6130
6131     my $config = load_config($vmid);
6132
6133     my $running = check_running($vmid);
6134
6135     my $freezefs = $running && $config->{agent};
6136     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6137
6138     my $drivehash = {};
6139
6140     if ($freezefs) {
6141         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6142         warn "guest-fsfreeze-freeze problems - $@" if $@;
6143     }
6144
6145     eval {
6146         # create internal snapshots of all drives
6147
6148         my $storecfg = PVE::Storage::config();
6149
6150         if ($running) {
6151             if ($snap->{vmstate}) {
6152                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6153                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6154                 &$savevm_wait($vmid);
6155             } else {
6156                 vm_mon_cmd($vmid, "savevm-start");
6157             }
6158         };
6159
6160         foreach_drive($snap, sub {
6161             my ($ds, $drive) = @_;
6162
6163             return if drive_is_cdrom($drive);
6164
6165             my $volid = $drive->{file};
6166             my $device = "drive-$ds";
6167
6168             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6169             $drivehash->{$ds} = 1;
6170        });
6171     };
6172     my $err = $@;
6173
6174     if ($running) {
6175         eval { vm_mon_cmd($vmid, "savevm-end")  };
6176         warn $@ if $@;
6177
6178         if ($freezefs) {
6179             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6180             warn "guest-fsfreeze-thaw problems - $@" if $@;
6181         }
6182
6183         # savevm-end is async, we need to wait
6184         for (;;) {
6185             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6186             if (!$stat->{bytes}) {
6187                 last;
6188             } else {
6189                 print "savevm not yet finished\n";
6190                 sleep(1);
6191                 next;
6192             }
6193         }
6194     }
6195
6196     if ($err) {
6197         warn "snapshot create failed: starting cleanup\n";
6198         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6199         warn $@ if $@;
6200         die $err;
6201     }
6202
6203     &$snapshot_commit($vmid, $snapname);
6204 }
6205
6206 # Note: $drivehash is only set when called from snapshot_create.
6207 sub snapshot_delete {
6208     my ($vmid, $snapname, $force, $drivehash) = @_;
6209
6210     my $prepare = 1;
6211
6212     my $snap;
6213     my $unused = [];
6214
6215     my $unlink_parent = sub {
6216         my ($confref, $new_parent) = @_;
6217
6218         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6219             if ($new_parent) {
6220                 $confref->{parent} = $new_parent;
6221             } else {
6222                 delete $confref->{parent};
6223             }
6224         }
6225     };
6226
6227     my $updatefn =  sub {
6228         my ($remove_drive) = @_;
6229
6230         my $conf = load_config($vmid);
6231
6232         if (!$drivehash) {
6233             check_lock($conf);
6234             die "you can't delete a snapshot if vm is a template\n"
6235                 if is_template($conf);
6236         }
6237
6238         $snap = $conf->{snapshots}->{$snapname};
6239
6240         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6241
6242         # remove parent refs
6243         if (!$prepare) {
6244             &$unlink_parent($conf, $snap->{parent});
6245             foreach my $sn (keys %{$conf->{snapshots}}) {
6246                 next if $sn eq $snapname;
6247                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6248             }
6249         }
6250
6251         if ($remove_drive) {
6252             if ($remove_drive eq 'vmstate') {
6253                 delete $snap->{$remove_drive};
6254             } else {
6255                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6256                 my $volid = $drive->{file};
6257                 delete $snap->{$remove_drive};
6258                 add_unused_volume($conf, $volid);
6259             }
6260         }
6261
6262         if ($prepare) {
6263             $snap->{snapstate} = 'delete';
6264         } else {
6265             delete $conf->{snapshots}->{$snapname};
6266             delete $conf->{lock} if $drivehash;
6267             foreach my $volid (@$unused) {
6268                 add_unused_volume($conf, $volid);
6269             }
6270         }
6271
6272         update_config_nolock($vmid, $conf, 1);
6273     };
6274
6275     lock_config($vmid, $updatefn);
6276
6277     # now remove vmstate file
6278
6279     my $storecfg = PVE::Storage::config();
6280
6281     if ($snap->{vmstate}) {
6282         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6283         if (my $err = $@) {
6284             die $err if !$force;
6285             warn $err;
6286         }
6287         # save changes (remove vmstate from snapshot)
6288         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6289     };
6290
6291     # now remove all internal snapshots
6292     foreach_drive($snap, sub {
6293         my ($ds, $drive) = @_;
6294
6295         return if drive_is_cdrom($drive);
6296
6297         my $volid = $drive->{file};
6298         my $device = "drive-$ds";
6299
6300         if (!$drivehash || $drivehash->{$ds}) {
6301             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6302             if (my $err = $@) {
6303                 die $err if !$force;
6304                 warn $err;
6305             }
6306         }
6307
6308         # save changes (remove drive fron snapshot)
6309         lock_config($vmid, $updatefn, $ds) if !$force;
6310         push @$unused, $volid;
6311     });
6312
6313     # now cleanup config
6314     $prepare = 0;
6315     lock_config($vmid, $updatefn);
6316 }
6317
6318 sub has_feature {
6319     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6320
6321     my $err;
6322     foreach_drive($conf, sub {
6323         my ($ds, $drive) = @_;
6324
6325         return if drive_is_cdrom($drive);
6326         my $volid = $drive->{file};
6327         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6328     });
6329
6330     return $err ? 0 : 1;
6331 }
6332
6333 sub template_create {
6334     my ($vmid, $conf, $disk) = @_;
6335
6336     my $storecfg = PVE::Storage::config();
6337
6338     foreach_drive($conf, sub {
6339         my ($ds, $drive) = @_;
6340
6341         return if drive_is_cdrom($drive);
6342         return if $disk && $ds ne $disk;
6343
6344         my $volid = $drive->{file};
6345         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6346
6347         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6348         $drive->{file} = $voliddst;
6349         $conf->{$ds} = print_drive($vmid, $drive);
6350         update_config_nolock($vmid, $conf, 1);
6351     });
6352 }
6353
6354 sub is_template {
6355     my ($conf) = @_;
6356
6357     return 1 if defined $conf->{template} && $conf->{template} == 1;
6358 }
6359
6360 sub qemu_img_convert {
6361     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6362
6363     my $storecfg = PVE::Storage::config();
6364     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6365     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6366
6367     if ($src_storeid && $dst_storeid) {
6368
6369         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6370
6371         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6372         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6373
6374         my $src_format = qemu_img_format($src_scfg, $src_volname);
6375         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6376
6377         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6378         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6379
6380         my $cmd = [];
6381         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6382         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6383         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6384
6385         my $parser = sub {
6386             my $line = shift;
6387             if($line =~ m/\((\S+)\/100\%\)/){
6388                 my $percent = $1;
6389                 my $transferred = int($size * $percent / 100);
6390                 my $remaining = $size - $transferred;
6391
6392                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6393             }
6394
6395         };
6396
6397         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6398         my $err = $@;
6399         die "copy failed: $err" if $err;
6400     }
6401 }
6402
6403 sub qemu_img_format {
6404     my ($scfg, $volname) = @_;
6405
6406     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6407         return $1;
6408     } else {
6409         return "raw";
6410     }
6411 }
6412
6413 sub qemu_drive_mirror {
6414     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6415
6416     my $storecfg = PVE::Storage::config();
6417     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6418
6419     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6420
6421     my $format = qemu_img_format($dst_scfg, $dst_volname);
6422
6423     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6424
6425     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6426     $opts->{format} = $format if $format;
6427
6428     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6429
6430     eval {
6431     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6432         while (1) {
6433             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6434             my $stat = @$stats[0];
6435             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6436             die "error job is not mirroring" if $stat->{type} ne "mirror";
6437
6438             my $busy = $stat->{busy};
6439             my $ready = $stat->{ready};
6440
6441             if (my $total = $stat->{len}) {
6442                 my $transferred = $stat->{offset} || 0;
6443                 my $remaining = $total - $transferred;
6444                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6445
6446                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6447             }
6448
6449
6450             if ($stat->{ready} eq 'true') {
6451
6452                 last if $vmiddst != $vmid;
6453
6454                 # try to switch the disk if source and destination are on the same guest
6455                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6456                 last if !$@;
6457                 die $@ if $@ !~ m/cannot be completed/;
6458             }
6459             sleep 1;
6460         }
6461
6462
6463     };
6464     my $err = $@;
6465
6466     my $cancel_job = sub {
6467         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6468         while (1) {
6469             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6470             my $stat = @$stats[0];
6471             last if !$stat;
6472             sleep 1;
6473         }
6474     };
6475
6476     if ($err) {
6477         eval { &$cancel_job(); };
6478         die "mirroring error: $err";
6479     }
6480
6481     if ($vmiddst != $vmid) {
6482         # if we clone a disk for a new target vm, we don't switch the disk
6483         &$cancel_job(); # so we call block-job-cancel
6484     }
6485 }
6486
6487 sub clone_disk {
6488     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6489         $newvmid, $storage, $format, $full, $newvollist) = @_;
6490
6491     my $newvolid;
6492
6493     if (!$full) {
6494         print "create linked clone of drive $drivename ($drive->{file})\n";
6495         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6496         push @$newvollist, $newvolid;
6497     } else {
6498         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6499         $storeid = $storage if $storage;
6500
6501         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6502         if (!$format) {
6503             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6504             $format = qemu_img_format($scfg, $volname);
6505         }
6506
6507         # test if requested format is supported - else use default
6508         my $supported = grep { $_ eq $format } @$validFormats;
6509         $format = $defFormat if !$supported;
6510
6511         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6512
6513         print "create full clone of drive $drivename ($drive->{file})\n";
6514         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6515         push @$newvollist, $newvolid;
6516
6517         PVE::Storage::activate_volumes($storecfg, $newvollist);
6518
6519         if (!$running || $snapname) {
6520             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6521         } else {
6522             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6523         }
6524     }
6525
6526     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6527
6528     my $disk = $drive;
6529     $disk->{format} = undef;
6530     $disk->{file} = $newvolid;
6531     $disk->{size} = $size;
6532
6533     return $disk;
6534 }
6535
6536 # this only works if VM is running
6537 sub get_current_qemu_machine {
6538     my ($vmid) = @_;
6539
6540     my $cmd = { execute => 'query-machines', arguments => {} };
6541     my $res = vm_qmp_command($vmid, $cmd);
6542
6543     my ($current, $default);
6544     foreach my $e (@$res) {
6545         $default = $e->{name} if $e->{'is-default'};
6546         $current = $e->{name} if $e->{'is-current'};
6547     }
6548
6549     # fallback to the default machine if current is not supported by qemu
6550     return $current || $default || 'pc';
6551 }
6552
6553 sub qemu_machine_feature_enabled {
6554     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6555
6556     my $current_major;
6557     my $current_minor;
6558
6559     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6560
6561         $current_major = $3;
6562         $current_minor = $4;
6563
6564     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6565
6566         $current_major = $1;
6567         $current_minor = $2;
6568     }
6569
6570     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6571
6572
6573 }
6574
6575 sub qemu_machine_pxe {
6576     my ($vmid, $conf, $machine) = @_;
6577
6578     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6579
6580     foreach my $opt (keys %$conf) {
6581         next if $opt !~ m/^net(\d+)$/;
6582         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6583         next if !$net;
6584         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6585         return $machine.".pxe" if $romfile =~ m/pxe/;
6586         last;
6587     }
6588
6589 }
6590
6591 sub qemu_use_old_bios_files {
6592     my ($machine_type) = @_;
6593
6594     return if !$machine_type;
6595
6596     my $use_old_bios_files = undef;
6597
6598     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6599         $machine_type = $1;
6600         $use_old_bios_files = 1;
6601     } else {
6602         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6603         # load new efi bios files on migration. So this hack is required to allow
6604         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6605         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6606         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6607     }
6608
6609     return ($use_old_bios_files, $machine_type);
6610 }
6611
6612 sub lspci {
6613
6614     my $devices = {};
6615
6616     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6617             my (undef, $id, $function) = @_;
6618             my $res = { id => $id, function => $function};
6619             push @{$devices->{$id}}, $res;
6620     });
6621
6622     return $devices;
6623 }
6624
6625 sub vm_iothreads_list {
6626     my ($vmid) = @_;
6627
6628     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6629
6630     my $iothreads = {};
6631     foreach my $iothread (@$res) {
6632         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6633     }
6634
6635     return $iothreads;
6636 }
6637
6638 sub scsihw_infos {
6639     my ($conf, $drive) = @_;
6640
6641     my $maxdev = 0;
6642
6643     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6644         $maxdev = 7;
6645     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6646         $maxdev = 1;
6647     } else {
6648         $maxdev = 256;
6649     }
6650
6651     my $controller = int($drive->{index} / $maxdev);
6652     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6653
6654     return ($maxdev, $controller, $controller_prefix);
6655 }
6656
6657 # bash completion helper
6658
6659 sub complete_backup_archives {
6660     my ($cmdname, $pname, $cvalue) = @_;
6661
6662     my $cfg = PVE::Storage::config();
6663
6664     my $storeid;
6665
6666     if ($cvalue =~ m/^([^:]+):/) {
6667         $storeid = $1;
6668     }
6669
6670     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6671
6672     my $res = [];
6673     foreach my $id (keys %$data) {
6674         foreach my $item (@{$data->{$id}}) {
6675             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6676             push @$res, $item->{volid} if defined($item->{volid});
6677         }
6678     }
6679
6680     return $res;
6681 }
6682
6683 my $complete_vmid_full = sub {
6684     my ($running) = @_;
6685
6686     my $idlist = vmstatus();
6687
6688     my $res = [];
6689
6690     foreach my $id (keys %$idlist) {
6691         my $d = $idlist->{$id};
6692         if (defined($running)) {
6693             next if $d->{template};
6694             next if $running && $d->{status} ne 'running';
6695             next if !$running && $d->{status} eq 'running';
6696         }
6697         push @$res, $id;
6698
6699     }
6700     return $res;
6701 };
6702
6703 sub complete_vmid {
6704     return &$complete_vmid_full();
6705 }
6706
6707 sub complete_vmid_stopped {
6708     return &$complete_vmid_full(0);
6709 }
6710
6711 sub complete_vmid_running {
6712     return &$complete_vmid_full(1);
6713 }
6714
6715 sub complete_storage {
6716
6717     my $cfg = PVE::Storage::config();
6718     my $ids = $cfg->{ids};
6719
6720     my $res = [];
6721     foreach my $sid (keys %$ids) {
6722         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6723         next if !$ids->{$sid}->{content}->{images};
6724         push @$res, $sid;
6725     }
6726
6727     return $res;
6728 }
6729
6730 1;