PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbformat = {
 731     host => {
 732         default_key => 1,
 733         type => 'string', format => 'pve-qm-usb-device',
 734         format_description => 'HOSTUSBDEVICE|spice',
 735         description => 'The Host USB device or port or the value spice',
 736     },
 737     usb3 => {
 738         optional => 1,
 739         type => 'boolean',
 740         format_description => 'yes|no',
 741         description => 'Specifies whether if given host option is a USB3 device or port',
 742     },
 743 };
 744
 745 my $usbdesc = {
 746     optional => 1,
 747     type => 'string', format => $usbformat,
 748     description => <<EODESCR,
 749 Configure an USB device (n is 0 to 4). This can be used to
 750 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 751
 752 'bus-port(.port)*' (decimal numbers) or
 753 'vendor_id:product_id' (hexadeciaml numbers) or
 754 'spice'
 755
 756 You can use the 'lsusb -t' command to list existing usb devices.
 757
 758 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 759
 760 The value 'spice' can be used to add a usb redirection devices for spice.
 761
 762 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 763
 764 EODESCR
 765 };
 766 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 767
 768 my $hostpcidesc = {
 769         optional => 1,
 770         type => 'string', format => 'pve-qm-hostpci',
 771         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 772         description => <<EODESCR,
 773 Map host pci devices. HOSTPCIDEVICE syntax is:
 774
 775 'bus:dev.func' (hexadecimal numbers)
 776
 777 You can us the 'lspci' command to list existing pci devices.
 778
 779 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 780
 781 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 782
 783 Experimental: user reported problems with this option.
 784 EODESCR
 785 };
 786 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 787
 788 my $serialdesc = {
 789         optional => 1,
 790         type => 'string',
 791         pattern => '(/dev/.+|socket)',
 792         description =>  <<EODESCR,
 793 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 794
 795 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 796
 797 Experimental: user reported problems with this option.
 798 EODESCR
 799 };
 800
 801 my $paralleldesc= {
 802         optional => 1,
 803         type => 'string',
 804         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 805         description =>  <<EODESCR,
 806 Map host parallel devices (n is 0 to 2).
 807
 808 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 809
 810 Experimental: user reported problems with this option.
 811 EODESCR
 812 };
 813
 814 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 815     $confdesc->{"parallel$i"} = $paralleldesc;
 816 }
 817
 818 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 819     $confdesc->{"serial$i"} = $serialdesc;
 820 }
 821
 822 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 823     $confdesc->{"hostpci$i"} = $hostpcidesc;
 824 }
 825
 826 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 827     $drivename_hash->{"ide$i"} = 1;
 828     $confdesc->{"ide$i"} = $idedesc;
 829 }
 830
 831 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 832     $drivename_hash->{"sata$i"} = 1;
 833     $confdesc->{"sata$i"} = $satadesc;
 834 }
 835
 836 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 837     $drivename_hash->{"scsi$i"} = 1;
 838     $confdesc->{"scsi$i"} = $scsidesc ;
 839 }
 840
 841 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 842     $drivename_hash->{"virtio$i"} = 1;
 843     $confdesc->{"virtio$i"} = $virtiodesc;
 844 }
 845
 846 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 847     $confdesc->{"usb$i"} = $usbdesc;
 848 }
 849
 850 my $unuseddesc = {
 851     optional => 1,
 852     type => 'string', format => 'pve-volume-id',
 853     description => "Reference to unused volumes.",
 854 };
 855
 856 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 857     $confdesc->{"unused$i"} = $unuseddesc;
 858 }
 859
 860 my $kvm_api_version = 0;
 861
 862 sub kvm_version {
 863
 864     return $kvm_api_version if $kvm_api_version;
 865
 866     my $fh = IO::File->new("</dev/kvm") ||
 867         return 0;
 868
 869     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 870         $kvm_api_version = $v;
 871     }
 872
 873     $fh->close();
 874
 875     return  $kvm_api_version;
 876 }
 877
 878 my $kvm_user_version;
 879
 880 sub kvm_user_version {
 881
 882     return $kvm_user_version if $kvm_user_version;
 883
 884     $kvm_user_version = 'unknown';
 885
 886     my $code = sub {
 887         my $line = shift;
 888         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 889             $kvm_user_version = $2;
 890         }
 891     };
 892
 893     eval { run_command("kvm -version", outfunc => $code); };
 894     warn $@ if $@;
 895
 896     return $kvm_user_version;
 897
 898 }
 899
 900 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 901
 902 sub disknames {
 903     # order is important - used to autoselect boot disk
 904     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 905             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 906             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 907             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 908 }
 909
 910 sub valid_drivename {
 911     my $dev = shift;
 912
 913     return defined($drivename_hash->{$dev});
 914 }
 915
 916 sub option_exists {
 917     my $key = shift;
 918     return defined($confdesc->{$key});
 919 }
 920
 921 sub nic_models {
 922     return $nic_model_list;
 923 }
 924
 925 sub os_list_description {
 926
 927     return {
 928         other => 'Other',
 929         wxp => 'Windows XP',
 930         w2k => 'Windows 2000',
 931         w2k3 =>, 'Windows 2003',
 932         w2k8 => 'Windows 2008',
 933         wvista => 'Windows Vista',
 934         win7 => 'Windows 7',
 935         win8 => 'Windows 8/2012',
 936         l24 => 'Linux 2.4',
 937         l26 => 'Linux 2.6',
 938     };
 939 }
 940
 941 my $cdrom_path;
 942
 943 sub get_cdrom_path {
 944
 945     return  $cdrom_path if $cdrom_path;
 946
 947     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 948     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 949     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 950 }
 951
 952 sub get_iso_path {
 953     my ($storecfg, $vmid, $cdrom) = @_;
 954
 955     if ($cdrom eq 'cdrom') {
 956         return get_cdrom_path();
 957     } elsif ($cdrom eq 'none') {
 958         return '';
 959     } elsif ($cdrom =~ m|^/|) {
 960         return $cdrom;
 961     } else {
 962         return PVE::Storage::path($storecfg, $cdrom);
 963     }
 964 }
 965
 966 # try to convert old style file names to volume IDs
 967 sub filename_to_volume_id {
 968     my ($vmid, $file, $media) = @_;
 969
 970     if (!($file eq 'none' || $file eq 'cdrom' ||
 971           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 972
 973         return undef if $file =~ m|/|;
 974
 975         if ($media && $media eq 'cdrom') {
 976             $file = "local:iso/$file";
 977         } else {
 978             $file = "local:$vmid/$file";
 979         }
 980     }
 981
 982     return $file;
 983 }
 984
 985 sub verify_media_type {
 986     my ($opt, $vtype, $media) = @_;
 987
 988     return if !$media;
 989
 990     my $etype;
 991     if ($media eq 'disk') {
 992         $etype = 'images';
 993     } elsif ($media eq 'cdrom') {
 994         $etype = 'iso';
 995     } else {
 996         die "internal error";
 997     }
 998
 999     return if ($vtype eq $etype);
1000
1001     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1002 }
1003
1004 sub cleanup_drive_path {
1005     my ($opt, $storecfg, $drive) = @_;
1006
1007     # try to convert filesystem paths to volume IDs
1008
1009     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1010         ($drive->{file} !~ m|^/dev/.+|) &&
1011         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1012         ($drive->{file} !~ m/^\d+$/)) {
1013         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1014         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1015         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1016         verify_media_type($opt, $vtype, $drive->{media});
1017         $drive->{file} = $volid;
1018     }
1019
1020     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1021 }
1022
1023 sub create_conf_nolock {
1024     my ($vmid, $settings) = @_;
1025
1026     my $filename = config_file($vmid);
1027
1028     die "configuration file '$filename' already exists\n" if -f $filename;
1029
1030     my $defaults = load_defaults();
1031
1032     $settings->{name} = "vm$vmid" if !$settings->{name};
1033     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
1034
1035     my $data = '';
1036     foreach my $opt (keys %$settings) {
1037         next if !$confdesc->{$opt};
1038
1039         my $value = $settings->{$opt};
1040         next if !$value;
1041
1042         $data .= "$opt: $value\n";
1043     }
1044
1045     PVE::Tools::file_set_contents($filename, $data);
1046 }
1047
1048 sub parse_hotplug_features {
1049     my ($data) = @_;
1050
1051     my $res = {};
1052
1053     return $res if $data eq '0';
1054
1055     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1056
1057     foreach my $feature (PVE::Tools::split_list($data)) {
1058         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1059             $res->{$1} = 1;
1060         } else {
1061             warn "ignoring unknown hotplug feature '$feature'\n";
1062         }
1063     }
1064     return $res;
1065 }
1066
1067 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1068 sub pve_verify_hotplug_features {
1069     my ($value, $noerr) = @_;
1070
1071     return $value if parse_hotplug_features($value);
1072
1073     return undef if $noerr;
1074
1075     die "unable to parse hotplug option\n";
1076 }
1077
1078 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1079 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1080 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1081 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1082 #        [,iothread=on][,serial=serial][,model=model]
1083
1084 sub parse_drive {
1085     my ($key, $data) = @_;
1086
1087     my ($interface, $index);
1088
1089     if ($key =~ m/^([^\d]+)(\d+)$/) {
1090         $interface = $1;
1091         $index = $2;
1092     } else {
1093         return undef;
1094     }
1095
1096     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1097                                      : $confdesc->{$key}->{format};
1098     if (!$desc) {
1099         warn "invalid drive key: $key\n";
1100         return undef;
1101     }
1102     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1103     return undef if !$res;
1104     $res->{interface} = $interface;
1105     $res->{index} = $index;
1106
1107     my $error = 0;
1108     foreach my $opt (qw(bps bps_rd bps_wr)) {
1109         if (my $bps = defined(delete $res->{$opt})) {
1110             if (defined($res->{"m$opt"})) {
1111                 warn "both $opt and m$opt specified\n";
1112                 ++$error;
1113                 next;
1114             }
1115             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1116         }
1117     }
1118     return undef if $error;
1119
1120     return undef if $res->{mbps_rd} && $res->{mbps};
1121     return undef if $res->{mbps_wr} && $res->{mbps};
1122     return undef if $res->{iops_rd} && $res->{iops};
1123     return undef if $res->{iops_wr} && $res->{iops};
1124
1125     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1126         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1127         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1128         return undef if $res->{interface} eq 'virtio';
1129     }
1130
1131     if (my $size = $res->{size}) {
1132         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1133     }
1134
1135     return $res;
1136 }
1137
1138 sub print_drive {
1139     my ($vmid, $drive) = @_;
1140     my $data = { %$drive };
1141     delete $data->{$_} for qw(index interface);
1142     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1143 }
1144
1145 sub scsi_inquiry {
1146     my($fh, $noerr) = @_;
1147
1148     my $SG_IO = 0x2285;
1149     my $SG_GET_VERSION_NUM = 0x2282;
1150
1151     my $versionbuf = "\x00" x 8;
1152     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1153     if (!$ret) {
1154         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1155         return undef;
1156     }
1157     my $version = unpack("I", $versionbuf);
1158     if ($version < 30000) {
1159         die "scsi generic interface too old\n"  if !$noerr;
1160         return undef;
1161     }
1162
1163     my $buf = "\x00" x 36;
1164     my $sensebuf = "\x00" x 8;
1165     my $cmd = pack("C x3 C x1", 0x12, 36);
1166
1167     # see /usr/include/scsi/sg.h
1168     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1169
1170     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1171                       length($sensebuf), 0, length($buf), $buf,
1172                       $cmd, $sensebuf, 6000);
1173
1174     $ret = ioctl($fh, $SG_IO, $packet);
1175     if (!$ret) {
1176         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1177         return undef;
1178     }
1179
1180     my @res = unpack($sg_io_hdr_t, $packet);
1181     if ($res[17] || $res[18]) {
1182         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1183         return undef;
1184     }
1185
1186     my $res = {};
1187     (my $byte0, my $byte1, $res->{vendor},
1188      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1189
1190     $res->{removable} = $byte1 & 128 ? 1 : 0;
1191     $res->{type} = $byte0 & 31;
1192
1193     return $res;
1194 }
1195
1196 sub path_is_scsi {
1197     my ($path) = @_;
1198
1199     my $fh = IO::File->new("+<$path") || return undef;
1200     my $res = scsi_inquiry($fh, 1);
1201     close($fh);
1202
1203     return $res;
1204 }
1205
1206 sub machine_type_is_q35 {
1207     my ($conf) = @_;
1208
1209     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1210 }
1211
1212 sub print_tabletdevice_full {
1213     my ($conf) = @_;
1214
1215     my $q35 = machine_type_is_q35($conf);
1216
1217     # we use uhci for old VMs because tablet driver was buggy in older qemu
1218     my $usbbus = $q35 ? "ehci" : "uhci";
1219
1220     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1221 }
1222
1223 sub print_drivedevice_full {
1224     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1225
1226     my $device = '';
1227     my $maxdev = 0;
1228
1229     if ($drive->{interface} eq 'virtio') {
1230         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1231         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1232         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1233     } elsif ($drive->{interface} eq 'scsi') {
1234
1235         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1236         my $unit = $drive->{index} % $maxdev;
1237         my $devicetype = 'hd';
1238         my $path = '';
1239         if (drive_is_cdrom($drive)) {
1240               $devicetype = 'cd';
1241         } else {
1242               if ($drive->{file} =~ m|^/|) {
1243                   $path = $drive->{file};
1244                   if (my $info = path_is_scsi($path)) {
1245                       if ($info->{type} == 0) {
1246                           $devicetype = 'block';
1247                       } elsif ($info->{type} == 1) { # tape
1248                           $devicetype = 'generic';
1249                       }
1250                   }
1251               } else {
1252                   $path = PVE::Storage::path($storecfg, $drive->{file});
1253               }
1254
1255               if($path =~ m/^iscsi\:\/\//){
1256                   $devicetype = 'generic';
1257               }
1258          }
1259
1260         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1261             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1262         } else {
1263             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1264         }
1265
1266     } elsif ($drive->{interface} eq 'ide'){
1267         $maxdev = 2;
1268         my $controller = int($drive->{index} / $maxdev);
1269         my $unit = $drive->{index} % $maxdev;
1270         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1271
1272         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1273         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1274             $device .= ",model=$model";
1275         }
1276     } elsif ($drive->{interface} eq 'sata'){
1277         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1278         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1279         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1280     } elsif ($drive->{interface} eq 'usb') {
1281         die "implement me";
1282         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1283     } else {
1284         die "unsupported interface type";
1285     }
1286
1287     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1288
1289     return $device;
1290 }
1291
1292 sub get_initiator_name {
1293     my $initiator;
1294
1295     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1296     while (defined(my $line = <$fh>)) {
1297         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1298         $initiator = $1;
1299         last;
1300     }
1301     $fh->close();
1302
1303     return $initiator;
1304 }
1305
1306 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1307 sub print_drive_full {
1308     my ($storecfg, $vmid, $drive) = @_;
1309
1310     my $path;
1311     my $volid = $drive->{file};
1312     my $format;
1313
1314     if (drive_is_cdrom($drive)) {
1315         $path = get_iso_path($storecfg, $vmid, $volid);
1316     } else {
1317         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1318         if ($storeid) {
1319             $path = PVE::Storage::path($storecfg, $volid);
1320             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1321             $format = qemu_img_format($scfg, $volname);
1322         } else {
1323             $path = $volid;
1324             $format = "raw";
1325         }
1326    }
1327
1328     my $opts = '';
1329     foreach my $o (@qemu_drive_options) {
1330         next if $o eq 'bootindex';
1331         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1332     }
1333
1334     $opts .= ",format=$format" if $format && !$drive->{format};
1335
1336     foreach my $o (qw(bps bps_rd bps_wr)) {
1337         my $v = $drive->{"m$o"};
1338         $opts .= ",$o=" . int($v*1024*1024) if $v;
1339     }
1340
1341     my $cache_direct = 0;
1342
1343     if (my $cache = $drive->{cache}) {
1344         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1345     } elsif (!drive_is_cdrom($drive)) {
1346         $opts .= ",cache=none";
1347         $cache_direct = 1;
1348     }
1349
1350     # aio native works only with O_DIRECT
1351     if (!$drive->{aio}) {
1352         if($cache_direct) {
1353             $opts .= ",aio=native";
1354         } else {
1355             $opts .= ",aio=threads";
1356         }
1357     }
1358
1359     if (!drive_is_cdrom($drive)) {
1360         my $detectzeroes;
1361         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1362             $detectzeroes = 'off';
1363         } elsif ($drive->{discard}) {
1364             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1365         } else {
1366             # This used to be our default with discard not being specified:
1367             $detectzeroes = 'on';
1368         }
1369         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1370     }
1371
1372     my $pathinfo = $path ? "file=$path," : '';
1373
1374     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1375 }
1376
1377 sub print_netdevice_full {
1378     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1379
1380     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1381
1382     my $device = $net->{model};
1383     if ($net->{model} eq 'virtio') {
1384          $device = 'virtio-net-pci';
1385      };
1386
1387     my $pciaddr = print_pci_addr("$netid", $bridges);
1388     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1389     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1390         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1391         my $vectors = $net->{queues} * 2 + 2;
1392         $tmpstr .= ",vectors=$vectors,mq=on";
1393     }
1394     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1395
1396     if ($use_old_bios_files) {
1397         my $romfile;
1398         if ($device eq 'virtio-net-pci') {
1399             $romfile = 'pxe-virtio.rom';
1400         } elsif ($device eq 'e1000') {
1401             $romfile = 'pxe-e1000.rom';
1402         } elsif ($device eq 'ne2k') {
1403             $romfile = 'pxe-ne2k_pci.rom';
1404         } elsif ($device eq 'pcnet') {
1405             $romfile = 'pxe-pcnet.rom';
1406         } elsif ($device eq 'rtl8139') {
1407             $romfile = 'pxe-rtl8139.rom';
1408         }
1409         $tmpstr .= ",romfile=$romfile" if $romfile;
1410     }
1411
1412     return $tmpstr;
1413 }
1414
1415 sub print_netdev_full {
1416     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1417
1418     my $i = '';
1419     if ($netid =~ m/^net(\d+)$/) {
1420         $i = int($1);
1421     }
1422
1423     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1424
1425     my $ifname = "tap${vmid}i$i";
1426
1427     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1428     die "interface name '$ifname' is too long (max 15 character)\n"
1429         if length($ifname) >= 16;
1430
1431     my $vhostparam = '';
1432     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1433
1434     my $vmname = $conf->{name} || "vm$vmid";
1435
1436     my $netdev = "";
1437     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1438
1439     if ($net->{bridge}) {
1440         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1441     } else {
1442         $netdev = "type=user,id=$netid,hostname=$vmname";
1443     }
1444
1445     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1446
1447     return $netdev;
1448 }
1449
1450 sub drive_is_cdrom {
1451     my ($drive) = @_;
1452
1453     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1454
1455 }
1456
1457 sub parse_numa {
1458     my ($data) = @_;
1459
1460     my $res = {};
1461
1462     foreach my $kvp (split(/,/, $data)) {
1463
1464         if ($kvp =~ m/^memory=(\S+)$/) {
1465             $res->{memory} = $1;
1466         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1467             $res->{policy} = $1;
1468         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1469             $res->{cpus}->{start} = $1;
1470             $res->{cpus}->{end} = $3;
1471         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1472             $res->{hostnodes}->{start} = $1;
1473             $res->{hostnodes}->{end} = $3;
1474         } else {
1475             return undef;
1476         }
1477     }
1478
1479     return $res;
1480 }
1481
1482 sub parse_hostpci {
1483     my ($value) = @_;
1484
1485     return undef if !$value;
1486
1487
1488     my @list = split(/,/, $value);
1489     my $found;
1490
1491     my $res = {};
1492     foreach my $kv (@list) {
1493
1494         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1495             $found = 1;
1496             if(defined($4)){
1497                 push @{$res->{pciid}}, { id => $2 , function => $4};
1498
1499             }else{
1500                 my $pcidevices = lspci($2);
1501                 $res->{pciid} = $pcidevices->{$2};
1502             }
1503         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1504             $res->{rombar} = $1;
1505         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1506             $res->{'x-vga'} = $1;
1507         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1508             $res->{pcie} = 1 if $1 == 1;
1509         } else {
1510             warn "unknown hostpci setting '$kv'\n";
1511         }
1512     }
1513
1514     return undef if !$found;
1515
1516     return $res;
1517 }
1518
1519 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1520 sub parse_net {
1521     my ($data) = @_;
1522
1523     my $res = {};
1524
1525     foreach my $kvp (split(/,/, $data)) {
1526
1527         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1528             my $model = lc($1);
1529             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1530             $res->{model} = $model;
1531             $res->{macaddr} = $mac;
1532         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1533             $res->{bridge} = $1;
1534         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1535             $res->{queues} = $1;
1536         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1537             $res->{rate} = $1;
1538         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1539             $res->{tag} = $1;
1540         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1541             $res->{trunks} = $1;
1542         } elsif ($kvp =~ m/^firewall=([01])$/) {
1543             $res->{firewall} = $1;
1544         } elsif ($kvp =~ m/^link_down=([01])$/) {
1545             $res->{link_down} = $1;
1546         } else {
1547             return undef;
1548         }
1549
1550     }
1551
1552     return undef if !$res->{model};
1553
1554     return $res;
1555 }
1556
1557 sub print_net {
1558     my $net = shift;
1559
1560     my $res = "$net->{model}";
1561     $res .= "=$net->{macaddr}" if $net->{macaddr};
1562     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1563     $res .= ",rate=$net->{rate}" if $net->{rate};
1564     $res .= ",tag=$net->{tag}" if $net->{tag};
1565     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1566     $res .= ",firewall=1" if $net->{firewall};
1567     $res .= ",link_down=1" if $net->{link_down};
1568     $res .= ",queues=$net->{queues}" if $net->{queues};
1569
1570     return $res;
1571 }
1572
1573 sub add_random_macs {
1574     my ($settings) = @_;
1575
1576     foreach my $opt (keys %$settings) {
1577         next if $opt !~ m/^net(\d+)$/;
1578         my $net = parse_net($settings->{$opt});
1579         next if !$net;
1580         $settings->{$opt} = print_net($net);
1581     }
1582 }
1583
1584 sub add_unused_volume {
1585     my ($config, $volid) = @_;
1586
1587     my $key;
1588     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1589         my $test = "unused$ind";
1590         if (my $vid = $config->{$test}) {
1591             return if $vid eq $volid; # do not add duplicates
1592         } else {
1593             $key = $test;
1594         }
1595     }
1596
1597     die "To many unused volume - please delete them first.\n" if !$key;
1598
1599     $config->{$key} = $volid;
1600
1601     return $key;
1602 }
1603
1604 sub vm_is_volid_owner {
1605     my ($storecfg, $vmid, $volid) = @_;
1606
1607     if ($volid !~  m|^/|) {
1608         my ($path, $owner);
1609         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1610         if ($owner && ($owner == $vmid)) {
1611             return 1;
1612         }
1613     }
1614
1615     return undef;
1616 }
1617
1618 sub split_flagged_list {
1619     my $text = shift || '';
1620     $text =~ s/[,;]/ /g;
1621     $text =~ s/^\s+//;
1622     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1623 }
1624
1625 sub join_flagged_list {
1626     my ($how, $lst) = @_;
1627     join $how, map { $lst->{$_} . $_ } keys %$lst;
1628 }
1629
1630 sub vmconfig_delete_pending_option {
1631     my ($conf, $key, $force) = @_;
1632
1633     delete $conf->{pending}->{$key};
1634     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1635     $pending_delete_hash->{$key} = $force ? '!' : '';
1636     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1637 }
1638
1639 sub vmconfig_undelete_pending_option {
1640     my ($conf, $key) = @_;
1641
1642     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1643     delete $pending_delete_hash->{$key};
1644
1645     if (%$pending_delete_hash) {
1646         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1647     } else {
1648         delete $conf->{pending}->{delete};
1649     }
1650 }
1651
1652 sub vmconfig_register_unused_drive {
1653     my ($storecfg, $vmid, $conf, $drive) = @_;
1654
1655     if (!drive_is_cdrom($drive)) {
1656         my $volid = $drive->{file};
1657         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1658             add_unused_volume($conf, $volid, $vmid);
1659         }
1660     }
1661 }
1662
1663 sub vmconfig_cleanup_pending {
1664     my ($conf) = @_;
1665
1666     # remove pending changes when nothing changed
1667     my $changes;
1668     foreach my $opt (keys %{$conf->{pending}}) {
1669         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1670             $changes = 1;
1671             delete $conf->{pending}->{$opt};
1672         }
1673     }
1674
1675     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1676     my $pending_delete_hash = {};
1677     while (my ($opt, $force) = each %$current_delete_hash) {
1678         if (defined($conf->{$opt})) {
1679             $pending_delete_hash->{$opt} = $force;
1680         } else {
1681             $changes = 1;
1682         }
1683     }
1684
1685     if (%$pending_delete_hash) {
1686         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1687     } else {
1688         delete $conf->{pending}->{delete};
1689     }
1690
1691     return $changes;
1692 }
1693
1694 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1695 my $smbios1_desc = {
1696     uuid => {
1697         type => 'string',
1698         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1699         format_description => 'UUID',
1700         optional => 1,
1701     },
1702     version => {
1703         type => 'string',
1704         pattern => '\S+',
1705         format_description => 'str',
1706         optional => 1,
1707     },
1708     serial => {
1709         type => 'string',
1710         pattern => '\S+',
1711         format_description => 'str',
1712         optional => 1,
1713     },
1714     manufacturer => {
1715         type => 'string',
1716         pattern => '\S+',
1717         format_description => 'name',
1718         optional => 1,
1719     },
1720     product => {
1721         type => 'string',
1722         pattern => '\S+',
1723         format_description => 'name',
1724         optional => 1,
1725     },
1726     sku => {
1727         type => 'string',
1728         pattern => '\S+',
1729         format_description => 'str',
1730         optional => 1,
1731     },
1732     family => {
1733         type => 'string',
1734         pattern => '\S+',
1735         format_description => 'str',
1736         optional => 1,
1737     },
1738 };
1739
1740 sub parse_smbios1 {
1741     my ($data) = @_;
1742
1743     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1744     warn $@ if $@;
1745     return $res;
1746 }
1747
1748 sub print_smbios1 {
1749     my ($smbios1) = @_;
1750     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1751 }
1752
1753 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1754
1755 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1756 sub verify_bootdisk {
1757     my ($value, $noerr) = @_;
1758
1759     return $value if valid_drivename($value);
1760
1761     return undef if $noerr;
1762
1763     die "invalid boot disk '$value'\n";
1764 }
1765
1766 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1767 sub verify_numa {
1768     my ($value, $noerr) = @_;
1769
1770     return $value if parse_numa($value);
1771
1772     return undef if $noerr;
1773
1774     die "unable to parse numa options\n";
1775 }
1776
1777 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1778 sub verify_net {
1779     my ($value, $noerr) = @_;
1780
1781     return $value if parse_net($value);
1782
1783     return undef if $noerr;
1784
1785     die "unable to parse network options\n";
1786 }
1787
1788 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1789 sub verify_hostpci {
1790     my ($value, $noerr) = @_;
1791
1792     return $value if parse_hostpci($value);
1793
1794     return undef if $noerr;
1795
1796     die "unable to parse pci id\n";
1797 }
1798
1799 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1800 sub verify_watchdog {
1801     my ($value, $noerr) = @_;
1802
1803     return $value if parse_watchdog($value);
1804
1805     return undef if $noerr;
1806
1807     die "unable to parse watchdog options\n";
1808 }
1809
1810 sub parse_watchdog {
1811     my ($value) = @_;
1812
1813     return undef if !$value;
1814
1815     my $res = {};
1816
1817     foreach my $p (split(/,/, $value)) {
1818         next if $p =~ m/^\s*$/;
1819
1820         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1821             $res->{model} = $2;
1822         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1823             $res->{action} = $2;
1824         } else {
1825             return undef;
1826         }
1827     }
1828
1829     return $res;
1830 }
1831
1832 sub parse_usb_device {
1833     my ($value) = @_;
1834
1835     return undef if !$value;
1836
1837     my $res = {};
1838     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1839         $res->{vendorid} = $2;
1840         $res->{productid} = $4;
1841     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1842         $res->{hostbus} = $1;
1843         $res->{hostport} = $2;
1844     } elsif ($value =~ m/^spice$/i) {
1845         $res->{spice} = 1;
1846     } else {
1847         return undef;
1848     }
1849
1850     return $res;
1851 }
1852
1853 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1854 sub verify_usb_device {
1855     my ($value, $noerr) = @_;
1856
1857     return $value if parse_usb_device($value);
1858
1859     return undef if $noerr;
1860
1861     die "unable to parse usb device\n";
1862 }
1863
1864 # add JSON properties for create and set function
1865 sub json_config_properties {
1866     my $prop = shift;
1867
1868     foreach my $opt (keys %$confdesc) {
1869         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1870         $prop->{$opt} = $confdesc->{$opt};
1871     }
1872
1873     return $prop;
1874 }
1875
1876 sub check_type {
1877     my ($key, $value) = @_;
1878
1879     die "unknown setting '$key'\n" if !$confdesc->{$key};
1880
1881     my $type = $confdesc->{$key}->{type};
1882
1883     if (!defined($value)) {
1884         die "got undefined value\n";
1885     }
1886
1887     if ($value =~ m/[\n\r]/) {
1888         die "property contains a line feed\n";
1889     }
1890
1891     if ($type eq 'boolean') {
1892         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1893         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1894         die "type check ('boolean') failed - got '$value'\n";
1895     } elsif ($type eq 'integer') {
1896         return int($1) if $value =~ m/^(\d+)$/;
1897         die "type check ('integer') failed - got '$value'\n";
1898     } elsif ($type eq 'number') {
1899         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1900         die "type check ('number') failed - got '$value'\n";
1901     } elsif ($type eq 'string') {
1902         if (my $fmt = $confdesc->{$key}->{format}) {
1903             if ($fmt eq 'pve-qm-drive') {
1904                 # special case - we need to pass $key to parse_drive()
1905                 my $drive = parse_drive($key, $value);
1906                 return $value if $drive;
1907                 die "unable to parse drive options\n";
1908             }
1909             PVE::JSONSchema::check_format($fmt, $value);
1910             return $value;
1911         }
1912         $value =~ s/^\"(.*)\"$/$1/;
1913         return $value;
1914     } else {
1915         die "internal error"
1916     }
1917 }
1918
1919 sub lock_config_full {
1920     my ($vmid, $timeout, $code, @param) = @_;
1921
1922     my $filename = config_file_lock($vmid);
1923
1924     my $res = lock_file($filename, $timeout, $code, @param);
1925
1926     die $@ if $@;
1927
1928     return $res;
1929 }
1930
1931 sub lock_config_mode {
1932     my ($vmid, $timeout, $shared, $code, @param) = @_;
1933
1934     my $filename = config_file_lock($vmid);
1935
1936     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1937
1938     die $@ if $@;
1939
1940     return $res;
1941 }
1942
1943 sub lock_config {
1944     my ($vmid, $code, @param) = @_;
1945
1946     return lock_config_full($vmid, 10, $code, @param);
1947 }
1948
1949 sub cfs_config_path {
1950     my ($vmid, $node) = @_;
1951
1952     $node = $nodename if !$node;
1953     return "nodes/$node/qemu-server/$vmid.conf";
1954 }
1955
1956 sub check_iommu_support{
1957     #fixme : need to check IOMMU support
1958     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1959
1960     my $iommu=1;
1961     return $iommu;
1962
1963 }
1964
1965 sub config_file {
1966     my ($vmid, $node) = @_;
1967
1968     my $cfspath = cfs_config_path($vmid, $node);
1969     return "/etc/pve/$cfspath";
1970 }
1971
1972 sub config_file_lock {
1973     my ($vmid) = @_;
1974
1975     return "$lock_dir/lock-$vmid.conf";
1976 }
1977
1978 sub touch_config {
1979     my ($vmid) = @_;
1980
1981     my $conf = config_file($vmid);
1982     utime undef, undef, $conf;
1983 }
1984
1985 sub destroy_vm {
1986     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1987
1988     my $conffile = config_file($vmid);
1989
1990     my $conf = load_config($vmid);
1991
1992     check_lock($conf) if !$skiplock;
1993
1994     # only remove disks owned by this VM
1995     foreach_drive($conf, sub {
1996         my ($ds, $drive) = @_;
1997
1998         return if drive_is_cdrom($drive);
1999
2000         my $volid = $drive->{file};
2001
2002         return if !$volid || $volid =~ m|^/|;
2003
2004         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2005         return if !$path || !$owner || ($owner != $vmid);
2006
2007         PVE::Storage::vdisk_free($storecfg, $volid);
2008     });
2009
2010     if ($keep_empty_config) {
2011         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2012     } else {
2013         unlink $conffile;
2014     }
2015
2016     # also remove unused disk
2017     eval {
2018         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2019
2020         eval {
2021             PVE::Storage::foreach_volid($dl, sub {
2022                 my ($volid, $sid, $volname, $d) = @_;
2023                 PVE::Storage::vdisk_free($storecfg, $volid);
2024             });
2025         };
2026         warn $@ if $@;
2027
2028     };
2029     warn $@ if $@;
2030 }
2031
2032 sub load_config {
2033     my ($vmid, $node) = @_;
2034
2035     my $cfspath = cfs_config_path($vmid, $node);
2036
2037     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2038
2039     die "no such VM ('$vmid')\n" if !defined($conf);
2040
2041     return $conf;
2042 }
2043
2044 sub parse_vm_config {
2045     my ($filename, $raw) = @_;
2046
2047     return undef if !defined($raw);
2048
2049     my $res = {
2050         digest => Digest::SHA::sha1_hex($raw),
2051         snapshots => {},
2052         pending => {},
2053     };
2054
2055     $filename =~ m|/qemu-server/(\d+)\.conf$|
2056         || die "got strange filename '$filename'";
2057
2058     my $vmid = $1;
2059
2060     my $conf = $res;
2061     my $descr;
2062     my $section = '';
2063
2064     my @lines = split(/\n/, $raw);
2065     foreach my $line (@lines) {
2066         next if $line =~ m/^\s*$/;
2067
2068         if ($line =~ m/^\[PENDING\]\s*$/i) {
2069             $section = 'pending';
2070             if (defined($descr)) {
2071                 $descr =~ s/\s+$//;
2072                 $conf->{description} = $descr;
2073             }
2074             $descr = undef;
2075             $conf = $res->{$section} = {};
2076             next;
2077
2078         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2079             $section = $1;
2080             if (defined($descr)) {
2081                 $descr =~ s/\s+$//;
2082                 $conf->{description} = $descr;
2083             }
2084             $descr = undef;
2085             $conf = $res->{snapshots}->{$section} = {};
2086             next;
2087         }
2088
2089         if ($line =~ m/^\#(.*)\s*$/) {
2090             $descr = '' if !defined($descr);
2091             $descr .= PVE::Tools::decode_text($1) . "\n";
2092             next;
2093         }
2094
2095         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2096             $descr = '' if !defined($descr);
2097             $descr .= PVE::Tools::decode_text($2);
2098         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2099             $conf->{snapstate} = $1;
2100         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2101             my $key = $1;
2102             my $value = $2;
2103             $conf->{$key} = $value;
2104         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2105             my $value = $1;
2106             if ($section eq 'pending') {
2107                 $conf->{delete} = $value; # we parse this later
2108             } else {
2109                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2110             }
2111         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2112             my $key = $1;
2113             my $value = $2;
2114             eval { $value = check_type($key, $value); };
2115             if ($@) {
2116                 warn "vm $vmid - unable to parse value of '$key' - $@";
2117             } else {
2118                 my $fmt = $confdesc->{$key}->{format};
2119                 if ($fmt && $fmt eq 'pve-qm-drive') {
2120                     my $v = parse_drive($key, $value);
2121                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2122                         $v->{file} = $volid;
2123                         $value = print_drive($vmid, $v);
2124                     } else {
2125                         warn "vm $vmid - unable to parse value of '$key'\n";
2126                         next;
2127                     }
2128                 }
2129
2130                 if ($key eq 'cdrom') {
2131                     $conf->{ide2} = $value;
2132                 } else {
2133                     $conf->{$key} = $value;
2134                 }
2135             }
2136         }
2137     }
2138
2139     if (defined($descr)) {
2140         $descr =~ s/\s+$//;
2141         $conf->{description} = $descr;
2142     }
2143     delete $res->{snapstate}; # just to be sure
2144
2145     return $res;
2146 }
2147
2148 sub write_vm_config {
2149     my ($filename, $conf) = @_;
2150
2151     delete $conf->{snapstate}; # just to be sure
2152
2153     if ($conf->{cdrom}) {
2154         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2155         $conf->{ide2} = $conf->{cdrom};
2156         delete $conf->{cdrom};
2157     }
2158
2159     # we do not use 'smp' any longer
2160     if ($conf->{sockets}) {
2161         delete $conf->{smp};
2162     } elsif ($conf->{smp}) {
2163         $conf->{sockets} = $conf->{smp};
2164         delete $conf->{cores};
2165         delete $conf->{smp};
2166     }
2167
2168     my $used_volids = {};
2169
2170     my $cleanup_config = sub {
2171         my ($cref, $pending, $snapname) = @_;
2172
2173         foreach my $key (keys %$cref) {
2174             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2175                 $key eq 'snapstate' || $key eq 'pending';
2176             my $value = $cref->{$key};
2177             if ($key eq 'delete') {
2178                 die "propertry 'delete' is only allowed in [PENDING]\n"
2179                     if !$pending;
2180                 # fixme: check syntax?
2181                 next;
2182             }
2183             eval { $value = check_type($key, $value); };
2184             die "unable to parse value of '$key' - $@" if $@;
2185
2186             $cref->{$key} = $value;
2187
2188             if (!$snapname && valid_drivename($key)) {
2189                 my $drive = parse_drive($key, $value);
2190                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2191             }
2192         }
2193     };
2194
2195     &$cleanup_config($conf);
2196
2197     &$cleanup_config($conf->{pending}, 1);
2198
2199     foreach my $snapname (keys %{$conf->{snapshots}}) {
2200         die "internal error" if $snapname eq 'pending';
2201         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2202     }
2203
2204     # remove 'unusedX' settings if we re-add a volume
2205     foreach my $key (keys %$conf) {
2206         my $value = $conf->{$key};
2207         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2208             delete $conf->{$key};
2209         }
2210     }
2211
2212     my $generate_raw_config = sub {
2213         my ($conf, $pending) = @_;
2214
2215         my $raw = '';
2216
2217         # add description as comment to top of file
2218         if (defined(my $descr = $conf->{description})) {
2219             if ($descr) {
2220                 foreach my $cl (split(/\n/, $descr)) {
2221                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2222                 }
2223             } else {
2224                 $raw .= "#\n" if $pending;
2225             }
2226         }
2227
2228         foreach my $key (sort keys %$conf) {
2229             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2230             $raw .= "$key: $conf->{$key}\n";
2231         }
2232         return $raw;
2233     };
2234
2235     my $raw = &$generate_raw_config($conf);
2236
2237     if (scalar(keys %{$conf->{pending}})){
2238         $raw .= "\n[PENDING]\n";
2239         $raw .= &$generate_raw_config($conf->{pending}, 1);
2240     }
2241
2242     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2243         $raw .= "\n[$snapname]\n";
2244         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2245     }
2246
2247     return $raw;
2248 }
2249
2250 sub write_config {
2251     my ($vmid, $conf) = @_;
2252
2253     my $cfspath = cfs_config_path($vmid);
2254
2255     PVE::Cluster::cfs_write_file($cfspath, $conf);
2256 }
2257
2258 sub load_defaults {
2259
2260     my $res = {};
2261
2262     # we use static defaults from our JSON schema configuration
2263     foreach my $key (keys %$confdesc) {
2264         if (defined(my $default = $confdesc->{$key}->{default})) {
2265             $res->{$key} = $default;
2266         }
2267     }
2268
2269     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2270     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2271
2272     return $res;
2273 }
2274
2275 sub config_list {
2276     my $vmlist = PVE::Cluster::get_vmlist();
2277     my $res = {};
2278     return $res if !$vmlist || !$vmlist->{ids};
2279     my $ids = $vmlist->{ids};
2280
2281     foreach my $vmid (keys %$ids) {
2282         my $d = $ids->{$vmid};
2283         next if !$d->{node} || $d->{node} ne $nodename;
2284         next if !$d->{type} || $d->{type} ne 'qemu';
2285         $res->{$vmid}->{exists} = 1;
2286     }
2287     return $res;
2288 }
2289
2290 # test if VM uses local resources (to prevent migration)
2291 sub check_local_resources {
2292     my ($conf, $noerr) = @_;
2293
2294     my $loc_res = 0;
2295
2296     $loc_res = 1 if $conf->{hostusb}; # old syntax
2297     $loc_res = 1 if $conf->{hostpci}; # old syntax
2298
2299     foreach my $k (keys %$conf) {
2300         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2301         # sockets are safe: they will recreated be on the target side post-migrate
2302         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2303         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2304     }
2305
2306     die "VM uses local resources\n" if $loc_res && !$noerr;
2307
2308     return $loc_res;
2309 }
2310
2311 # check if used storages are available on all nodes (use by migrate)
2312 sub check_storage_availability {
2313     my ($storecfg, $conf, $node) = @_;
2314
2315     foreach_drive($conf, sub {
2316         my ($ds, $drive) = @_;
2317
2318         my $volid = $drive->{file};
2319         return if !$volid;
2320
2321         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2322         return if !$sid;
2323
2324         # check if storage is available on both nodes
2325         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2326         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2327    });
2328 }
2329
2330 # list nodes where all VM images are available (used by has_feature API)
2331 sub shared_nodes {
2332     my ($conf, $storecfg) = @_;
2333
2334     my $nodelist = PVE::Cluster::get_nodelist();
2335     my $nodehash = { map { $_ => 1 } @$nodelist };
2336     my $nodename = PVE::INotify::nodename();
2337
2338     foreach_drive($conf, sub {
2339         my ($ds, $drive) = @_;
2340
2341         my $volid = $drive->{file};
2342         return if !$volid;
2343
2344         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2345         if ($storeid) {
2346             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2347             if ($scfg->{disable}) {
2348                 $nodehash = {};
2349             } elsif (my $avail = $scfg->{nodes}) {
2350                 foreach my $node (keys %$nodehash) {
2351                     delete $nodehash->{$node} if !$avail->{$node};
2352                 }
2353             } elsif (!$scfg->{shared}) {
2354                 foreach my $node (keys %$nodehash) {
2355                     delete $nodehash->{$node} if $node ne $nodename
2356                 }
2357             }
2358         }
2359     });
2360
2361     return $nodehash
2362 }
2363
2364 sub check_lock {
2365     my ($conf) = @_;
2366
2367     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2368 }
2369
2370 sub check_cmdline {
2371     my ($pidfile, $pid) = @_;
2372
2373     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2374     if (defined($fh)) {
2375         my $line = <$fh>;
2376         $fh->close;
2377         return undef if !$line;
2378         my @param = split(/\0/, $line);
2379
2380         my $cmd = $param[0];
2381         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2382
2383         for (my $i = 0; $i < scalar (@param); $i++) {
2384             my $p = $param[$i];
2385             next if !$p;
2386             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2387                 my $p = $param[$i+1];
2388                 return 1 if $p && ($p eq $pidfile);
2389                 return undef;
2390             }
2391         }
2392     }
2393     return undef;
2394 }
2395
2396 sub check_running {
2397     my ($vmid, $nocheck, $node) = @_;
2398
2399     my $filename = config_file($vmid, $node);
2400
2401     die "unable to find configuration file for VM $vmid - no such machine\n"
2402         if !$nocheck && ! -f $filename;
2403
2404     my $pidfile = pidfile_name($vmid);
2405
2406     if (my $fd = IO::File->new("<$pidfile")) {
2407         my $st = stat($fd);
2408         my $line = <$fd>;
2409         close($fd);
2410
2411         my $mtime = $st->mtime;
2412         if ($mtime > time()) {
2413             warn "file '$filename' modified in future\n";
2414         }
2415
2416         if ($line =~ m/^(\d+)$/) {
2417             my $pid = $1;
2418             if (check_cmdline($pidfile, $pid)) {
2419                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2420                     return $pid;
2421                 }
2422             }
2423         }
2424     }
2425
2426     return undef;
2427 }
2428
2429 sub vzlist {
2430
2431     my $vzlist = config_list();
2432
2433     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2434
2435     while (defined(my $de = $fd->read)) {
2436         next if $de !~ m/^(\d+)\.pid$/;
2437         my $vmid = $1;
2438         next if !defined($vzlist->{$vmid});
2439         if (my $pid = check_running($vmid)) {
2440             $vzlist->{$vmid}->{pid} = $pid;
2441         }
2442     }
2443
2444     return $vzlist;
2445 }
2446
2447 sub disksize {
2448     my ($storecfg, $conf) = @_;
2449
2450     my $bootdisk = $conf->{bootdisk};
2451     return undef if !$bootdisk;
2452     return undef if !valid_drivename($bootdisk);
2453
2454     return undef if !$conf->{$bootdisk};
2455
2456     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2457     return undef if !defined($drive);
2458
2459     return undef if drive_is_cdrom($drive);
2460
2461     my $volid = $drive->{file};
2462     return undef if !$volid;
2463
2464     return $drive->{size};
2465 }
2466
2467 my $last_proc_pid_stat;
2468
2469 # get VM status information
2470 # This must be fast and should not block ($full == false)
2471 # We only query KVM using QMP if $full == true (this can be slow)
2472 sub vmstatus {
2473     my ($opt_vmid, $full) = @_;
2474
2475     my $res = {};
2476
2477     my $storecfg = PVE::Storage::config();
2478
2479     my $list = vzlist();
2480     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2481
2482     my $cpucount = $cpuinfo->{cpus} || 1;
2483
2484     foreach my $vmid (keys %$list) {
2485         next if $opt_vmid && ($vmid ne $opt_vmid);
2486
2487         my $cfspath = cfs_config_path($vmid);
2488         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2489
2490         my $d = {};
2491         $d->{pid} = $list->{$vmid}->{pid};
2492
2493         # fixme: better status?
2494         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2495
2496         my $size = disksize($storecfg, $conf);
2497         if (defined($size)) {
2498             $d->{disk} = 0; # no info available
2499             $d->{maxdisk} = $size;
2500         } else {
2501             $d->{disk} = 0;
2502             $d->{maxdisk} = 0;
2503         }
2504
2505         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2506         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2507         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2508
2509         $d->{name} = $conf->{name} || "VM $vmid";
2510         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2511
2512         if ($conf->{balloon}) {
2513             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2514             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2515         }
2516
2517         $d->{uptime} = 0;
2518         $d->{cpu} = 0;
2519         $d->{mem} = 0;
2520
2521         $d->{netout} = 0;
2522         $d->{netin} = 0;
2523
2524         $d->{diskread} = 0;
2525         $d->{diskwrite} = 0;
2526
2527         $d->{template} = is_template($conf);
2528
2529         $res->{$vmid} = $d;
2530     }
2531
2532     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2533     foreach my $dev (keys %$netdev) {
2534         next if $dev !~ m/^tap([1-9]\d*)i/;
2535         my $vmid = $1;
2536         my $d = $res->{$vmid};
2537         next if !$d;
2538
2539         $d->{netout} += $netdev->{$dev}->{receive};
2540         $d->{netin} += $netdev->{$dev}->{transmit};
2541
2542         if ($full) {
2543             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2544             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2545         }
2546
2547     }
2548
2549     my $ctime = gettimeofday;
2550
2551     foreach my $vmid (keys %$list) {
2552
2553         my $d = $res->{$vmid};
2554         my $pid = $d->{pid};
2555         next if !$pid;
2556
2557         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2558         next if !$pstat; # not running
2559
2560         my $used = $pstat->{utime} + $pstat->{stime};
2561
2562         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2563
2564         if ($pstat->{vsize}) {
2565             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2566         }
2567
2568         my $old = $last_proc_pid_stat->{$pid};
2569         if (!$old) {
2570             $last_proc_pid_stat->{$pid} = {
2571                 time => $ctime,
2572                 used => $used,
2573                 cpu => 0,
2574             };
2575             next;
2576         }
2577
2578         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2579
2580         if ($dtime > 1000) {
2581             my $dutime = $used -  $old->{used};
2582
2583             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2584             $last_proc_pid_stat->{$pid} = {
2585                 time => $ctime,
2586                 used => $used,
2587                 cpu => $d->{cpu},
2588             };
2589         } else {
2590             $d->{cpu} = $old->{cpu};
2591         }
2592     }
2593
2594     return $res if !$full;
2595
2596     my $qmpclient = PVE::QMPClient->new();
2597
2598     my $ballooncb = sub {
2599         my ($vmid, $resp) = @_;
2600
2601         my $info = $resp->{'return'};
2602         return if !$info->{max_mem};
2603
2604         my $d = $res->{$vmid};
2605
2606         # use memory assigned to VM
2607         $d->{maxmem} = $info->{max_mem};
2608         $d->{balloon} = $info->{actual};
2609
2610         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2611             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2612             $d->{freemem} = $info->{free_mem};
2613         }
2614
2615         $d->{ballooninfo} = $info;
2616     };
2617
2618     my $blockstatscb = sub {
2619         my ($vmid, $resp) = @_;
2620         my $data = $resp->{'return'} || [];
2621         my $totalrdbytes = 0;
2622         my $totalwrbytes = 0;
2623
2624         for my $blockstat (@$data) {
2625             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2626             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2627
2628             $blockstat->{device} =~ s/drive-//;
2629             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2630         }
2631         $res->{$vmid}->{diskread} = $totalrdbytes;
2632         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2633     };
2634
2635     my $statuscb = sub {
2636         my ($vmid, $resp) = @_;
2637
2638         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2639         # this fails if ballon driver is not loaded, so this must be
2640         # the last commnand (following command are aborted if this fails).
2641         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2642
2643         my $status = 'unknown';
2644         if (!defined($status = $resp->{'return'}->{status})) {
2645             warn "unable to get VM status\n";
2646             return;
2647         }
2648
2649         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2650     };
2651
2652     foreach my $vmid (keys %$list) {
2653         next if $opt_vmid && ($vmid ne $opt_vmid);
2654         next if !$res->{$vmid}->{pid}; # not running
2655         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2656     }
2657
2658     $qmpclient->queue_execute(undef, 1);
2659
2660     foreach my $vmid (keys %$list) {
2661         next if $opt_vmid && ($vmid ne $opt_vmid);
2662         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2663     }
2664
2665     return $res;
2666 }
2667
2668 sub foreach_dimm {
2669     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2670
2671     my $dimm_id = 0;
2672     my $current_size = 1024;
2673     my $dimm_size = 512;
2674     return if $current_size == $memory;
2675
2676     for (my $j = 0; $j < 8; $j++) {
2677         for (my $i = 0; $i < 32; $i++) {
2678             my $name = "dimm${dimm_id}";
2679             $dimm_id++;
2680             my $numanode = $i % $sockets;
2681             $current_size += $dimm_size;
2682             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2683             return  $current_size if $current_size >= $memory;
2684         }
2685         $dimm_size *= 2;
2686     }
2687 }
2688
2689 sub foreach_reverse_dimm {
2690     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2691
2692     my $dimm_id = 253;
2693     my $current_size = 4177920;
2694     my $dimm_size = 65536;
2695     return if $current_size == $memory;
2696
2697     for (my $j = 0; $j < 8; $j++) {
2698         for (my $i = 0; $i < 32; $i++) {
2699             my $name = "dimm${dimm_id}";
2700             $dimm_id--;
2701             my $numanode = $i % $sockets;
2702             $current_size -= $dimm_size;
2703             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2704             return  $current_size if $current_size <= $memory;
2705         }
2706         $dimm_size /= 2;
2707     }
2708 }
2709
2710 sub foreach_drive {
2711     my ($conf, $func) = @_;
2712
2713     foreach my $ds (keys %$conf) {
2714         next if !valid_drivename($ds);
2715
2716         my $drive = parse_drive($ds, $conf->{$ds});
2717         next if !$drive;
2718
2719         &$func($ds, $drive);
2720     }
2721 }
2722
2723 sub foreach_volid {
2724     my ($conf, $func) = @_;
2725
2726     my $volhash = {};
2727
2728     my $test_volid = sub {
2729         my ($volid, $is_cdrom) = @_;
2730
2731         return if !$volid;
2732
2733         $volhash->{$volid} = $is_cdrom || 0;
2734     };
2735
2736     foreach_drive($conf, sub {
2737         my ($ds, $drive) = @_;
2738         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2739     });
2740
2741     foreach my $snapname (keys %{$conf->{snapshots}}) {
2742         my $snap = $conf->{snapshots}->{$snapname};
2743         &$test_volid($snap->{vmstate}, 0);
2744         foreach_drive($snap, sub {
2745             my ($ds, $drive) = @_;
2746             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2747         });
2748     }
2749
2750     foreach my $volid (keys %$volhash) {
2751         &$func($volid, $volhash->{$volid});
2752     }
2753 }
2754
2755 sub vga_conf_has_spice {
2756     my ($vga) = @_;
2757
2758     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2759
2760     return $1 || 1;
2761 }
2762
2763 sub config_to_command {
2764     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2765
2766     my $cmd = [];
2767     my $globalFlags = [];
2768     my $machineFlags = [];
2769     my $rtcFlags = [];
2770     my $cpuFlags = [];
2771     my $devices = [];
2772     my $pciaddr = '';
2773     my $bridges = {};
2774     my $kvmver = kvm_user_version();
2775     my $vernum = 0; # unknown
2776     my $ostype = $conf->{ostype};
2777     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2778         $vernum = $1*1000000+$2*1000;
2779     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2780         $vernum = $1*1000000+$2*1000+$3;
2781     }
2782
2783     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2784
2785     my $have_ovz = -f '/proc/vz/vestat';
2786
2787     my $q35 = machine_type_is_q35($conf);
2788     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2789     my $machine_type = $forcemachine || $conf->{machine};
2790     my $use_old_bios_files = undef;
2791     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2792
2793     my $cpuunits = defined($conf->{cpuunits}) ?
2794             $conf->{cpuunits} : $defaults->{cpuunits};
2795
2796     push @$cmd, '/usr/bin/systemd-run';
2797     push @$cmd, '--scope';
2798     push @$cmd, '--slice', "qemu";
2799     push @$cmd, '--unit', $vmid;
2800     # set KillMode=none, so that systemd don't kill those scopes
2801     # at shutdown (pve-manager service should stop the VMs instead)
2802     push @$cmd, '-p', "KillMode=none";
2803     push @$cmd, '-p', "CPUShares=$cpuunits";
2804     if ($conf->{cpulimit}) {
2805         my $cpulimit = int($conf->{cpulimit} * 100);
2806         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2807     }
2808
2809     push @$cmd, '/usr/bin/kvm';
2810
2811     push @$cmd, '-id', $vmid;
2812
2813     my $use_virtio = 0;
2814
2815     my $qmpsocket = qmp_socket($vmid);
2816     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2817     push @$cmd, '-mon', "chardev=qmp,mode=control";
2818
2819
2820     push @$cmd, '-pidfile' , pidfile_name($vmid);
2821
2822     push @$cmd, '-daemonize';
2823
2824     if ($conf->{smbios1}) {
2825         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2826     }
2827
2828     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2829         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2830         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2831         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2832         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2833         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2834         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2835     }
2836
2837     if ($q35) {
2838         # the q35 chipset support native usb2, so we enable usb controller
2839         # by default for this machine type
2840         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2841     } else {
2842         $pciaddr = print_pci_addr("piix3", $bridges);
2843         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2844
2845         my $use_usb2 = 0;
2846         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2847             next if !$conf->{"usb$i"};
2848             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2849             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2850             $use_usb2 = 1;
2851         }
2852         # include usb device config
2853         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2854     }
2855
2856     # add usb3 controller if needed
2857
2858     my $use_usb3 = 0;
2859     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2860         next if !$conf->{"usb$i"};
2861         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2862         next if !$d || !$d->{usb3};
2863         $use_usb3 = 1;
2864     }
2865
2866     $pciaddr = print_pci_addr("xhci", $bridges);
2867     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2868
2869     my $vga = $conf->{vga};
2870
2871     my $qxlnum = vga_conf_has_spice($vga);
2872     $vga = 'qxl' if $qxlnum;
2873
2874     if (!$vga) {
2875         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2876                                 $conf->{ostype} eq 'win7' ||
2877                                 $conf->{ostype} eq 'w2k8')) {
2878             $vga = 'std';
2879         } else {
2880             $vga = 'cirrus';
2881         }
2882     }
2883
2884     # enable absolute mouse coordinates (needed by vnc)
2885     my $tablet;
2886     if (defined($conf->{tablet})) {
2887         $tablet = $conf->{tablet};
2888     } else {
2889         $tablet = $defaults->{tablet};
2890         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2891         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2892     }
2893
2894     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2895
2896     my $kvm_off = 0;
2897     # host pci devices
2898     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2899         my $d = parse_hostpci($conf->{"hostpci$i"});
2900         next if !$d;
2901
2902         my $pcie = $d->{pcie};
2903         if($pcie){
2904             die "q35 machine model is not enabled" if !$q35;
2905             $pciaddr = print_pcie_addr("hostpci$i");
2906         }else{
2907             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2908         }
2909
2910         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2911         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2912         if ($xvga && $xvga ne '') {
2913             $kvm_off = 1;
2914             $vga = 'none';
2915             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2916                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2917             }
2918             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2919                 $xvga = "";
2920             }
2921         }
2922         my $pcidevices = $d->{pciid};
2923         my $multifunction = 1 if @$pcidevices > 1;
2924
2925         my $j=0;
2926         foreach my $pcidevice (@$pcidevices) {
2927
2928             my $id = "hostpci$i";
2929             $id .= ".$j" if $multifunction;
2930             my $addr = $pciaddr;
2931             $addr .= ".$j" if $multifunction;
2932             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2933
2934             if($j == 0){
2935                 $devicestr .= "$rombar$xvga";
2936                 $devicestr .= ",multifunction=on" if $multifunction;
2937             }
2938
2939             push @$devices, '-device', $devicestr;
2940             $j++;
2941         }
2942     }
2943
2944     # usb devices
2945     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2946         next if !$conf->{"usb$i"};
2947         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2948         next if !$d;
2949
2950         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2951         my $usbbus = '';
2952         if (defined($d->{usb3}) && $d->{usb3}) {
2953             $usbbus = ',bus=xhci.0';
2954         }
2955
2956         if (defined($d->{host})) {
2957             $d = parse_usb_device($d->{host});
2958             if (defined($d->{vendorid}) && defined($d->{productid})) {
2959                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2960             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2961                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2962             } elsif (defined($d->{spice}) && $d->{spice}) {
2963                 # usb redir support for spice, currently no usb3
2964                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2965                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2966             }
2967         }
2968     }
2969
2970     # serial devices
2971     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2972         if (my $path = $conf->{"serial$i"}) {
2973             if ($path eq 'socket') {
2974                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2975                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2976                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2977             } else {
2978                 die "no such serial device\n" if ! -c $path;
2979                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2980                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2981             }
2982         }
2983     }
2984
2985     # parallel devices
2986     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2987         if (my $path = $conf->{"parallel$i"}) {
2988             die "no such parallel device\n" if ! -c $path;
2989             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2990             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2991             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2992         }
2993     }
2994
2995     my $vmname = $conf->{name} || "vm$vmid";
2996
2997     push @$cmd, '-name', $vmname;
2998
2999     my $sockets = 1;
3000     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3001     $sockets = $conf->{sockets} if  $conf->{sockets};
3002
3003     my $cores = $conf->{cores} || 1;
3004
3005     my $maxcpus = $sockets * $cores;
3006
3007     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3008
3009     my $allowed_vcpus = $cpuinfo->{cpus};
3010
3011     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3012         if ($allowed_vcpus < $maxcpus);
3013
3014     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3015
3016     push @$cmd, '-nodefaults';
3017
3018     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3019
3020     my $bootindex_hash = {};
3021     my $i = 1;
3022     foreach my $o (split(//, $bootorder)) {
3023         $bootindex_hash->{$o} = $i*100;
3024         $i++;
3025     }
3026
3027     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3028
3029     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3030
3031     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3032
3033     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3034
3035     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3036         my $socket = vnc_socket($vmid);
3037         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3038     } else {
3039         push @$cmd, '-nographic';
3040     }
3041
3042     # time drift fix
3043     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3044
3045     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3046     my $useLocaltime = $conf->{localtime};
3047
3048     if ($ostype) {
3049         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3050
3051         if ($ostype =~ m/^w/) { # windows
3052             $useLocaltime = 1 if !defined($conf->{localtime});
3053
3054             # use time drift fix when acpi is enabled
3055             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3056                 $tdf = 1 if !defined($conf->{tdf});
3057             }
3058         }
3059
3060         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3061             $ostype eq 'wvista') {
3062             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3063             push @$cmd, '-no-hpet';
3064             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3065                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3066                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3067                 push @$cpuFlags , 'hv_time' if !$nokvm;
3068
3069             } else {
3070                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3071             }
3072         }
3073
3074         if ($ostype eq 'win7' || $ostype eq 'win8') {
3075             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3076         }
3077     }
3078
3079     push @$rtcFlags, 'driftfix=slew' if $tdf;
3080
3081     if ($nokvm) {
3082         push @$machineFlags, 'accel=tcg';
3083     } else {
3084         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3085     }
3086
3087     if ($machine_type) {
3088         push @$machineFlags, "type=${machine_type}";
3089     }
3090
3091     if ($conf->{startdate}) {
3092         push @$rtcFlags, "base=$conf->{startdate}";
3093     } elsif ($useLocaltime) {
3094         push @$rtcFlags, 'base=localtime';
3095     }
3096
3097     my $cpu = $nokvm ? "qemu64" : "kvm64";
3098     if (my $cputype = $conf->{cpu}) {
3099         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3100             or die "Cannot parse cpu description: $cputype\n";
3101         $cpu = $cpuconf->{cputype};
3102         $kvm_off = 1 if $cpuconf->{hidden};
3103     }
3104
3105     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3106
3107     push @$cpuFlags , '-x2apic'
3108         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3109
3110     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3111
3112     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3113
3114     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3115
3116         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3117         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3118     }
3119
3120     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3121
3122     push @$cpuFlags, 'kvm=off' if $kvm_off;
3123
3124     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3125
3126     push @$cmd, '-cpu', $cpu;
3127
3128     my $memory = $conf->{memory} || $defaults->{memory};
3129     my $static_memory = 0;
3130     my $dimm_memory = 0;
3131
3132     if ($hotplug_features->{memory}) {
3133         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3134         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3135         $static_memory = $STATICMEM;
3136         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3137         $dimm_memory = $memory - $static_memory;
3138         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3139
3140     } else {
3141
3142         $static_memory = $memory;
3143         push @$cmd, '-m', $static_memory;
3144     }
3145
3146     if ($conf->{numa}) {
3147
3148         my $numa_totalmemory = undef;
3149         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3150             next if !$conf->{"numa$i"};
3151             my $numa = parse_numa($conf->{"numa$i"});
3152             next if !$numa;
3153             # memory
3154             die "missing numa node$i memory value\n" if !$numa->{memory};
3155             my $numa_memory = $numa->{memory};
3156             $numa_totalmemory += $numa_memory;
3157             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3158
3159             # cpus
3160             my $cpus_start = $numa->{cpus}->{start};
3161             die "missing numa node$i cpus\n" if !defined($cpus_start);
3162             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3163             my $cpus = $cpus_start;
3164             if (defined($cpus_end)) {
3165                 $cpus .= "-$cpus_end";
3166                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3167             }
3168
3169             # hostnodes
3170             my $hostnodes_start = $numa->{hostnodes}->{start};
3171             if (defined($hostnodes_start)) {
3172                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3173                 my $hostnodes = $hostnodes_start;
3174                 if (defined($hostnodes_end)) {
3175                     $hostnodes .= "-$hostnodes_end";
3176                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3177                 }
3178
3179                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3180                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3181                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3182                 }
3183
3184                 # policy
3185                 my $policy = $numa->{policy};
3186                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3187                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3188             }
3189
3190             push @$cmd, '-object', $numa_object;
3191             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3192         }
3193
3194         die "total memory for NUMA nodes must be equal to vm static memory\n"
3195             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3196
3197         #if no custom tology, we split memory and cores across numa nodes
3198         if(!$numa_totalmemory) {
3199
3200             my $numa_memory = ($static_memory / $sockets) . "M";
3201
3202             for (my $i = 0; $i < $sockets; $i++)  {
3203
3204                 my $cpustart = ($cores * $i);
3205                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3206                 my $cpus = $cpustart;
3207                 $cpus .= "-$cpuend" if $cpuend;
3208
3209                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3210                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3211             }
3212         }
3213     }
3214
3215     if ($hotplug_features->{memory}) {
3216         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3217             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3218             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3219             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3220
3221             #if dimm_memory is not aligned to dimm map
3222             if($current_size > $memory) {
3223                  $conf->{memory} = $current_size;
3224                  write_config($vmid, $conf);
3225             }
3226         });
3227     }
3228
3229     push @$cmd, '-S' if $conf->{freeze};
3230
3231     # set keyboard layout
3232     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3233     push @$cmd, '-k', $kb if $kb;
3234
3235     # enable sound
3236     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3237     #push @$cmd, '-soundhw', 'es1370';
3238     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3239
3240     if($conf->{agent}) {
3241         my $qgasocket = qmp_socket($vmid, 1);
3242         my $pciaddr = print_pci_addr("qga0", $bridges);
3243         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3244         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3245         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3246     }
3247
3248     my $spice_port;
3249
3250     if ($qxlnum) {
3251         if ($qxlnum > 1) {
3252             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3253                 for(my $i = 1; $i < $qxlnum; $i++){
3254                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3255                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3256                 }
3257             } else {
3258                 # assume other OS works like Linux
3259                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3260                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3261             }
3262         }
3263
3264         my $pciaddr = print_pci_addr("spice", $bridges);
3265
3266         my $nodename = PVE::INotify::nodename();
3267         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3268         $spice_port = PVE::Tools::next_spice_port($pfamily);
3269
3270         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3271
3272         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3273         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3274         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3275     }
3276
3277     # enable balloon by default, unless explicitly disabled
3278     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3279         $pciaddr = print_pci_addr("balloon0", $bridges);
3280         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3281     }
3282
3283     if ($conf->{watchdog}) {
3284         my $wdopts = parse_watchdog($conf->{watchdog});
3285         $pciaddr = print_pci_addr("watchdog", $bridges);
3286         my $watchdog = $wdopts->{model} || 'i6300esb';
3287         push @$devices, '-device', "$watchdog$pciaddr";
3288         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3289     }
3290
3291     my $vollist = [];
3292     my $scsicontroller = {};
3293     my $ahcicontroller = {};
3294     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3295
3296     # Add iscsi initiator name if available
3297     if (my $initiator = get_initiator_name()) {
3298         push @$devices, '-iscsi', "initiator-name=$initiator";
3299     }
3300
3301     foreach_drive($conf, sub {
3302         my ($ds, $drive) = @_;
3303
3304         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3305             push @$vollist, $drive->{file};
3306         }
3307
3308         $use_virtio = 1 if $ds =~ m/^virtio/;
3309
3310         if (drive_is_cdrom ($drive)) {
3311             if ($bootindex_hash->{d}) {
3312                 $drive->{bootindex} = $bootindex_hash->{d};
3313                 $bootindex_hash->{d} += 1;
3314             }
3315         } else {
3316             if ($bootindex_hash->{c}) {
3317                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3318                 $bootindex_hash->{c} += 1;
3319             }
3320         }
3321
3322         if($drive->{interface} eq 'virtio'){
3323            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3324         }
3325
3326         if ($drive->{interface} eq 'scsi') {
3327
3328             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3329
3330             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3331             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3332
3333             my $iothread = '';
3334             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3335                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3336                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3337             }
3338
3339             my $queues = '';
3340             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3341                 $queues = ",num_queues=$drive->{queues}";
3342             }
3343
3344             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3345             $scsicontroller->{$controller}=1;
3346         }
3347
3348         if ($drive->{interface} eq 'sata') {
3349            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3350            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3351            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3352            $ahcicontroller->{$controller}=1;
3353         }
3354
3355         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3356         push @$devices, '-drive',$drive_cmd;
3357         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3358     });
3359
3360     for (my $i = 0; $i < $MAX_NETS; $i++) {
3361          next if !$conf->{"net$i"};
3362          my $d = parse_net($conf->{"net$i"});
3363          next if !$d;
3364
3365          $use_virtio = 1 if $d->{model} eq 'virtio';
3366
3367          if ($bootindex_hash->{n}) {
3368             $d->{bootindex} = $bootindex_hash->{n};
3369             $bootindex_hash->{n} += 1;
3370          }
3371
3372          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3373          push @$devices, '-netdev', $netdevfull;
3374
3375          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3376          push @$devices, '-device', $netdevicefull;
3377     }
3378
3379     if (!$q35) {
3380         # add pci bridges
3381         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3382            $bridges->{1} = 1;
3383            $bridges->{2} = 1;
3384         }
3385
3386         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3387
3388         while (my ($k, $v) = each %$bridges) {
3389             $pciaddr = print_pci_addr("pci.$k");
3390             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3391         }
3392     }
3393
3394     # add custom args
3395     if ($conf->{args}) {
3396         my $aa = PVE::Tools::split_args($conf->{args});
3397         push @$cmd, @$aa;
3398     }
3399
3400     push @$cmd, @$devices;
3401     push @$cmd, '-rtc', join(',', @$rtcFlags)
3402         if scalar(@$rtcFlags);
3403     push @$cmd, '-machine', join(',', @$machineFlags)
3404         if scalar(@$machineFlags);
3405     push @$cmd, '-global', join(',', @$globalFlags)
3406         if scalar(@$globalFlags);
3407
3408     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3409 }
3410
3411 sub vnc_socket {
3412     my ($vmid) = @_;
3413     return "${var_run_tmpdir}/$vmid.vnc";
3414 }
3415
3416 sub spice_port {
3417     my ($vmid) = @_;
3418
3419     my $res = vm_mon_cmd($vmid, 'query-spice');
3420
3421     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3422 }
3423
3424 sub qmp_socket {
3425     my ($vmid, $qga) = @_;
3426     my $sockettype = $qga ? 'qga' : 'qmp';
3427     return "${var_run_tmpdir}/$vmid.$sockettype";
3428 }
3429
3430 sub pidfile_name {
3431     my ($vmid) = @_;
3432     return "${var_run_tmpdir}/$vmid.pid";
3433 }
3434
3435 sub vm_devices_list {
3436     my ($vmid) = @_;
3437
3438     my $res = vm_mon_cmd($vmid, 'query-pci');
3439     my $devices = {};
3440     foreach my $pcibus (@$res) {
3441         foreach my $device (@{$pcibus->{devices}}) {
3442             next if !$device->{'qdev_id'};
3443             if ($device->{'pci_bridge'}) {
3444                 $devices->{$device->{'qdev_id'}} = 1;
3445                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3446                     next if !$bridge_device->{'qdev_id'};
3447                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3448                     $devices->{$device->{'qdev_id'}}++;
3449                 }
3450             } else {
3451                 $devices->{$device->{'qdev_id'}} = 1;
3452             }
3453         }
3454     }
3455
3456     my $resblock = vm_mon_cmd($vmid, 'query-block');
3457     foreach my $block (@$resblock) {
3458         if($block->{device} =~ m/^drive-(\S+)/){
3459                 $devices->{$1} = 1;
3460         }
3461     }
3462
3463     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3464     foreach my $mice (@$resmice) {
3465         if ($mice->{name} eq 'QEMU HID Tablet') {
3466             $devices->{tablet} = 1;
3467             last;
3468         }
3469     }
3470
3471     return $devices;
3472 }
3473
3474 sub vm_deviceplug {
3475     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3476
3477     my $q35 = machine_type_is_q35($conf);
3478
3479     my $devices_list = vm_devices_list($vmid);
3480     return 1 if defined($devices_list->{$deviceid});
3481
3482     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3483
3484     if ($deviceid eq 'tablet') {
3485
3486         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3487
3488     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3489
3490         qemu_iothread_add($vmid, $deviceid, $device);
3491
3492         qemu_driveadd($storecfg, $vmid, $device);
3493         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3494
3495         qemu_deviceadd($vmid, $devicefull);
3496         eval { qemu_deviceaddverify($vmid, $deviceid); };
3497         if (my $err = $@) {
3498             eval { qemu_drivedel($vmid, $deviceid); };
3499             warn $@ if $@;
3500             die $err;
3501         }
3502
3503     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3504
3505
3506         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3507         my $pciaddr = print_pci_addr($deviceid);
3508         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3509
3510         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3511
3512         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3513             qemu_iothread_add($vmid, $deviceid, $device);
3514             $devicefull .= ",iothread=iothread-$deviceid";
3515         }
3516
3517         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3518             $devicefull .= ",num_queues=$device->{queues}";
3519         }
3520
3521         qemu_deviceadd($vmid, $devicefull);
3522         qemu_deviceaddverify($vmid, $deviceid);
3523
3524     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3525
3526         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3527         qemu_driveadd($storecfg, $vmid, $device);
3528
3529         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3530         eval { qemu_deviceadd($vmid, $devicefull); };
3531         if (my $err = $@) {
3532             eval { qemu_drivedel($vmid, $deviceid); };
3533             warn $@ if $@;
3534             die $err;
3535         }
3536
3537     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3538
3539         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3540
3541         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3542         my $use_old_bios_files = undef;
3543         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3544
3545         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3546         qemu_deviceadd($vmid, $netdevicefull);
3547         eval { qemu_deviceaddverify($vmid, $deviceid); };
3548         if (my $err = $@) {
3549             eval { qemu_netdevdel($vmid, $deviceid); };
3550             warn $@ if $@;
3551             die $err;
3552         }
3553
3554     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3555
3556         my $bridgeid = $2;
3557         my $pciaddr = print_pci_addr($deviceid);
3558         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3559
3560         qemu_deviceadd($vmid, $devicefull);
3561         qemu_deviceaddverify($vmid, $deviceid);
3562
3563     } else {
3564         die "can't hotplug device '$deviceid'\n";
3565     }
3566
3567     return 1;
3568 }
3569
3570 # fixme: this should raise exceptions on error!
3571 sub vm_deviceunplug {
3572     my ($vmid, $conf, $deviceid) = @_;
3573
3574     my $devices_list = vm_devices_list($vmid);
3575     return 1 if !defined($devices_list->{$deviceid});
3576
3577     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3578
3579     if ($deviceid eq 'tablet') {
3580
3581         qemu_devicedel($vmid, $deviceid);
3582
3583     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3584
3585         qemu_devicedel($vmid, $deviceid);
3586         qemu_devicedelverify($vmid, $deviceid);
3587         qemu_drivedel($vmid, $deviceid);
3588         qemu_iothread_del($conf, $vmid, $deviceid);
3589
3590     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3591
3592         qemu_devicedel($vmid, $deviceid);
3593         qemu_devicedelverify($vmid, $deviceid);
3594         qemu_iothread_del($conf, $vmid, $deviceid);
3595
3596     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3597
3598         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3599         my $device = parse_drive($deviceid, $conf->{$deviceid});
3600         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3601
3602         qemu_devicedel($vmid, $deviceid);
3603         qemu_drivedel($vmid, $deviceid);
3604         qemu_deletescsihw($conf, $vmid, $deviceid);
3605
3606     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3607
3608         qemu_devicedel($vmid, $deviceid);
3609         qemu_devicedelverify($vmid, $deviceid);
3610         qemu_netdevdel($vmid, $deviceid);
3611
3612     } else {
3613         die "can't unplug device '$deviceid'\n";
3614     }
3615
3616     return 1;
3617 }
3618
3619 sub qemu_deviceadd {
3620     my ($vmid, $devicefull) = @_;
3621
3622     $devicefull = "driver=".$devicefull;
3623     my %options =  split(/[=,]/, $devicefull);
3624
3625     vm_mon_cmd($vmid, "device_add" , %options);
3626 }
3627
3628 sub qemu_devicedel {
3629     my ($vmid, $deviceid) = @_;
3630
3631     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3632 }
3633
3634 sub qemu_iothread_add {
3635     my($vmid, $deviceid, $device) = @_;
3636
3637     if ($device->{iothread}) {
3638         my $iothreads = vm_iothreads_list($vmid);
3639         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3640     }
3641 }
3642
3643 sub qemu_iothread_del {
3644     my($conf, $vmid, $deviceid) = @_;
3645
3646     my $device = parse_drive($deviceid, $conf->{$deviceid});
3647     if ($device->{iothread}) {
3648         my $iothreads = vm_iothreads_list($vmid);
3649         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3650     }
3651 }
3652
3653 sub qemu_objectadd {
3654     my($vmid, $objectid, $qomtype) = @_;
3655
3656     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3657
3658     return 1;
3659 }
3660
3661 sub qemu_objectdel {
3662     my($vmid, $objectid) = @_;
3663
3664     vm_mon_cmd($vmid, "object-del", id => $objectid);
3665
3666     return 1;
3667 }
3668
3669 sub qemu_driveadd {
3670     my ($storecfg, $vmid, $device) = @_;
3671
3672     my $drive = print_drive_full($storecfg, $vmid, $device);
3673     $drive =~ s/\\/\\\\/g;
3674     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3675
3676     # If the command succeeds qemu prints: "OK"
3677     return 1 if $ret =~ m/OK/s;
3678
3679     die "adding drive failed: $ret\n";
3680 }
3681
3682 sub qemu_drivedel {
3683     my($vmid, $deviceid) = @_;
3684
3685     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3686     $ret =~ s/^\s+//;
3687
3688     return 1 if $ret eq "";
3689
3690     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3691     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3692
3693     die "deleting drive $deviceid failed : $ret\n";
3694 }
3695
3696 sub qemu_deviceaddverify {
3697     my ($vmid, $deviceid) = @_;
3698
3699     for (my $i = 0; $i <= 5; $i++) {
3700          my $devices_list = vm_devices_list($vmid);
3701          return 1 if defined($devices_list->{$deviceid});
3702          sleep 1;
3703     }
3704
3705     die "error on hotplug device '$deviceid'\n";
3706 }
3707
3708
3709 sub qemu_devicedelverify {
3710     my ($vmid, $deviceid) = @_;
3711
3712     # need to verify that the device is correctly removed as device_del
3713     # is async and empty return is not reliable
3714
3715     for (my $i = 0; $i <= 5; $i++) {
3716          my $devices_list = vm_devices_list($vmid);
3717          return 1 if !defined($devices_list->{$deviceid});
3718          sleep 1;
3719     }
3720
3721     die "error on hot-unplugging device '$deviceid'\n";
3722 }
3723
3724 sub qemu_findorcreatescsihw {
3725     my ($storecfg, $conf, $vmid, $device) = @_;
3726
3727     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3728
3729     my $scsihwid="$controller_prefix$controller";
3730     my $devices_list = vm_devices_list($vmid);
3731
3732     if(!defined($devices_list->{$scsihwid})) {
3733         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3734     }
3735
3736     return 1;
3737 }
3738
3739 sub qemu_deletescsihw {
3740     my ($conf, $vmid, $opt) = @_;
3741
3742     my $device = parse_drive($opt, $conf->{$opt});
3743
3744     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3745         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3746         return 1;
3747     }
3748
3749     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3750
3751     my $devices_list = vm_devices_list($vmid);
3752     foreach my $opt (keys %{$devices_list}) {
3753         if (PVE::QemuServer::valid_drivename($opt)) {
3754             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3755             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3756                 return 1;
3757             }
3758         }
3759     }
3760
3761     my $scsihwid="scsihw$controller";
3762
3763     vm_deviceunplug($vmid, $conf, $scsihwid);
3764
3765     return 1;
3766 }
3767
3768 sub qemu_add_pci_bridge {
3769     my ($storecfg, $conf, $vmid, $device) = @_;
3770
3771     my $bridges = {};
3772
3773     my $bridgeid;
3774
3775     print_pci_addr($device, $bridges);
3776
3777     while (my ($k, $v) = each %$bridges) {
3778         $bridgeid = $k;
3779     }
3780     return 1 if !defined($bridgeid) || $bridgeid < 1;
3781
3782     my $bridge = "pci.$bridgeid";
3783     my $devices_list = vm_devices_list($vmid);
3784
3785     if (!defined($devices_list->{$bridge})) {
3786         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3787     }
3788
3789     return 1;
3790 }
3791
3792 sub qemu_set_link_status {
3793     my ($vmid, $device, $up) = @_;
3794
3795     vm_mon_cmd($vmid, "set_link", name => $device,
3796                up => $up ? JSON::true : JSON::false);
3797 }
3798
3799 sub qemu_netdevadd {
3800     my ($vmid, $conf, $device, $deviceid) = @_;
3801
3802     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3803     my %options =  split(/[=,]/, $netdev);
3804
3805     vm_mon_cmd($vmid, "netdev_add",  %options);
3806     return 1;
3807 }
3808
3809 sub qemu_netdevdel {
3810     my ($vmid, $deviceid) = @_;
3811
3812     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3813 }
3814
3815 sub qemu_cpu_hotplug {
3816     my ($vmid, $conf, $vcpus) = @_;
3817
3818     my $sockets = 1;
3819     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3820     $sockets = $conf->{sockets} if  $conf->{sockets};
3821     my $cores = $conf->{cores} || 1;
3822     my $maxcpus = $sockets * $cores;
3823
3824     $vcpus = $maxcpus if !$vcpus;
3825
3826     die "you can't add more vcpus than maxcpus\n"
3827         if $vcpus > $maxcpus;
3828
3829     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3830     die "online cpu unplug is not yet possible\n"
3831         if $vcpus < $currentvcpus;
3832
3833     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3834     die "vcpus in running vm is different than configuration\n"
3835         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3836
3837     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3838         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3839     }
3840 }
3841
3842 sub qemu_memory_hotplug {
3843     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3844
3845     return $value if !check_running($vmid);
3846
3847     my $memory = $conf->{memory} || $defaults->{memory};
3848     $value = $defaults->{memory} if !$value;
3849     return $value if $value == $memory;
3850
3851     my $static_memory = $STATICMEM;
3852     my $dimm_memory = $memory - $static_memory;
3853
3854     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3855     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3856
3857
3858     my $sockets = 1;
3859     $sockets = $conf->{sockets} if $conf->{sockets};
3860
3861     if($value > $memory) {
3862
3863         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3864             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3865
3866                 return if $current_size <= $conf->{memory};
3867
3868                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3869                 if (my $err = $@) {
3870                     eval { qemu_objectdel($vmid, "mem-$name"); };
3871                     die $err;
3872                 }
3873
3874                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3875                 if (my $err = $@) {
3876                     eval { qemu_objectdel($vmid, "mem-$name"); };
3877                     die $err;
3878                 }
3879                 #update conf after each succesful module hotplug
3880                 $conf->{memory} = $current_size;
3881                 write_config($vmid, $conf);
3882         });
3883
3884     } else {
3885
3886         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3887             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3888
3889                 return if $current_size >= $conf->{memory};
3890                 print "try to unplug memory dimm $name\n";
3891
3892                 my $retry = 0;
3893                 while (1) {
3894                     eval { qemu_devicedel($vmid, $name) };
3895                     sleep 3;
3896                     my $dimm_list = qemu_dimm_list($vmid);
3897                     last if !$dimm_list->{$name};
3898                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3899                     $retry++;
3900                 }
3901
3902                 #update conf after each succesful module unplug
3903                 $conf->{memory} = $current_size;
3904
3905                 eval { qemu_objectdel($vmid, "mem-$name"); };
3906                 write_config($vmid, $conf);
3907         });
3908     }
3909 }
3910
3911 sub qemu_dimm_list {
3912     my ($vmid) = @_;
3913
3914     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3915     my $dimms = {};
3916
3917     foreach my $dimm (@$dimmarray) {
3918
3919         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3920         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3921         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3922         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3923         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3924     }
3925     return $dimms;
3926 }
3927
3928 sub qemu_block_set_io_throttle {
3929     my ($vmid, $deviceid,
3930         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3931         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3932
3933     return if !check_running($vmid) ;
3934
3935     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3936         bps => int($bps),
3937         bps_rd => int($bps_rd),
3938         bps_wr => int($bps_wr),
3939         iops => int($iops),
3940         iops_rd => int($iops_rd),
3941         iops_wr => int($iops_wr),
3942         bps_max => int($bps_max),
3943         bps_rd_max => int($bps_rd_max),
3944         bps_wr_max => int($bps_wr_max),
3945         iops_max => int($iops_max),
3946         iops_rd_max => int($iops_rd_max),
3947         iops_wr_max => int($iops_wr_max)
3948     );
3949
3950 }
3951
3952 # old code, only used to shutdown old VM after update
3953 sub __read_avail {
3954     my ($fh, $timeout) = @_;
3955
3956     my $sel = new IO::Select;
3957     $sel->add($fh);
3958
3959     my $res = '';
3960     my $buf;
3961
3962     my @ready;
3963     while (scalar (@ready = $sel->can_read($timeout))) {
3964         my $count;
3965         if ($count = $fh->sysread($buf, 8192)) {
3966             if ($buf =~ /^(.*)\(qemu\) $/s) {
3967                 $res .= $1;
3968                 last;
3969             } else {
3970                 $res .= $buf;
3971             }
3972         } else {
3973             if (!defined($count)) {
3974                 die "$!\n";
3975             }
3976             last;
3977         }
3978     }
3979
3980     die "monitor read timeout\n" if !scalar(@ready);
3981
3982     return $res;
3983 }
3984
3985 # old code, only used to shutdown old VM after update
3986 sub vm_monitor_command {
3987     my ($vmid, $cmdstr, $nocheck) = @_;
3988
3989     my $res;
3990
3991     eval {
3992         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3993
3994         my $sname = "${var_run_tmpdir}/$vmid.mon";
3995
3996         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3997             die "unable to connect to VM $vmid socket - $!\n";
3998
3999         my $timeout = 3;
4000
4001         # hack: migrate sometime blocks the monitor (when migrate_downtime
4002         # is set)
4003         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4004             $timeout = 60*60; # 1 hour
4005         }
4006
4007         # read banner;
4008         my $data = __read_avail($sock, $timeout);
4009
4010         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4011             die "got unexpected qemu monitor banner\n";
4012         }
4013
4014         my $sel = new IO::Select;
4015         $sel->add($sock);
4016
4017         if (!scalar(my @ready = $sel->can_write($timeout))) {
4018             die "monitor write error - timeout";
4019         }
4020
4021         my $fullcmd = "$cmdstr\r";
4022
4023         # syslog('info', "VM $vmid monitor command: $cmdstr");
4024
4025         my $b;
4026         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4027             die "monitor write error - $!";
4028         }
4029
4030         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4031
4032         $timeout = 20;
4033
4034         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4035             $timeout = 60*60; # 1 hour
4036         } elsif ($cmdstr =~ m/^(eject|change)/) {
4037             $timeout = 60; # note: cdrom mount command is slow
4038         }
4039         if ($res = __read_avail($sock, $timeout)) {
4040
4041             my @lines = split("\r?\n", $res);
4042
4043             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4044
4045             $res = join("\n", @lines);
4046             $res .= "\n";
4047         }
4048     };
4049
4050     my $err = $@;
4051
4052     if ($err) {
4053         syslog("err", "VM $vmid monitor command failed - $err");
4054         die $err;
4055     }
4056
4057     return $res;
4058 }
4059
4060 sub qemu_block_resize {
4061     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4062
4063     my $running = check_running($vmid);
4064
4065     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4066
4067     return if !$running;
4068
4069     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4070
4071 }
4072
4073 sub qemu_volume_snapshot {
4074     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4075
4076     my $running = check_running($vmid);
4077
4078     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4079         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4080     } else {
4081         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4082     }
4083 }
4084
4085 sub qemu_volume_snapshot_delete {
4086     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4087
4088     my $running = check_running($vmid);
4089
4090     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4091
4092     return if !$running;
4093
4094     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4095 }
4096
4097 sub set_migration_caps {
4098     my ($vmid) = @_;
4099
4100     my $cap_ref = [];
4101
4102     my $enabled_cap = {
4103         "auto-converge" => 1,
4104         "xbzrle" => 1,
4105         "x-rdma-pin-all" => 0,
4106         "zero-blocks" => 0,
4107         "compress" => 0
4108     };
4109
4110     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4111
4112     for my $supported_capability (@$supported_capabilities) {
4113         push @$cap_ref, {
4114             capability => $supported_capability->{capability},
4115             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4116         };
4117     }
4118
4119     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4120 }
4121
4122 my $fast_plug_option = {
4123     'lock' => 1,
4124     'name' => 1,
4125     'onboot' => 1,
4126     'shares' => 1,
4127     'startup' => 1,
4128     'description' => 1,
4129 };
4130
4131 # hotplug changes in [PENDING]
4132 # $selection hash can be used to only apply specified options, for
4133 # example: { cores => 1 } (only apply changed 'cores')
4134 # $errors ref is used to return error messages
4135 sub vmconfig_hotplug_pending {
4136     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4137
4138     my $defaults = load_defaults();
4139
4140     # commit values which do not have any impact on running VM first
4141     # Note: those option cannot raise errors, we we do not care about
4142     # $selection and always apply them.
4143
4144     my $add_error = sub {
4145         my ($opt, $msg) = @_;
4146         $errors->{$opt} = "hotplug problem - $msg";
4147     };
4148
4149     my $changes = 0;
4150     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4151         if ($fast_plug_option->{$opt}) {
4152             $conf->{$opt} = $conf->{pending}->{$opt};
4153             delete $conf->{pending}->{$opt};
4154             $changes = 1;
4155         }
4156     }
4157
4158     if ($changes) {
4159         write_config($vmid, $conf);
4160         $conf = load_config($vmid); # update/reload
4161     }
4162
4163     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4164
4165     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4166     while (my ($opt, $force) = each %$pending_delete_hash) {
4167         next if $selection && !$selection->{$opt};
4168         eval {
4169             if ($opt eq 'hotplug') {
4170                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4171             } elsif ($opt eq 'tablet') {
4172                 die "skip\n" if !$hotplug_features->{usb};
4173                 if ($defaults->{tablet}) {
4174                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4175                 } else {
4176                     vm_deviceunplug($vmid, $conf, $opt);
4177                 }
4178             } elsif ($opt eq 'vcpus') {
4179                 die "skip\n" if !$hotplug_features->{cpu};
4180                 qemu_cpu_hotplug($vmid, $conf, undef);
4181             } elsif ($opt eq 'balloon') {
4182                 # enable balloon device is not hotpluggable
4183                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4184             } elsif ($fast_plug_option->{$opt}) {
4185                 # do nothing
4186             } elsif ($opt =~ m/^net(\d+)$/) {
4187                 die "skip\n" if !$hotplug_features->{network};
4188                 vm_deviceunplug($vmid, $conf, $opt);
4189             } elsif (valid_drivename($opt)) {
4190                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4191                 vm_deviceunplug($vmid, $conf, $opt);
4192                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4193             } elsif ($opt =~ m/^memory$/) {
4194                 die "skip\n" if !$hotplug_features->{memory};
4195                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4196             } elsif ($opt eq 'cpuunits') {
4197                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4198             } elsif ($opt eq 'cpulimit') {
4199                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4200             } else {
4201                 die "skip\n";
4202             }
4203         };
4204         if (my $err = $@) {
4205             &$add_error($opt, $err) if $err ne "skip\n";
4206         } else {
4207             # save new config if hotplug was successful
4208             delete $conf->{$opt};
4209             vmconfig_undelete_pending_option($conf, $opt);
4210             write_config($vmid, $conf);
4211             $conf = load_config($vmid); # update/reload
4212         }
4213     }
4214
4215     foreach my $opt (keys %{$conf->{pending}}) {
4216         next if $selection && !$selection->{$opt};
4217         my $value = $conf->{pending}->{$opt};
4218         eval {
4219             if ($opt eq 'hotplug') {
4220                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4221             } elsif ($opt eq 'tablet') {
4222                 die "skip\n" if !$hotplug_features->{usb};
4223                 if ($value == 1) {
4224                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4225                 } elsif ($value == 0) {
4226                     vm_deviceunplug($vmid, $conf, $opt);
4227                 }
4228             } elsif ($opt eq 'vcpus') {
4229                 die "skip\n" if !$hotplug_features->{cpu};
4230                 qemu_cpu_hotplug($vmid, $conf, $value);
4231             } elsif ($opt eq 'balloon') {
4232                 # enable/disable balloning device is not hotpluggable
4233                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4234                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4235                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4236
4237                 # allow manual ballooning if shares is set to zero
4238                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4239                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4240                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4241                 }
4242             } elsif ($opt =~ m/^net(\d+)$/) {
4243                 # some changes can be done without hotplug
4244                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4245                                     $vmid, $opt, $value);
4246             } elsif (valid_drivename($opt)) {
4247                 # some changes can be done without hotplug
4248                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4249                                      $vmid, $opt, $value, 1);
4250             } elsif ($opt =~ m/^memory$/) { #dimms
4251                 die "skip\n" if !$hotplug_features->{memory};
4252                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4253             } elsif ($opt eq 'cpuunits') {
4254                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4255             } elsif ($opt eq 'cpulimit') {
4256                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4257                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4258             } else {
4259                 die "skip\n";  # skip non-hot-pluggable options
4260             }
4261         };
4262         if (my $err = $@) {
4263             &$add_error($opt, $err) if $err ne "skip\n";
4264         } else {
4265             # save new config if hotplug was successful
4266             $conf->{$opt} = $value;
4267             delete $conf->{pending}->{$opt};
4268             write_config($vmid, $conf);
4269             $conf = load_config($vmid); # update/reload
4270         }
4271     }
4272 }
4273
4274 sub try_deallocate_drive {
4275     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4276
4277     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4278         my $volid = $drive->{file};
4279         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4280             my $sid = PVE::Storage::parse_volume_id($volid);
4281             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4282
4283             # check if the disk is really unused
4284             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4285                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4286             PVE::Storage::vdisk_free($storecfg, $volid);
4287             return 1;
4288         } else {
4289             # If vm is not owner of this disk remove from config
4290             return 1;
4291         }
4292     }
4293
4294     return undef;
4295 }
4296
4297 sub vmconfig_delete_or_detach_drive {
4298     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4299
4300     my $drive = parse_drive($opt, $conf->{$opt});
4301
4302     my $rpcenv = PVE::RPCEnvironment::get();
4303     my $authuser = $rpcenv->get_user();
4304
4305     if ($force) {
4306         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4307         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4308     } else {
4309         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4310     }
4311 }
4312
4313 sub vmconfig_apply_pending {
4314     my ($vmid, $conf, $storecfg) = @_;
4315
4316     # cold plug
4317
4318     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4319     while (my ($opt, $force) = each %$pending_delete_hash) {
4320         die "internal error" if $opt =~ m/^unused/;
4321         $conf = load_config($vmid); # update/reload
4322         if (!defined($conf->{$opt})) {
4323             vmconfig_undelete_pending_option($conf, $opt);
4324             write_config($vmid, $conf);
4325         } elsif (valid_drivename($opt)) {
4326             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4327             vmconfig_undelete_pending_option($conf, $opt);
4328             delete $conf->{$opt};
4329             write_config($vmid, $conf);
4330         } else {
4331             vmconfig_undelete_pending_option($conf, $opt);
4332             delete $conf->{$opt};
4333             write_config($vmid, $conf);
4334         }
4335     }
4336
4337     $conf = load_config($vmid); # update/reload
4338
4339     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4340         $conf = load_config($vmid); # update/reload
4341
4342         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4343             # skip if nothing changed
4344         } elsif (valid_drivename($opt)) {
4345             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4346                 if defined($conf->{$opt});
4347             $conf->{$opt} = $conf->{pending}->{$opt};
4348         } else {
4349             $conf->{$opt} = $conf->{pending}->{$opt};
4350         }
4351
4352         delete $conf->{pending}->{$opt};
4353         write_config($vmid, $conf);
4354     }
4355 }
4356
4357 my $safe_num_ne = sub {
4358     my ($a, $b) = @_;
4359
4360     return 0 if !defined($a) && !defined($b);
4361     return 1 if !defined($a);
4362     return 1 if !defined($b);
4363
4364     return $a != $b;
4365 };
4366
4367 my $safe_string_ne = sub {
4368     my ($a, $b) = @_;
4369
4370     return 0 if !defined($a) && !defined($b);
4371     return 1 if !defined($a);
4372     return 1 if !defined($b);
4373
4374     return $a ne $b;
4375 };
4376
4377 sub vmconfig_update_net {
4378     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4379
4380     my $newnet = parse_net($value);
4381
4382     if ($conf->{$opt}) {
4383         my $oldnet = parse_net($conf->{$opt});
4384
4385         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4386             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4387             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4388             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4389
4390             # for non online change, we try to hot-unplug
4391             die "skip\n" if !$hotplug;
4392             vm_deviceunplug($vmid, $conf, $opt);
4393         } else {
4394
4395             die "internal error" if $opt !~ m/net(\d+)/;
4396             my $iface = "tap${vmid}i$1";
4397
4398             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4399                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4400             }
4401
4402             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4403                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4404                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4405                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4406                 PVE::Network::tap_unplug($iface);
4407                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4408             }
4409
4410             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4411                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4412             }
4413
4414             return 1;
4415         }
4416     }
4417
4418     if ($hotplug) {
4419         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4420     } else {
4421         die "skip\n";
4422     }
4423 }
4424
4425 sub vmconfig_update_disk {
4426     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4427
4428     # fixme: do we need force?
4429
4430     my $drive = parse_drive($opt, $value);
4431
4432     if ($conf->{$opt}) {
4433
4434         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4435
4436             my $media = $drive->{media} || 'disk';
4437             my $oldmedia = $old_drive->{media} || 'disk';
4438             die "unable to change media type\n" if $media ne $oldmedia;
4439
4440             if (!drive_is_cdrom($old_drive)) {
4441
4442                 if ($drive->{file} ne $old_drive->{file}) {
4443
4444                     die "skip\n" if !$hotplug;
4445
4446                     # unplug and register as unused
4447                     vm_deviceunplug($vmid, $conf, $opt);
4448                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4449
4450                 } else {
4451                     # update existing disk
4452
4453                     # skip non hotpluggable value
4454                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4455                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4456                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4457                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4458                         die "skip\n";
4459                     }
4460
4461                     # apply throttle
4462                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4463                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4464                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4465                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4466                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4467                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4468                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4469                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4470                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4471                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4472                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4473                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4474
4475                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4476                                                    ($drive->{mbps} || 0)*1024*1024,
4477                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4478                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4479                                                    $drive->{iops} || 0,
4480                                                    $drive->{iops_rd} || 0,
4481                                                    $drive->{iops_wr} || 0,
4482                                                    ($drive->{mbps_max} || 0)*1024*1024,
4483                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4484                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4485                                                    $drive->{iops_max} || 0,
4486                                                    $drive->{iops_rd_max} || 0,
4487                                                    $drive->{iops_wr_max} || 0);
4488
4489                     }
4490
4491                     return 1;
4492                 }
4493
4494             } else { # cdrom
4495
4496                 if ($drive->{file} eq 'none') {
4497                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4498                 } else {
4499                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4500                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4501                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4502                 }
4503
4504                 return 1;
4505             }
4506         }
4507     }
4508
4509     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4510     # hotplug new disks
4511     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4512     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4513 }
4514
4515 sub vm_start {
4516     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4517         $forcemachine, $spice_ticket) = @_;
4518
4519     lock_config($vmid, sub {
4520         my $conf = load_config($vmid, $migratedfrom);
4521
4522         die "you can't start a vm if it's a template\n" if is_template($conf);
4523
4524         check_lock($conf) if !$skiplock;
4525
4526         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4527
4528         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4529             vmconfig_apply_pending($vmid, $conf, $storecfg);
4530             $conf = load_config($vmid); # update/reload
4531         }
4532
4533         my $defaults = load_defaults();
4534
4535         # set environment variable useful inside network script
4536         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4537
4538         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4539
4540         my $migrate_port = 0;
4541         my $migrate_uri;
4542         if ($statefile) {
4543             if ($statefile eq 'tcp') {
4544                 my $localip = "localhost";
4545                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4546                 my $nodename = PVE::INotify::nodename();
4547                 if ($datacenterconf->{migration_unsecure}) {
4548                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4549                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4550                 }
4551                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4552                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4553                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4554                 push @$cmd, '-incoming', $migrate_uri;
4555                 push @$cmd, '-S';
4556             } else {
4557                 push @$cmd, '-loadstate', $statefile;
4558             }
4559         } elsif ($paused) {
4560             push @$cmd, '-S';
4561         }
4562
4563         # host pci devices
4564         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4565           my $d = parse_hostpci($conf->{"hostpci$i"});
4566           next if !$d;
4567           my $pcidevices = $d->{pciid};
4568           foreach my $pcidevice (@$pcidevices) {
4569                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4570
4571                 my $info = pci_device_info("0000:$pciid");
4572                 die "IOMMU not present\n" if !check_iommu_support();
4573                 die "no pci device info for device '$pciid'\n" if !$info;
4574                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4575                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4576           }
4577         }
4578
4579         PVE::Storage::activate_volumes($storecfg, $vollist);
4580
4581         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4582                     umask => 0077); };
4583
4584         if (my $err = $@) {
4585             # deactivate volumes if start fails
4586             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4587             die "start failed: $err";
4588         }
4589
4590         print "migration listens on $migrate_uri\n" if $migrate_uri;
4591
4592         if ($statefile && $statefile ne 'tcp')  {
4593             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4594             warn $@ if $@;
4595         }
4596
4597         if ($migratedfrom) {
4598
4599             eval {
4600                 set_migration_caps($vmid);
4601             };
4602             warn $@ if $@;
4603
4604             if ($spice_port) {
4605                 print "spice listens on port $spice_port\n";
4606                 if ($spice_ticket) {
4607                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4608                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4609                 }
4610             }
4611
4612         } else {
4613
4614             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4615                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4616                     if $conf->{balloon};
4617             }
4618
4619             foreach my $opt (keys %$conf) {
4620                 next if $opt !~  m/^net\d+$/;
4621                 my $nicconf = parse_net($conf->{$opt});
4622                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4623             }
4624         }
4625
4626         vm_mon_cmd_nocheck($vmid, 'qom-set',
4627                     path => "machine/peripheral/balloon0",
4628                     property => "guest-stats-polling-interval",
4629                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4630
4631     });
4632 }
4633
4634 sub vm_mon_cmd {
4635     my ($vmid, $execute, %params) = @_;
4636
4637     my $cmd = { execute => $execute, arguments => \%params };
4638     vm_qmp_command($vmid, $cmd);
4639 }
4640
4641 sub vm_mon_cmd_nocheck {
4642     my ($vmid, $execute, %params) = @_;
4643
4644     my $cmd = { execute => $execute, arguments => \%params };
4645     vm_qmp_command($vmid, $cmd, 1);
4646 }
4647
4648 sub vm_qmp_command {
4649     my ($vmid, $cmd, $nocheck) = @_;
4650
4651     my $res;
4652
4653     my $timeout;
4654     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4655         $timeout = $cmd->{arguments}->{timeout};
4656         delete $cmd->{arguments}->{timeout};
4657     }
4658
4659     eval {
4660         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4661         my $sname = qmp_socket($vmid);
4662         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4663             my $qmpclient = PVE::QMPClient->new();
4664
4665             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4666         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4667             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4668                 if scalar(%{$cmd->{arguments}});
4669             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4670         } else {
4671             die "unable to open monitor socket\n";
4672         }
4673     };
4674     if (my $err = $@) {
4675         syslog("err", "VM $vmid qmp command failed - $err");
4676         die $err;
4677     }
4678
4679     return $res;
4680 }
4681
4682 sub vm_human_monitor_command {
4683     my ($vmid, $cmdline) = @_;
4684
4685     my $res;
4686
4687     my $cmd = {
4688         execute => 'human-monitor-command',
4689         arguments => { 'command-line' => $cmdline},
4690     };
4691
4692     return vm_qmp_command($vmid, $cmd);
4693 }
4694
4695 sub vm_commandline {
4696     my ($storecfg, $vmid) = @_;
4697
4698     my $conf = load_config($vmid);
4699
4700     my $defaults = load_defaults();
4701
4702     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4703
4704     return join(' ', @$cmd);
4705 }
4706
4707 sub vm_reset {
4708     my ($vmid, $skiplock) = @_;
4709
4710     lock_config($vmid, sub {
4711
4712         my $conf = load_config($vmid);
4713
4714         check_lock($conf) if !$skiplock;
4715
4716         vm_mon_cmd($vmid, "system_reset");
4717     });
4718 }
4719
4720 sub get_vm_volumes {
4721     my ($conf) = @_;
4722
4723     my $vollist = [];
4724     foreach_volid($conf, sub {
4725         my ($volid, $is_cdrom) = @_;
4726
4727         return if $volid =~ m|^/|;
4728
4729         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4730         return if !$sid;
4731
4732         push @$vollist, $volid;
4733     });
4734
4735     return $vollist;
4736 }
4737
4738 sub vm_stop_cleanup {
4739     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4740
4741     eval {
4742
4743         if (!$keepActive) {
4744             my $vollist = get_vm_volumes($conf);
4745             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4746         }
4747
4748         foreach my $ext (qw(mon qmp pid vnc qga)) {
4749             unlink "/var/run/qemu-server/${vmid}.$ext";
4750         }
4751
4752         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4753     };
4754     warn $@ if $@; # avoid errors - just warn
4755 }
4756
4757 # Note: use $nockeck to skip tests if VM configuration file exists.
4758 # We need that when migration VMs to other nodes (files already moved)
4759 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4760 sub vm_stop {
4761     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4762
4763     $force = 1 if !defined($force) && !$shutdown;
4764
4765     if ($migratedfrom){
4766         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4767         kill 15, $pid if $pid;
4768         my $conf = load_config($vmid, $migratedfrom);
4769         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4770         return;
4771     }
4772
4773     lock_config($vmid, sub {
4774
4775         my $pid = check_running($vmid, $nocheck);
4776         return if !$pid;
4777
4778         my $conf;
4779         if (!$nocheck) {
4780             $conf = load_config($vmid);
4781             check_lock($conf) if !$skiplock;
4782             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4783                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4784                 $timeout = $opts->{down} if $opts->{down};
4785             }
4786         }
4787
4788         $timeout = 60 if !defined($timeout);
4789
4790         eval {
4791             if ($shutdown) {
4792                 if (defined($conf) && $conf->{agent}) {
4793                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4794                 } else {
4795                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4796                 }
4797             } else {
4798                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4799             }
4800         };
4801         my $err = $@;
4802
4803         if (!$err) {
4804             my $count = 0;
4805             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4806                 $count++;
4807                 sleep 1;
4808             }
4809
4810             if ($count >= $timeout) {
4811                 if ($force) {
4812                     warn "VM still running - terminating now with SIGTERM\n";
4813                     kill 15, $pid;
4814                 } else {
4815                     die "VM quit/powerdown failed - got timeout\n";
4816                 }
4817             } else {
4818                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4819                 return;
4820             }
4821         } else {
4822             if ($force) {
4823                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4824                 kill 15, $pid;
4825             } else {
4826                 die "VM quit/powerdown failed\n";
4827             }
4828         }
4829
4830         # wait again
4831         $timeout = 10;
4832
4833         my $count = 0;
4834         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4835             $count++;
4836             sleep 1;
4837         }
4838
4839         if ($count >= $timeout) {
4840             warn "VM still running - terminating now with SIGKILL\n";
4841             kill 9, $pid;
4842             sleep 1;
4843         }
4844
4845         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4846    });
4847 }
4848
4849 sub vm_suspend {
4850     my ($vmid, $skiplock) = @_;
4851
4852     lock_config($vmid, sub {
4853
4854         my $conf = load_config($vmid);
4855
4856         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4857
4858         vm_mon_cmd($vmid, "stop");
4859     });
4860 }
4861
4862 sub vm_resume {
4863     my ($vmid, $skiplock, $nocheck) = @_;
4864
4865     lock_config($vmid, sub {
4866
4867         if (!$nocheck) {
4868
4869             my $conf = load_config($vmid);
4870
4871             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4872
4873             vm_mon_cmd($vmid, "cont");
4874
4875         } else {
4876             vm_mon_cmd_nocheck($vmid, "cont");
4877         }
4878     });
4879 }
4880
4881 sub vm_sendkey {
4882     my ($vmid, $skiplock, $key) = @_;
4883
4884     lock_config($vmid, sub {
4885
4886         my $conf = load_config($vmid);
4887
4888         # there is no qmp command, so we use the human monitor command
4889         vm_human_monitor_command($vmid, "sendkey $key");
4890     });
4891 }
4892
4893 sub vm_destroy {
4894     my ($storecfg, $vmid, $skiplock) = @_;
4895
4896     lock_config($vmid, sub {
4897
4898         my $conf = load_config($vmid);
4899
4900         if (!check_running($vmid)) {
4901             destroy_vm($storecfg, $vmid, undef, $skiplock);
4902         } else {
4903             die "VM $vmid is running - destroy failed\n";
4904         }
4905     });
4906 }
4907
4908 # pci helpers
4909
4910 sub file_write {
4911     my ($filename, $buf) = @_;
4912
4913     my $fh = IO::File->new($filename, "w");
4914     return undef if !$fh;
4915
4916     my $res = print $fh $buf;
4917
4918     $fh->close();
4919
4920     return $res;
4921 }
4922
4923 sub pci_device_info {
4924     my ($name) = @_;
4925
4926     my $res;
4927
4928     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4929     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4930
4931     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4932     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4933
4934     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4935     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4936
4937     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4938     return undef if !defined($product) || $product !~ s/^0x//;
4939
4940     $res = {
4941         name => $name,
4942         vendor => $vendor,
4943         product => $product,
4944         domain => $domain,
4945         bus => $bus,
4946         slot => $slot,
4947         func => $func,
4948         irq => $irq,
4949         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4950     };
4951
4952     return $res;
4953 }
4954
4955 sub pci_dev_reset {
4956     my ($dev) = @_;
4957
4958     my $name = $dev->{name};
4959
4960     my $fn = "$pcisysfs/devices/$name/reset";
4961
4962     return file_write($fn, "1");
4963 }
4964
4965 sub pci_dev_bind_to_vfio {
4966     my ($dev) = @_;
4967
4968     my $name = $dev->{name};
4969
4970     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4971
4972     if (!-d $vfio_basedir) {
4973         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4974     }
4975     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4976
4977     my $testdir = "$vfio_basedir/$name";
4978     return 1 if -d $testdir;
4979
4980     my $data = "$dev->{vendor} $dev->{product}";
4981     return undef if !file_write("$vfio_basedir/new_id", $data);
4982
4983     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4984     if (!file_write($fn, $name)) {
4985         return undef if -f $fn;
4986     }
4987
4988     $fn = "$vfio_basedir/bind";
4989     if (! -d $testdir) {
4990         return undef if !file_write($fn, $name);
4991     }
4992
4993     return -d $testdir;
4994 }
4995
4996 sub pci_dev_group_bind_to_vfio {
4997     my ($pciid) = @_;
4998
4999     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5000
5001     if (!-d $vfio_basedir) {
5002         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5003     }
5004     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5005
5006     # get IOMMU group devices
5007     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5008       my @devs = grep /^0000:/, readdir($D);
5009     closedir($D);
5010
5011     foreach my $pciid (@devs) {
5012         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5013
5014         # pci bridges, switches or root ports are not supported
5015         # they have a pci_bus subdirectory so skip them
5016         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5017
5018         my $info = pci_device_info($1);
5019         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5020     }
5021
5022     return 1;
5023 }
5024
5025 sub print_pci_addr {
5026     my ($id, $bridges) = @_;
5027
5028     my $res = '';
5029     my $devices = {
5030         piix3 => { bus => 0, addr => 1 },
5031         #addr2 : first videocard
5032         balloon0 => { bus => 0, addr => 3 },
5033         watchdog => { bus => 0, addr => 4 },
5034         scsihw0 => { bus => 0, addr => 5 },
5035         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5036         scsihw1 => { bus => 0, addr => 6 },
5037         ahci0 => { bus => 0, addr => 7 },
5038         qga0 => { bus => 0, addr => 8 },
5039         spice => { bus => 0, addr => 9 },
5040         virtio0 => { bus => 0, addr => 10 },
5041         virtio1 => { bus => 0, addr => 11 },
5042         virtio2 => { bus => 0, addr => 12 },
5043         virtio3 => { bus => 0, addr => 13 },
5044         virtio4 => { bus => 0, addr => 14 },
5045         virtio5 => { bus => 0, addr => 15 },
5046         hostpci0 => { bus => 0, addr => 16 },
5047         hostpci1 => { bus => 0, addr => 17 },
5048         net0 => { bus => 0, addr => 18 },
5049         net1 => { bus => 0, addr => 19 },
5050         net2 => { bus => 0, addr => 20 },
5051         net3 => { bus => 0, addr => 21 },
5052         net4 => { bus => 0, addr => 22 },
5053         net5 => { bus => 0, addr => 23 },
5054         vga1 => { bus => 0, addr => 24 },
5055         vga2 => { bus => 0, addr => 25 },
5056         vga3 => { bus => 0, addr => 26 },
5057         hostpci2 => { bus => 0, addr => 27 },
5058         hostpci3 => { bus => 0, addr => 28 },
5059         #addr29 : usb-host (pve-usb.cfg)
5060         'pci.1' => { bus => 0, addr => 30 },
5061         'pci.2' => { bus => 0, addr => 31 },
5062         'net6' => { bus => 1, addr => 1 },
5063         'net7' => { bus => 1, addr => 2 },
5064         'net8' => { bus => 1, addr => 3 },
5065         'net9' => { bus => 1, addr => 4 },
5066         'net10' => { bus => 1, addr => 5 },
5067         'net11' => { bus => 1, addr => 6 },
5068         'net12' => { bus => 1, addr => 7 },
5069         'net13' => { bus => 1, addr => 8 },
5070         'net14' => { bus => 1, addr => 9 },
5071         'net15' => { bus => 1, addr => 10 },
5072         'net16' => { bus => 1, addr => 11 },
5073         'net17' => { bus => 1, addr => 12 },
5074         'net18' => { bus => 1, addr => 13 },
5075         'net19' => { bus => 1, addr => 14 },
5076         'net20' => { bus => 1, addr => 15 },
5077         'net21' => { bus => 1, addr => 16 },
5078         'net22' => { bus => 1, addr => 17 },
5079         'net23' => { bus => 1, addr => 18 },
5080         'net24' => { bus => 1, addr => 19 },
5081         'net25' => { bus => 1, addr => 20 },
5082         'net26' => { bus => 1, addr => 21 },
5083         'net27' => { bus => 1, addr => 22 },
5084         'net28' => { bus => 1, addr => 23 },
5085         'net29' => { bus => 1, addr => 24 },
5086         'net30' => { bus => 1, addr => 25 },
5087         'net31' => { bus => 1, addr => 26 },
5088         'xhci' => { bus => 1, addr => 27 },
5089         'virtio6' => { bus => 2, addr => 1 },
5090         'virtio7' => { bus => 2, addr => 2 },
5091         'virtio8' => { bus => 2, addr => 3 },
5092         'virtio9' => { bus => 2, addr => 4 },
5093         'virtio10' => { bus => 2, addr => 5 },
5094         'virtio11' => { bus => 2, addr => 6 },
5095         'virtio12' => { bus => 2, addr => 7 },
5096         'virtio13' => { bus => 2, addr => 8 },
5097         'virtio14' => { bus => 2, addr => 9 },
5098         'virtio15' => { bus => 2, addr => 10 },
5099         'virtioscsi0' => { bus => 3, addr => 1 },
5100         'virtioscsi1' => { bus => 3, addr => 2 },
5101         'virtioscsi2' => { bus => 3, addr => 3 },
5102         'virtioscsi3' => { bus => 3, addr => 4 },
5103         'virtioscsi4' => { bus => 3, addr => 5 },
5104         'virtioscsi5' => { bus => 3, addr => 6 },
5105         'virtioscsi6' => { bus => 3, addr => 7 },
5106         'virtioscsi7' => { bus => 3, addr => 8 },
5107         'virtioscsi8' => { bus => 3, addr => 9 },
5108         'virtioscsi9' => { bus => 3, addr => 10 },
5109         'virtioscsi10' => { bus => 3, addr => 11 },
5110         'virtioscsi11' => { bus => 3, addr => 12 },
5111         'virtioscsi12' => { bus => 3, addr => 13 },
5112         'virtioscsi13' => { bus => 3, addr => 14 },
5113         'virtioscsi14' => { bus => 3, addr => 15 },
5114         'virtioscsi15' => { bus => 3, addr => 16 },
5115         'virtioscsi16' => { bus => 3, addr => 17 },
5116         'virtioscsi17' => { bus => 3, addr => 18 },
5117         'virtioscsi18' => { bus => 3, addr => 19 },
5118         'virtioscsi19' => { bus => 3, addr => 20 },
5119         'virtioscsi20' => { bus => 3, addr => 21 },
5120         'virtioscsi21' => { bus => 3, addr => 22 },
5121         'virtioscsi22' => { bus => 3, addr => 23 },
5122         'virtioscsi23' => { bus => 3, addr => 24 },
5123         'virtioscsi24' => { bus => 3, addr => 25 },
5124         'virtioscsi25' => { bus => 3, addr => 26 },
5125         'virtioscsi26' => { bus => 3, addr => 27 },
5126         'virtioscsi27' => { bus => 3, addr => 28 },
5127         'virtioscsi28' => { bus => 3, addr => 29 },
5128         'virtioscsi29' => { bus => 3, addr => 30 },
5129         'virtioscsi30' => { bus => 3, addr => 31 },
5130
5131     };
5132
5133     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5134            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5135            my $bus = $devices->{$id}->{bus};
5136            $res = ",bus=pci.$bus,addr=$addr";
5137            $bridges->{$bus} = 1 if $bridges;
5138     }
5139     return $res;
5140
5141 }
5142
5143 sub print_pcie_addr {
5144     my ($id) = @_;
5145
5146     my $res = '';
5147     my $devices = {
5148         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5149         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5150         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5151         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5152     };
5153
5154     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5155            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5156            my $bus = $devices->{$id}->{bus};
5157            $res = ",bus=$bus,addr=$addr";
5158     }
5159     return $res;
5160
5161 }
5162
5163 # vzdump restore implementaion
5164
5165 sub tar_archive_read_firstfile {
5166     my $archive = shift;
5167
5168     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5169
5170     # try to detect archive type first
5171     my $pid = open (TMP, "tar tf '$archive'|") ||
5172         die "unable to open file '$archive'\n";
5173     my $firstfile = <TMP>;
5174     kill 15, $pid;
5175     close TMP;
5176
5177     die "ERROR: archive contaions no data\n" if !$firstfile;
5178     chomp $firstfile;
5179
5180     return $firstfile;
5181 }
5182
5183 sub tar_restore_cleanup {
5184     my ($storecfg, $statfile) = @_;
5185
5186     print STDERR "starting cleanup\n";
5187
5188     if (my $fd = IO::File->new($statfile, "r")) {
5189         while (defined(my $line = <$fd>)) {
5190             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5191                 my $volid = $2;
5192                 eval {
5193                     if ($volid =~ m|^/|) {
5194                         unlink $volid || die 'unlink failed\n';
5195                     } else {
5196                         PVE::Storage::vdisk_free($storecfg, $volid);
5197                     }
5198                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5199                 };
5200                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5201             } else {
5202                 print STDERR "unable to parse line in statfile - $line";
5203             }
5204         }
5205         $fd->close();
5206     }
5207 }
5208
5209 sub restore_archive {
5210     my ($archive, $vmid, $user, $opts) = @_;
5211
5212     my $format = $opts->{format};
5213     my $comp;
5214
5215     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5216         $format = 'tar' if !$format;
5217         $comp = 'gzip';
5218     } elsif ($archive =~ m/\.tar$/) {
5219         $format = 'tar' if !$format;
5220     } elsif ($archive =~ m/.tar.lzo$/) {
5221         $format = 'tar' if !$format;
5222         $comp = 'lzop';
5223     } elsif ($archive =~ m/\.vma$/) {
5224         $format = 'vma' if !$format;
5225     } elsif ($archive =~ m/\.vma\.gz$/) {
5226         $format = 'vma' if !$format;
5227         $comp = 'gzip';
5228     } elsif ($archive =~ m/\.vma\.lzo$/) {
5229         $format = 'vma' if !$format;
5230         $comp = 'lzop';
5231     } else {
5232         $format = 'vma' if !$format; # default
5233     }
5234
5235     # try to detect archive format
5236     if ($format eq 'tar') {
5237         return restore_tar_archive($archive, $vmid, $user, $opts);
5238     } else {
5239         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5240     }
5241 }
5242
5243 sub restore_update_config_line {
5244     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5245
5246     return if $line =~ m/^\#qmdump\#/;
5247     return if $line =~ m/^\#vzdump\#/;
5248     return if $line =~ m/^lock:/;
5249     return if $line =~ m/^unused\d+:/;
5250     return if $line =~ m/^parent:/;
5251     return if $line =~ m/^template:/; # restored VM is never a template
5252
5253     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5254         # try to convert old 1.X settings
5255         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5256         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5257             my ($model, $macaddr) = split(/\=/, $devconfig);
5258             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5259             my $net = {
5260                 model => $model,
5261                 bridge => "vmbr$ind",
5262                 macaddr => $macaddr,
5263             };
5264             my $netstr = print_net($net);
5265
5266             print $outfd "net$cookie->{netcount}: $netstr\n";
5267             $cookie->{netcount}++;
5268         }
5269     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5270         my ($id, $netstr) = ($1, $2);
5271         my $net = parse_net($netstr);
5272         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5273         $netstr = print_net($net);
5274         print $outfd "$id: $netstr\n";
5275     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5276         my $virtdev = $1;
5277         my $value = $3;
5278         my $di = parse_drive($virtdev, $value);
5279         if (defined($di->{backup}) && !$di->{backup}) {
5280             print $outfd "#$line";
5281         } elsif ($map->{$virtdev}) {
5282             delete $di->{format}; # format can change on restore
5283             $di->{file} = $map->{$virtdev};
5284             $value = print_drive($vmid, $di);
5285             print $outfd "$virtdev: $value\n";
5286         } else {
5287             print $outfd $line;
5288         }
5289     } else {
5290         print $outfd $line;
5291     }
5292 }
5293
5294 sub scan_volids {
5295     my ($cfg, $vmid) = @_;
5296
5297     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5298
5299     my $volid_hash = {};
5300     foreach my $storeid (keys %$info) {
5301         foreach my $item (@{$info->{$storeid}}) {
5302             next if !($item->{volid} && $item->{size});
5303             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5304             $volid_hash->{$item->{volid}} = $item;
5305         }
5306     }
5307
5308     return $volid_hash;
5309 }
5310
5311 sub is_volume_in_use {
5312     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5313
5314     my $path = PVE::Storage::path($storecfg, $volid);
5315
5316     my $scan_config = sub {
5317         my ($cref, $snapname) = @_;
5318
5319         foreach my $key (keys %$cref) {
5320             my $value = $cref->{$key};
5321             if (valid_drivename($key)) {
5322                 next if $skip_drive && $key eq $skip_drive;
5323                 my $drive = parse_drive($key, $value);
5324                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5325                 return 1 if $volid eq $drive->{file};
5326                 if ($drive->{file} =~ m!^/!) {
5327                     return 1 if $drive->{file} eq $path;
5328                 } else {
5329                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5330                     next if !$storeid;
5331                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5332                     next if !$scfg;
5333                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5334                 }
5335             }
5336         }
5337
5338         return 0;
5339     };
5340
5341     return 1 if &$scan_config($conf);
5342
5343     undef $skip_drive;
5344
5345     foreach my $snapname (keys %{$conf->{snapshots}}) {
5346         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5347     }
5348
5349     return 0;
5350 }
5351
5352 sub update_disksize {
5353     my ($vmid, $conf, $volid_hash) = @_;
5354
5355     my $changes;
5356
5357     my $used = {};
5358
5359     # Note: it is allowed to define multiple storages with same path (alias), so
5360     # we need to check both 'volid' and real 'path' (two different volid can point
5361     # to the same path).
5362
5363     my $usedpath = {};
5364
5365     # update size info
5366     foreach my $opt (keys %$conf) {
5367         if (valid_drivename($opt)) {
5368             my $drive = parse_drive($opt, $conf->{$opt});
5369             my $volid = $drive->{file};
5370             next if !$volid;
5371
5372             $used->{$volid} = 1;
5373             if ($volid_hash->{$volid} &&
5374                 (my $path = $volid_hash->{$volid}->{path})) {
5375                 $usedpath->{$path} = 1;
5376             }
5377
5378             next if drive_is_cdrom($drive);
5379             next if !$volid_hash->{$volid};
5380
5381             $drive->{size} = $volid_hash->{$volid}->{size};
5382             my $new = print_drive($vmid, $drive);
5383             if ($new ne $conf->{$opt}) {
5384                 $changes = 1;
5385                 $conf->{$opt} = $new;
5386             }
5387         }
5388     }
5389
5390     # remove 'unusedX' entry if volume is used
5391     foreach my $opt (keys %$conf) {
5392         next if $opt !~ m/^unused\d+$/;
5393         my $volid = $conf->{$opt};
5394         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5395         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5396             $changes = 1;
5397             delete $conf->{$opt};
5398         }
5399     }
5400
5401     foreach my $volid (sort keys %$volid_hash) {
5402         next if $volid =~ m/vm-$vmid-state-/;
5403         next if $used->{$volid};
5404         my $path = $volid_hash->{$volid}->{path};
5405         next if !$path; # just to be sure
5406         next if $usedpath->{$path};
5407         $changes = 1;
5408         add_unused_volume($conf, $volid);
5409         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5410     }
5411
5412     return $changes;
5413 }
5414
5415 sub rescan {
5416     my ($vmid, $nolock) = @_;
5417
5418     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5419
5420     my $volid_hash = scan_volids($cfg, $vmid);
5421
5422     my $updatefn =  sub {
5423         my ($vmid) = @_;
5424
5425         my $conf = load_config($vmid);
5426
5427         check_lock($conf);
5428
5429         my $vm_volids = {};
5430         foreach my $volid (keys %$volid_hash) {
5431             my $info = $volid_hash->{$volid};
5432             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5433         }
5434
5435         my $changes = update_disksize($vmid, $conf, $vm_volids);
5436
5437         write_config($vmid, $conf) if $changes;
5438     };
5439
5440     if (defined($vmid)) {
5441         if ($nolock) {
5442             &$updatefn($vmid);
5443         } else {
5444             lock_config($vmid, $updatefn, $vmid);
5445         }
5446     } else {
5447         my $vmlist = config_list();
5448         foreach my $vmid (keys %$vmlist) {
5449             if ($nolock) {
5450                 &$updatefn($vmid);
5451             } else {
5452                 lock_config($vmid, $updatefn, $vmid);
5453             }
5454         }
5455     }
5456 }
5457
5458 sub restore_vma_archive {
5459     my ($archive, $vmid, $user, $opts, $comp) = @_;
5460
5461     my $input = $archive eq '-' ? "<&STDIN" : undef;
5462     my $readfrom = $archive;
5463
5464     my $uncomp = '';
5465     if ($comp) {
5466         $readfrom = '-';
5467         my $qarchive = PVE::Tools::shellquote($archive);
5468         if ($comp eq 'gzip') {
5469             $uncomp = "zcat $qarchive|";
5470         } elsif ($comp eq 'lzop') {
5471             $uncomp = "lzop -d -c $qarchive|";
5472         } else {
5473             die "unknown compression method '$comp'\n";
5474         }
5475
5476     }
5477
5478     my $tmpdir = "/var/tmp/vzdumptmp$$";
5479     rmtree $tmpdir;
5480
5481     # disable interrupts (always do cleanups)
5482     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5483         warn "got interrupt - ignored\n";
5484     };
5485
5486     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5487     POSIX::mkfifo($mapfifo, 0600);
5488     my $fifofh;
5489
5490     my $openfifo = sub {
5491         open($fifofh, '>', $mapfifo) || die $!;
5492     };
5493
5494     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5495
5496     my $oldtimeout;
5497     my $timeout = 5;
5498
5499     my $devinfo = {};
5500
5501     my $rpcenv = PVE::RPCEnvironment::get();
5502
5503     my $conffile = config_file($vmid);
5504     my $tmpfn = "$conffile.$$.tmp";
5505
5506     # Note: $oldconf is undef if VM does not exists
5507     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5508
5509     my $print_devmap = sub {
5510         my $virtdev_hash = {};
5511
5512         my $cfgfn = "$tmpdir/qemu-server.conf";
5513
5514         # we can read the config - that is already extracted
5515         my $fh = IO::File->new($cfgfn, "r") ||
5516             "unable to read qemu-server.conf - $!\n";
5517
5518         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5519         if (-f $fwcfgfn) {
5520             my $pve_firewall_dir = '/etc/pve/firewall';
5521             mkdir $pve_firewall_dir; # make sure the dir exists
5522             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5523         }
5524
5525         while (defined(my $line = <$fh>)) {
5526             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5527                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5528                 die "archive does not contain data for drive '$virtdev'\n"
5529                     if !$devinfo->{$devname};
5530                 if (defined($opts->{storage})) {
5531                     $storeid = $opts->{storage} || 'local';
5532                 } elsif (!$storeid) {
5533                     $storeid = 'local';
5534                 }
5535                 $format = 'raw' if !$format;
5536                 $devinfo->{$devname}->{devname} = $devname;
5537                 $devinfo->{$devname}->{virtdev} = $virtdev;
5538                 $devinfo->{$devname}->{format} = $format;
5539                 $devinfo->{$devname}->{storeid} = $storeid;
5540
5541                 # check permission on storage
5542                 my $pool = $opts->{pool}; # todo: do we need that?
5543                 if ($user ne 'root@pam') {
5544                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5545                 }
5546
5547                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5548             }
5549         }
5550
5551         foreach my $devname (keys %$devinfo) {
5552             die "found no device mapping information for device '$devname'\n"
5553                 if !$devinfo->{$devname}->{virtdev};
5554         }
5555
5556         my $cfg = cfs_read_file('storage.cfg');
5557
5558         # create empty/temp config
5559         if ($oldconf) {
5560             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5561             foreach_drive($oldconf, sub {
5562                 my ($ds, $drive) = @_;
5563
5564                 return if drive_is_cdrom($drive);
5565
5566                 my $volid = $drive->{file};
5567
5568                 return if !$volid || $volid =~ m|^/|;
5569
5570                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5571                 return if !$path || !$owner || ($owner != $vmid);
5572
5573                 # Note: only delete disk we want to restore
5574                 # other volumes will become unused
5575                 if ($virtdev_hash->{$ds}) {
5576                     PVE::Storage::vdisk_free($cfg, $volid);
5577                 }
5578             });
5579
5580             # delete vmstate files
5581             # since after the restore we have no snapshots anymore
5582             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5583                 my $snap = $oldconf->{snapshots}->{$snapname};
5584                 if ($snap->{vmstate}) {
5585                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5586                     if (my $err = $@) {
5587                         warn $err;
5588                     }
5589                 }
5590             }
5591         }
5592
5593         my $map = {};
5594         foreach my $virtdev (sort keys %$virtdev_hash) {
5595             my $d = $virtdev_hash->{$virtdev};
5596             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5597             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5598
5599             # test if requested format is supported
5600             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5601             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5602             $d->{format} = $defFormat if !$supported;
5603
5604             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5605                                                   $d->{format}, undef, $alloc_size);
5606             print STDERR "new volume ID is '$volid'\n";
5607             $d->{volid} = $volid;
5608             my $path = PVE::Storage::path($cfg, $volid);
5609
5610             PVE::Storage::activate_volumes($cfg,[$volid]);
5611
5612             my $write_zeros = 1;
5613             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5614                 $write_zeros = 0;
5615             }
5616
5617             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5618
5619             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5620             $map->{$virtdev} = $volid;
5621         }
5622
5623         $fh->seek(0, 0) || die "seek failed - $!\n";
5624
5625         my $outfd = new IO::File ($tmpfn, "w") ||
5626             die "unable to write config for VM $vmid\n";
5627
5628         my $cookie = { netcount => 0 };
5629         while (defined(my $line = <$fh>)) {
5630             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5631         }
5632
5633         $fh->close();
5634         $outfd->close();
5635     };
5636
5637     eval {
5638         # enable interrupts
5639         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5640             die "interrupted by signal\n";
5641         };
5642         local $SIG{ALRM} = sub { die "got timeout\n"; };
5643
5644         $oldtimeout = alarm($timeout);
5645
5646         my $parser = sub {
5647             my $line = shift;
5648
5649             print "$line\n";
5650
5651             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5652                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5653                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5654             } elsif ($line =~ m/^CTIME: /) {
5655                 # we correctly received the vma config, so we can disable
5656                 # the timeout now for disk allocation (set to 10 minutes, so
5657                 # that we always timeout if something goes wrong)
5658                 alarm(600);
5659                 &$print_devmap();
5660                 print $fifofh "done\n";
5661                 my $tmp = $oldtimeout || 0;
5662                 $oldtimeout = undef;
5663                 alarm($tmp);
5664                 close($fifofh);
5665             }
5666         };
5667
5668         print "restore vma archive: $cmd\n";
5669         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5670     };
5671     my $err = $@;
5672
5673     alarm($oldtimeout) if $oldtimeout;
5674
5675     my $vollist = [];
5676     foreach my $devname (keys %$devinfo) {
5677         my $volid = $devinfo->{$devname}->{volid};
5678         push @$vollist, $volid if $volid;
5679     }
5680
5681     my $cfg = cfs_read_file('storage.cfg');
5682     PVE::Storage::deactivate_volumes($cfg, $vollist);
5683
5684     unlink $mapfifo;
5685
5686     if ($err) {
5687         rmtree $tmpdir;
5688         unlink $tmpfn;
5689
5690         foreach my $devname (keys %$devinfo) {
5691             my $volid = $devinfo->{$devname}->{volid};
5692             next if !$volid;
5693             eval {
5694                 if ($volid =~ m|^/|) {
5695                     unlink $volid || die 'unlink failed\n';
5696                 } else {
5697                     PVE::Storage::vdisk_free($cfg, $volid);
5698                 }
5699                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5700             };
5701             print STDERR "unable to cleanup '$volid' - $@" if $@;
5702         }
5703         die $err;
5704     }
5705
5706     rmtree $tmpdir;
5707
5708     rename($tmpfn, $conffile) ||
5709         die "unable to commit configuration file '$conffile'\n";
5710
5711     PVE::Cluster::cfs_update(); # make sure we read new file
5712
5713     eval { rescan($vmid, 1); };
5714     warn $@ if $@;
5715 }
5716
5717 sub restore_tar_archive {
5718     my ($archive, $vmid, $user, $opts) = @_;
5719
5720     if ($archive ne '-') {
5721         my $firstfile = tar_archive_read_firstfile($archive);
5722         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5723             if $firstfile ne 'qemu-server.conf';
5724     }
5725
5726     my $storecfg = cfs_read_file('storage.cfg');
5727
5728     # destroy existing data - keep empty config
5729     my $vmcfgfn = config_file($vmid);
5730     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5731
5732     my $tocmd = "/usr/lib/qemu-server/qmextract";
5733
5734     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5735     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5736     $tocmd .= ' --prealloc' if $opts->{prealloc};
5737     $tocmd .= ' --info' if $opts->{info};
5738
5739     # tar option "xf" does not autodetect compression when read from STDIN,
5740     # so we pipe to zcat
5741     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5742         PVE::Tools::shellquote("--to-command=$tocmd");
5743
5744     my $tmpdir = "/var/tmp/vzdumptmp$$";
5745     mkpath $tmpdir;
5746
5747     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5748     local $ENV{VZDUMP_VMID} = $vmid;
5749     local $ENV{VZDUMP_USER} = $user;
5750
5751     my $conffile = config_file($vmid);
5752     my $tmpfn = "$conffile.$$.tmp";
5753
5754     # disable interrupts (always do cleanups)
5755     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5756         print STDERR "got interrupt - ignored\n";
5757     };
5758
5759     eval {
5760         # enable interrupts
5761         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5762             die "interrupted by signal\n";
5763         };
5764
5765         if ($archive eq '-') {
5766             print "extracting archive from STDIN\n";
5767             run_command($cmd, input => "<&STDIN");
5768         } else {
5769             print "extracting archive '$archive'\n";
5770             run_command($cmd);
5771         }
5772
5773         return if $opts->{info};
5774
5775         # read new mapping
5776         my $map = {};
5777         my $statfile = "$tmpdir/qmrestore.stat";
5778         if (my $fd = IO::File->new($statfile, "r")) {
5779             while (defined (my $line = <$fd>)) {
5780                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5781                     $map->{$1} = $2 if $1;
5782                 } else {
5783                     print STDERR "unable to parse line in statfile - $line\n";
5784                 }
5785             }
5786             $fd->close();
5787         }
5788
5789         my $confsrc = "$tmpdir/qemu-server.conf";
5790
5791         my $srcfd = new IO::File($confsrc, "r") ||
5792             die "unable to open file '$confsrc'\n";
5793
5794         my $outfd = new IO::File ($tmpfn, "w") ||
5795             die "unable to write config for VM $vmid\n";
5796
5797         my $cookie = { netcount => 0 };
5798         while (defined (my $line = <$srcfd>)) {
5799             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5800         }
5801
5802         $srcfd->close();
5803         $outfd->close();
5804     };
5805     my $err = $@;
5806
5807     if ($err) {
5808
5809         unlink $tmpfn;
5810
5811         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5812
5813         die $err;
5814     }
5815
5816     rmtree $tmpdir;
5817
5818     rename $tmpfn, $conffile ||
5819         die "unable to commit configuration file '$conffile'\n";
5820
5821     PVE::Cluster::cfs_update(); # make sure we read new file
5822
5823     eval { rescan($vmid, 1); };
5824     warn $@ if $@;
5825 };
5826
5827
5828 # Internal snapshots
5829
5830 # NOTE: Snapshot create/delete involves several non-atomic
5831 # action, and can take a long time.
5832 # So we try to avoid locking the file and use 'lock' variable
5833 # inside the config file instead.
5834
5835 my $snapshot_copy_config = sub {
5836     my ($source, $dest) = @_;
5837
5838     foreach my $k (keys %$source) {
5839         next if $k eq 'snapshots';
5840         next if $k eq 'snapstate';
5841         next if $k eq 'snaptime';
5842         next if $k eq 'vmstate';
5843         next if $k eq 'lock';
5844         next if $k eq 'digest';
5845         next if $k eq 'description';
5846         next if $k =~ m/^unused\d+$/;
5847
5848         $dest->{$k} = $source->{$k};
5849     }
5850 };
5851
5852 my $snapshot_apply_config = sub {
5853     my ($conf, $snap) = @_;
5854
5855     # copy snapshot list
5856     my $newconf = {
5857         snapshots => $conf->{snapshots},
5858     };
5859
5860     # keep description and list of unused disks
5861     foreach my $k (keys %$conf) {
5862         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5863         $newconf->{$k} = $conf->{$k};
5864     }
5865
5866     &$snapshot_copy_config($snap, $newconf);
5867
5868     return $newconf;
5869 };
5870
5871 sub foreach_writable_storage {
5872     my ($conf, $func) = @_;
5873
5874     my $sidhash = {};
5875
5876     foreach my $ds (keys %$conf) {
5877         next if !valid_drivename($ds);
5878
5879         my $drive = parse_drive($ds, $conf->{$ds});
5880         next if !$drive;
5881         next if drive_is_cdrom($drive);
5882
5883         my $volid = $drive->{file};
5884
5885         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5886         $sidhash->{$sid} = $sid if $sid;
5887     }
5888
5889     foreach my $sid (sort keys %$sidhash) {
5890         &$func($sid);
5891     }
5892 }
5893
5894 my $alloc_vmstate_volid = sub {
5895     my ($storecfg, $vmid, $conf, $snapname) = @_;
5896
5897     # Note: we try to be smart when selecting a $target storage
5898
5899     my $target;
5900
5901     # search shared storage first
5902     foreach_writable_storage($conf, sub {
5903         my ($sid) = @_;
5904         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5905         return if !$scfg->{shared};
5906
5907         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5908     });
5909
5910     if (!$target) {
5911         # now search local storage
5912         foreach_writable_storage($conf, sub {
5913             my ($sid) = @_;
5914             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5915             return if $scfg->{shared};
5916
5917             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5918         });
5919     }
5920
5921     $target = 'local' if !$target;
5922
5923     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5924     # we abort live save after $conf->{memory}, so we need at max twice that space
5925     my $size = $conf->{memory}*2 + $driver_state_size;
5926
5927     my $name = "vm-$vmid-state-$snapname";
5928     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5929     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5930     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5931
5932     return $volid;
5933 };
5934
5935 sub snapshot_save_vmstate {
5936     my ($vmid, $conf, $snapname, $storecfg) = @_;
5937
5938     my $snap = $conf->{snapshots}->{$snapname};
5939
5940     $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5941     # always overwrite machine if we save vmstate. This makes sure we
5942     # can restore it later using correct machine type
5943     $snap->{machine} = get_current_qemu_machine($vmid);
5944 }
5945
5946 sub snapshot_prepare {
5947     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5948
5949     my $snap;
5950
5951     my $updatefn =  sub {
5952
5953         my $conf = load_config($vmid);
5954
5955         die "you can't take a snapshot if it's a template\n"
5956             if is_template($conf);
5957
5958         check_lock($conf);
5959
5960         $conf->{lock} = 'snapshot';
5961
5962         die "snapshot name '$snapname' already used\n"
5963             if defined($conf->{snapshots}->{$snapname});
5964
5965         my $storecfg = PVE::Storage::config();
5966         die "snapshot feature is not available\n"
5967             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
5968
5969         $snap = $conf->{snapshots}->{$snapname} = {};
5970
5971         if ($save_vmstate && check_running($vmid)) {
5972             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
5973         }
5974
5975         &$snapshot_copy_config($conf, $snap);
5976
5977         $snap->{snapstate} = "prepare";
5978         $snap->{snaptime} = time();
5979         $snap->{description} = $comment if $comment;
5980
5981         write_config($vmid, $conf);
5982     };
5983
5984     lock_config($vmid, $updatefn);
5985
5986     return $snap;
5987 }
5988
5989 sub snapshot_commit {
5990     my ($vmid, $snapname) = @_;
5991
5992     my $updatefn = sub {
5993
5994         my $conf = load_config($vmid);
5995
5996         die "missing snapshot lock\n"
5997             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5998
5999         my $has_machine_config = defined($conf->{machine});
6000
6001         my $snap = $conf->{snapshots}->{$snapname};
6002
6003         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6004
6005         die "wrong snapshot state\n"
6006             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
6007
6008         delete $snap->{snapstate};
6009         delete $conf->{lock};
6010
6011         my $newconf = &$snapshot_apply_config($conf, $snap);
6012
6013         delete $newconf->{machine} if !$has_machine_config;
6014
6015         $newconf->{parent} = $snapname;
6016
6017         write_config($vmid, $newconf);
6018     };
6019
6020     lock_config($vmid, $updatefn);
6021 }
6022
6023 sub snapshot_rollback {
6024     my ($vmid, $snapname) = @_;
6025
6026     my $prepare = 1;
6027
6028     my $storecfg = PVE::Storage::config();
6029
6030     my $conf = load_config($vmid);
6031
6032     my $get_snapshot_config = sub {
6033
6034         die "you can't rollback if vm is a template\n" if is_template($conf);
6035
6036         my $res = $conf->{snapshots}->{$snapname};
6037
6038         die "snapshot '$snapname' does not exist\n" if !defined($res);
6039
6040         return $res;
6041     };
6042
6043     my $snap = &$get_snapshot_config();
6044
6045     foreach_drive($snap, sub {
6046         my ($ds, $drive) = @_;
6047
6048         return if drive_is_cdrom($drive);
6049
6050         my $volid = $drive->{file};
6051
6052         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
6053     });
6054
6055     my $updatefn = sub {
6056
6057         $conf = load_config($vmid);
6058
6059         $snap = &$get_snapshot_config();
6060
6061         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6062             if $snap->{snapstate};
6063
6064         if ($prepare) {
6065             check_lock($conf);
6066             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6067         }
6068
6069         die "unable to rollback vm $vmid: vm is running\n"
6070             if check_running($vmid);
6071
6072         if ($prepare) {
6073             $conf->{lock} = 'rollback';
6074         } else {
6075             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6076             delete $conf->{lock};
6077         }
6078
6079         my $forcemachine;
6080
6081         if (!$prepare) {
6082             my $has_machine_config = defined($conf->{machine});
6083
6084             # copy snapshot config to current config
6085             $conf = &$snapshot_apply_config($conf, $snap);
6086             $conf->{parent} = $snapname;
6087
6088             # Note: old code did not store 'machine', so we try to be smart
6089             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6090             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6091             # we remove the 'machine' configuration if not explicitly specified
6092             # in the original config.
6093             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6094         }
6095
6096         write_config($vmid, $conf);
6097
6098         if (!$prepare && $snap->{vmstate}) {
6099             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6100             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6101         }
6102     };
6103
6104     lock_config($vmid, $updatefn);
6105
6106     foreach_drive($snap, sub {
6107         my ($ds, $drive) = @_;
6108
6109         return if drive_is_cdrom($drive);
6110
6111         my $volid = $drive->{file};
6112         my $device = "drive-$ds";
6113
6114         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6115     });
6116
6117     $prepare = 0;
6118     lock_config($vmid, $updatefn);
6119 }
6120
6121 my $savevm_wait = sub {
6122     my ($vmid) = @_;
6123
6124     for(;;) {
6125         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6126         if (!$stat->{status}) {
6127             die "savevm not active\n";
6128         } elsif ($stat->{status} eq 'active') {
6129             sleep(1);
6130             next;
6131         } elsif ($stat->{status} eq 'completed') {
6132             last;
6133         } else {
6134             die "query-savevm returned status '$stat->{status}'\n";
6135         }
6136     }
6137 };
6138
6139 sub do_snapshots_with_qemu {
6140     my ($storecfg, $volid) = @_;
6141
6142     my $storage_name = PVE::Storage::parse_volume_id($volid);
6143
6144     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6145         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6146         return 1;
6147     }
6148
6149     if ($volid =~ m/\.(qcow2|qed)$/){
6150         return 1;
6151     }
6152
6153     return undef;
6154 }
6155
6156 sub qga_check_running {
6157     my ($vmid) = @_;
6158
6159     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6160     if ($@) {
6161         warn "Qemu Guest Agent are not running - $@";
6162         return 0;
6163     }
6164     return 1;
6165 }
6166
6167 sub check_freeze_needed {
6168     my ($vmid, $config, $save_vmstate) = @_;
6169
6170     my $running = check_running($vmid);
6171     if ($save_vmstate) {
6172         return ($running, $running && $config->{agent} && qga_check_running($vmid));
6173     } else {
6174         return ($running, 0);
6175     }
6176 }
6177
6178 sub snapshot_create {
6179     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6180
6181     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6182
6183     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6184
6185     my $config = load_config($vmid);
6186
6187     my ($running, $freezefs) = check_freeze_needed($vmid, $config, $snap->{vmstate});
6188
6189     my $drivehash = {};
6190
6191     eval {
6192         if ($freezefs) {
6193             eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6194             warn "guest-fsfreeze-freeze problems - $@" if $@;
6195         }
6196
6197         # create internal snapshots of all drives
6198
6199         my $storecfg = PVE::Storage::config();
6200
6201         if ($running) {
6202             if ($snap->{vmstate}) {
6203                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6204                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6205                 &$savevm_wait($vmid);
6206             } else {
6207                 vm_mon_cmd($vmid, "savevm-start");
6208             }
6209         };
6210
6211         foreach_drive($snap, sub {
6212             my ($ds, $drive) = @_;
6213
6214             return if drive_is_cdrom($drive);
6215
6216             my $volid = $drive->{file};
6217             my $device = "drive-$ds";
6218
6219             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6220             $drivehash->{$ds} = 1;
6221        });
6222     };
6223     my $err = $@;
6224
6225     if ($running) {
6226         eval { vm_mon_cmd($vmid, "savevm-end")  };
6227         warn $@ if $@;
6228
6229         if ($freezefs) {
6230             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6231             warn "guest-fsfreeze-thaw problems - $@" if $@;
6232         }
6233
6234         # savevm-end is async, we need to wait
6235         for (;;) {
6236             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6237             if (!$stat->{bytes}) {
6238                 last;
6239             } else {
6240                 print "savevm not yet finished\n";
6241                 sleep(1);
6242                 next;
6243             }
6244         }
6245     }
6246
6247     if ($err) {
6248         warn "snapshot create failed: starting cleanup\n";
6249         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
6250         warn $@ if $@;
6251         die $err;
6252     }
6253
6254     snapshot_commit($vmid, $snapname);
6255 }
6256
6257 # Note: $drivehash is only set when called from snapshot_create.
6258 sub snapshot_delete {
6259     my ($vmid, $snapname, $force, $drivehash) = @_;
6260
6261     my $prepare = 1;
6262
6263     my $snap;
6264     my $unused = [];
6265
6266     my $unlink_parent = sub {
6267         my ($confref, $new_parent) = @_;
6268
6269         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6270             if ($new_parent) {
6271                 $confref->{parent} = $new_parent;
6272             } else {
6273                 delete $confref->{parent};
6274             }
6275         }
6276     };
6277
6278     my $updatefn =  sub {
6279         my ($remove_drive) = @_;
6280
6281         my $conf = load_config($vmid);
6282
6283         if (!$drivehash) {
6284             check_lock($conf);
6285             die "you can't delete a snapshot if vm is a template\n"
6286                 if is_template($conf);
6287         }
6288
6289         $snap = $conf->{snapshots}->{$snapname};
6290
6291         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6292
6293         # remove parent refs
6294         if (!$prepare) {
6295             &$unlink_parent($conf, $snap->{parent});
6296             foreach my $sn (keys %{$conf->{snapshots}}) {
6297                 next if $sn eq $snapname;
6298                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6299             }
6300         }
6301
6302         if ($remove_drive) {
6303             if ($remove_drive eq 'vmstate') {
6304                 delete $snap->{$remove_drive};
6305             } else {
6306                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6307                 my $volid = $drive->{file};
6308                 delete $snap->{$remove_drive};
6309                 add_unused_volume($conf, $volid);
6310             }
6311         }
6312
6313         if ($prepare) {
6314             $snap->{snapstate} = 'delete';
6315         } else {
6316             delete $conf->{snapshots}->{$snapname};
6317             delete $conf->{lock} if $drivehash;
6318             foreach my $volid (@$unused) {
6319                 add_unused_volume($conf, $volid);
6320             }
6321         }
6322
6323         write_config($vmid, $conf);
6324     };
6325
6326     lock_config($vmid, $updatefn);
6327
6328     # now remove vmstate file
6329
6330     my $storecfg = PVE::Storage::config();
6331
6332     if ($snap->{vmstate}) {
6333         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6334         if (my $err = $@) {
6335             die $err if !$force;
6336             warn $err;
6337         }
6338         # save changes (remove vmstate from snapshot)
6339         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6340     };
6341
6342     # now remove all internal snapshots
6343     foreach_drive($snap, sub {
6344         my ($ds, $drive) = @_;
6345
6346         return if drive_is_cdrom($drive);
6347
6348         my $volid = $drive->{file};
6349         my $device = "drive-$ds";
6350
6351         if (!$drivehash || $drivehash->{$ds}) {
6352             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6353             if (my $err = $@) {
6354                 die $err if !$force;
6355                 warn $err;
6356             }
6357         }
6358
6359         # save changes (remove drive fron snapshot)
6360         lock_config($vmid, $updatefn, $ds) if !$force;
6361         push @$unused, $volid;
6362     });
6363
6364     # now cleanup config
6365     $prepare = 0;
6366     lock_config($vmid, $updatefn);
6367 }
6368
6369 sub has_feature {
6370     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
6371
6372     my $err;
6373     foreach_drive($conf, sub {
6374         my ($ds, $drive) = @_;
6375
6376         return if drive_is_cdrom($drive);
6377         return if $backup_only && !$drive->{backup};
6378         my $volid = $drive->{file};
6379         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6380     });
6381
6382     return $err ? 0 : 1;
6383 }
6384
6385 sub template_create {
6386     my ($vmid, $conf, $disk) = @_;
6387
6388     my $storecfg = PVE::Storage::config();
6389
6390     foreach_drive($conf, sub {
6391         my ($ds, $drive) = @_;
6392
6393         return if drive_is_cdrom($drive);
6394         return if $disk && $ds ne $disk;
6395
6396         my $volid = $drive->{file};
6397         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6398
6399         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6400         $drive->{file} = $voliddst;
6401         $conf->{$ds} = print_drive($vmid, $drive);
6402         write_config($vmid, $conf);
6403     });
6404 }
6405
6406 sub is_template {
6407     my ($conf) = @_;
6408
6409     return 1 if defined $conf->{template} && $conf->{template} == 1;
6410 }
6411
6412 sub qemu_img_convert {
6413     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6414
6415     my $storecfg = PVE::Storage::config();
6416     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6417     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6418
6419     if ($src_storeid && $dst_storeid) {
6420
6421         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6422
6423         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6424         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6425
6426         my $src_format = qemu_img_format($src_scfg, $src_volname);
6427         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6428
6429         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6430         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6431
6432         my $cmd = [];
6433         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6434         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6435         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6436
6437         my $parser = sub {
6438             my $line = shift;
6439             if($line =~ m/\((\S+)\/100\%\)/){
6440                 my $percent = $1;
6441                 my $transferred = int($size * $percent / 100);
6442                 my $remaining = $size - $transferred;
6443
6444                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6445             }
6446
6447         };
6448
6449         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6450         my $err = $@;
6451         die "copy failed: $err" if $err;
6452     }
6453 }
6454
6455 sub qemu_img_format {
6456     my ($scfg, $volname) = @_;
6457
6458     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6459         return $1;
6460     } else {
6461         return "raw";
6462     }
6463 }
6464
6465 sub qemu_drive_mirror {
6466     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6467
6468     my $storecfg = PVE::Storage::config();
6469     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6470
6471     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6472
6473     my $format = qemu_img_format($dst_scfg, $dst_volname);
6474
6475     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6476
6477     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6478     $opts->{format} = $format if $format;
6479
6480     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6481
6482     eval {
6483     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6484         while (1) {
6485             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6486             my $stat = @$stats[0];
6487             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6488             die "error job is not mirroring" if $stat->{type} ne "mirror";
6489
6490             my $busy = $stat->{busy};
6491             my $ready = $stat->{ready};
6492
6493             if (my $total = $stat->{len}) {
6494                 my $transferred = $stat->{offset} || 0;
6495                 my $remaining = $total - $transferred;
6496                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6497
6498                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6499             }
6500
6501
6502             if ($stat->{ready} eq 'true') {
6503
6504                 last if $vmiddst != $vmid;
6505
6506                 # try to switch the disk if source and destination are on the same guest
6507                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6508                 last if !$@;
6509                 die $@ if $@ !~ m/cannot be completed/;
6510             }
6511             sleep 1;
6512         }
6513
6514
6515     };
6516     my $err = $@;
6517
6518     my $cancel_job = sub {
6519         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6520         while (1) {
6521             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6522             my $stat = @$stats[0];
6523             last if !$stat;
6524             sleep 1;
6525         }
6526     };
6527
6528     if ($err) {
6529         eval { &$cancel_job(); };
6530         die "mirroring error: $err";
6531     }
6532
6533     if ($vmiddst != $vmid) {
6534         # if we clone a disk for a new target vm, we don't switch the disk
6535         &$cancel_job(); # so we call block-job-cancel
6536     }
6537 }
6538
6539 sub clone_disk {
6540     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6541         $newvmid, $storage, $format, $full, $newvollist) = @_;
6542
6543     my $newvolid;
6544
6545     if (!$full) {
6546         print "create linked clone of drive $drivename ($drive->{file})\n";
6547         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6548         push @$newvollist, $newvolid;
6549     } else {
6550         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6551         $storeid = $storage if $storage;
6552
6553         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6554         if (!$format) {
6555             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6556             $format = qemu_img_format($scfg, $volname);
6557         }
6558
6559         # test if requested format is supported - else use default
6560         my $supported = grep { $_ eq $format } @$validFormats;
6561         $format = $defFormat if !$supported;
6562
6563         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6564
6565         print "create full clone of drive $drivename ($drive->{file})\n";
6566         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6567         push @$newvollist, $newvolid;
6568
6569         PVE::Storage::activate_volumes($storecfg, $newvollist);
6570
6571         if (!$running || $snapname) {
6572             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6573         } else {
6574             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6575         }
6576     }
6577
6578     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6579
6580     my $disk = $drive;
6581     $disk->{format} = undef;
6582     $disk->{file} = $newvolid;
6583     $disk->{size} = $size;
6584
6585     return $disk;
6586 }
6587
6588 # this only works if VM is running
6589 sub get_current_qemu_machine {
6590     my ($vmid) = @_;
6591
6592     my $cmd = { execute => 'query-machines', arguments => {} };
6593     my $res = vm_qmp_command($vmid, $cmd);
6594
6595     my ($current, $default);
6596     foreach my $e (@$res) {
6597         $default = $e->{name} if $e->{'is-default'};
6598         $current = $e->{name} if $e->{'is-current'};
6599     }
6600
6601     # fallback to the default machine if current is not supported by qemu
6602     return $current || $default || 'pc';
6603 }
6604
6605 sub qemu_machine_feature_enabled {
6606     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6607
6608     my $current_major;
6609     my $current_minor;
6610
6611     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6612
6613         $current_major = $3;
6614         $current_minor = $4;
6615
6616     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6617
6618         $current_major = $1;
6619         $current_minor = $2;
6620     }
6621
6622     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6623
6624
6625 }
6626
6627 sub qemu_machine_pxe {
6628     my ($vmid, $conf, $machine) = @_;
6629
6630     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6631
6632     foreach my $opt (keys %$conf) {
6633         next if $opt !~ m/^net(\d+)$/;
6634         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6635         next if !$net;
6636         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6637         return $machine.".pxe" if $romfile =~ m/pxe/;
6638         last;
6639     }
6640
6641     return $machine;
6642 }
6643
6644 sub qemu_use_old_bios_files {
6645     my ($machine_type) = @_;
6646
6647     return if !$machine_type;
6648
6649     my $use_old_bios_files = undef;
6650
6651     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6652         $machine_type = $1;
6653         $use_old_bios_files = 1;
6654     } else {
6655         my $kvmver = kvm_user_version();
6656         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6657         # load new efi bios files on migration. So this hack is required to allow
6658         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6659         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6660         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6661     }
6662
6663     return ($use_old_bios_files, $machine_type);
6664 }
6665
6666 sub lspci {
6667
6668     my $devices = {};
6669
6670     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6671             my (undef, $id, $function) = @_;
6672             my $res = { id => $id, function => $function};
6673             push @{$devices->{$id}}, $res;
6674     });
6675
6676     return $devices;
6677 }
6678
6679 sub vm_iothreads_list {
6680     my ($vmid) = @_;
6681
6682     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6683
6684     my $iothreads = {};
6685     foreach my $iothread (@$res) {
6686         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6687     }
6688
6689     return $iothreads;
6690 }
6691
6692 sub scsihw_infos {
6693     my ($conf, $drive) = @_;
6694
6695     my $maxdev = 0;
6696
6697     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6698         $maxdev = 7;
6699     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6700         $maxdev = 1;
6701     } else {
6702         $maxdev = 256;
6703     }
6704
6705     my $controller = int($drive->{index} / $maxdev);
6706     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6707
6708     return ($maxdev, $controller, $controller_prefix);
6709 }
6710
6711 # bash completion helper
6712
6713 sub complete_backup_archives {
6714     my ($cmdname, $pname, $cvalue) = @_;
6715
6716     my $cfg = PVE::Storage::config();
6717
6718     my $storeid;
6719
6720     if ($cvalue =~ m/^([^:]+):/) {
6721         $storeid = $1;
6722     }
6723
6724     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6725
6726     my $res = [];
6727     foreach my $id (keys %$data) {
6728         foreach my $item (@{$data->{$id}}) {
6729             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6730             push @$res, $item->{volid} if defined($item->{volid});
6731         }
6732     }
6733
6734     return $res;
6735 }
6736
6737 my $complete_vmid_full = sub {
6738     my ($running) = @_;
6739
6740     my $idlist = vmstatus();
6741
6742     my $res = [];
6743
6744     foreach my $id (keys %$idlist) {
6745         my $d = $idlist->{$id};
6746         if (defined($running)) {
6747             next if $d->{template};
6748             next if $running && $d->{status} ne 'running';
6749             next if !$running && $d->{status} eq 'running';
6750         }
6751         push @$res, $id;
6752
6753     }
6754     return $res;
6755 };
6756
6757 sub complete_vmid {
6758     return &$complete_vmid_full();
6759 }
6760
6761 sub complete_vmid_stopped {
6762     return &$complete_vmid_full(0);
6763 }
6764
6765 sub complete_vmid_running {
6766     return &$complete_vmid_full(1);
6767 }
6768
6769 sub complete_storage {
6770
6771     my $cfg = PVE::Storage::config();
6772     my $ids = $cfg->{ids};
6773
6774     my $res = [];
6775     foreach my $sid (keys %$ids) {
6776         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6777         next if !$ids->{$sid}->{content}->{images};
6778         push @$res, $sid;
6779     }
6780
6781     return $res;
6782 }
6783
6784 1;