PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbformat = {
 731     host => {
 732         default_key => 1,
 733         type => 'string', format => 'pve-qm-usb-device',
 734         format_description => 'HOSTUSBDEVICE|spice',
 735         description => 'The Host USB device or port or the value spice',
 736     },
 737     usb3 => {
 738         optional => 1,
 739         type => 'boolean',
 740         format_description => 'yes|no',
 741         description => 'Specifies whether if given host option is a USB3 device or port',
 742     },
 743 };
 744
 745 my $usbdesc = {
 746     optional => 1,
 747     type => 'string', format => $usbformat,
 748     description => <<EODESCR,
 749 Configure an USB device (n is 0 to 4). This can be used to
 750 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 751
 752 'bus-port(.port)*' (decimal numbers) or
 753 'vendor_id:product_id' (hexadeciaml numbers) or
 754 'spice'
 755
 756 You can use the 'lsusb -t' command to list existing usb devices.
 757
 758 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 759
 760 The value 'spice' can be used to add a usb redirection devices for spice.
 761
 762 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 763
 764 EODESCR
 765 };
 766 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 767
 768 my $hostpcidesc = {
 769         optional => 1,
 770         type => 'string', format => 'pve-qm-hostpci',
 771         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 772         description => <<EODESCR,
 773 Map host pci devices. HOSTPCIDEVICE syntax is:
 774
 775 'bus:dev.func' (hexadecimal numbers)
 776
 777 You can us the 'lspci' command to list existing pci devices.
 778
 779 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 780
 781 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 782
 783 Experimental: user reported problems with this option.
 784 EODESCR
 785 };
 786 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 787
 788 my $serialdesc = {
 789         optional => 1,
 790         type => 'string',
 791         pattern => '(/dev/.+|socket)',
 792         description =>  <<EODESCR,
 793 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 794
 795 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 796
 797 Experimental: user reported problems with this option.
 798 EODESCR
 799 };
 800
 801 my $paralleldesc= {
 802         optional => 1,
 803         type => 'string',
 804         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 805         description =>  <<EODESCR,
 806 Map host parallel devices (n is 0 to 2).
 807
 808 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 809
 810 Experimental: user reported problems with this option.
 811 EODESCR
 812 };
 813
 814 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 815     $confdesc->{"parallel$i"} = $paralleldesc;
 816 }
 817
 818 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 819     $confdesc->{"serial$i"} = $serialdesc;
 820 }
 821
 822 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 823     $confdesc->{"hostpci$i"} = $hostpcidesc;
 824 }
 825
 826 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 827     $drivename_hash->{"ide$i"} = 1;
 828     $confdesc->{"ide$i"} = $idedesc;
 829 }
 830
 831 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 832     $drivename_hash->{"sata$i"} = 1;
 833     $confdesc->{"sata$i"} = $satadesc;
 834 }
 835
 836 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 837     $drivename_hash->{"scsi$i"} = 1;
 838     $confdesc->{"scsi$i"} = $scsidesc ;
 839 }
 840
 841 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 842     $drivename_hash->{"virtio$i"} = 1;
 843     $confdesc->{"virtio$i"} = $virtiodesc;
 844 }
 845
 846 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 847     $confdesc->{"usb$i"} = $usbdesc;
 848 }
 849
 850 my $unuseddesc = {
 851     optional => 1,
 852     type => 'string', format => 'pve-volume-id',
 853     description => "Reference to unused volumes.",
 854 };
 855
 856 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 857     $confdesc->{"unused$i"} = $unuseddesc;
 858 }
 859
 860 my $kvm_api_version = 0;
 861
 862 sub kvm_version {
 863
 864     return $kvm_api_version if $kvm_api_version;
 865
 866     my $fh = IO::File->new("</dev/kvm") ||
 867         return 0;
 868
 869     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 870         $kvm_api_version = $v;
 871     }
 872
 873     $fh->close();
 874
 875     return  $kvm_api_version;
 876 }
 877
 878 my $kvm_user_version;
 879
 880 sub kvm_user_version {
 881
 882     return $kvm_user_version if $kvm_user_version;
 883
 884     $kvm_user_version = 'unknown';
 885
 886     my $code = sub {
 887         my $line = shift;
 888         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 889             $kvm_user_version = $2;
 890         }
 891     };
 892
 893     eval { run_command("kvm -version", outfunc => $code); };
 894     warn $@ if $@;
 895
 896     return $kvm_user_version;
 897
 898 }
 899
 900 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 901
 902 sub disknames {
 903     # order is important - used to autoselect boot disk
 904     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 905             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 906             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 907             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 908 }
 909
 910 sub valid_drivename {
 911     my $dev = shift;
 912
 913     return defined($drivename_hash->{$dev});
 914 }
 915
 916 sub option_exists {
 917     my $key = shift;
 918     return defined($confdesc->{$key});
 919 }
 920
 921 sub nic_models {
 922     return $nic_model_list;
 923 }
 924
 925 sub os_list_description {
 926
 927     return {
 928         other => 'Other',
 929         wxp => 'Windows XP',
 930         w2k => 'Windows 2000',
 931         w2k3 =>, 'Windows 2003',
 932         w2k8 => 'Windows 2008',
 933         wvista => 'Windows Vista',
 934         win7 => 'Windows 7',
 935         win8 => 'Windows 8/2012',
 936         l24 => 'Linux 2.4',
 937         l26 => 'Linux 2.6',
 938     };
 939 }
 940
 941 my $cdrom_path;
 942
 943 sub get_cdrom_path {
 944
 945     return  $cdrom_path if $cdrom_path;
 946
 947     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 948     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 949     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 950 }
 951
 952 sub get_iso_path {
 953     my ($storecfg, $vmid, $cdrom) = @_;
 954
 955     if ($cdrom eq 'cdrom') {
 956         return get_cdrom_path();
 957     } elsif ($cdrom eq 'none') {
 958         return '';
 959     } elsif ($cdrom =~ m|^/|) {
 960         return $cdrom;
 961     } else {
 962         return PVE::Storage::path($storecfg, $cdrom);
 963     }
 964 }
 965
 966 # try to convert old style file names to volume IDs
 967 sub filename_to_volume_id {
 968     my ($vmid, $file, $media) = @_;
 969
 970     if (!($file eq 'none' || $file eq 'cdrom' ||
 971           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 972
 973         return undef if $file =~ m|/|;
 974
 975         if ($media && $media eq 'cdrom') {
 976             $file = "local:iso/$file";
 977         } else {
 978             $file = "local:$vmid/$file";
 979         }
 980     }
 981
 982     return $file;
 983 }
 984
 985 sub verify_media_type {
 986     my ($opt, $vtype, $media) = @_;
 987
 988     return if !$media;
 989
 990     my $etype;
 991     if ($media eq 'disk') {
 992         $etype = 'images';
 993     } elsif ($media eq 'cdrom') {
 994         $etype = 'iso';
 995     } else {
 996         die "internal error";
 997     }
 998
 999     return if ($vtype eq $etype);
1000
1001     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1002 }
1003
1004 sub cleanup_drive_path {
1005     my ($opt, $storecfg, $drive) = @_;
1006
1007     # try to convert filesystem paths to volume IDs
1008
1009     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1010         ($drive->{file} !~ m|^/dev/.+|) &&
1011         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1012         ($drive->{file} !~ m/^\d+$/)) {
1013         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1014         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1015         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1016         verify_media_type($opt, $vtype, $drive->{media});
1017         $drive->{file} = $volid;
1018     }
1019
1020     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1021 }
1022
1023 sub parse_hotplug_features {
1024     my ($data) = @_;
1025
1026     my $res = {};
1027
1028     return $res if $data eq '0';
1029
1030     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1031
1032     foreach my $feature (PVE::Tools::split_list($data)) {
1033         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1034             $res->{$1} = 1;
1035         } else {
1036             warn "ignoring unknown hotplug feature '$feature'\n";
1037         }
1038     }
1039     return $res;
1040 }
1041
1042 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1043 sub pve_verify_hotplug_features {
1044     my ($value, $noerr) = @_;
1045
1046     return $value if parse_hotplug_features($value);
1047
1048     return undef if $noerr;
1049
1050     die "unable to parse hotplug option\n";
1051 }
1052
1053 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1054 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1055 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1056 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1057 #        [,iothread=on][,serial=serial][,model=model]
1058
1059 sub parse_drive {
1060     my ($key, $data) = @_;
1061
1062     my ($interface, $index);
1063
1064     if ($key =~ m/^([^\d]+)(\d+)$/) {
1065         $interface = $1;
1066         $index = $2;
1067     } else {
1068         return undef;
1069     }
1070
1071     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1072                                      : $confdesc->{$key}->{format};
1073     if (!$desc) {
1074         warn "invalid drive key: $key\n";
1075         return undef;
1076     }
1077     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1078     return undef if !$res;
1079     $res->{interface} = $interface;
1080     $res->{index} = $index;
1081
1082     my $error = 0;
1083     foreach my $opt (qw(bps bps_rd bps_wr)) {
1084         if (my $bps = defined(delete $res->{$opt})) {
1085             if (defined($res->{"m$opt"})) {
1086                 warn "both $opt and m$opt specified\n";
1087                 ++$error;
1088                 next;
1089             }
1090             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1091         }
1092     }
1093     return undef if $error;
1094
1095     return undef if $res->{mbps_rd} && $res->{mbps};
1096     return undef if $res->{mbps_wr} && $res->{mbps};
1097     return undef if $res->{iops_rd} && $res->{iops};
1098     return undef if $res->{iops_wr} && $res->{iops};
1099
1100     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1101         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1102         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1103         return undef if $res->{interface} eq 'virtio';
1104     }
1105
1106     if (my $size = $res->{size}) {
1107         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1108     }
1109
1110     return $res;
1111 }
1112
1113 sub print_drive {
1114     my ($vmid, $drive) = @_;
1115     my $data = { %$drive };
1116     delete $data->{$_} for qw(index interface);
1117     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1118 }
1119
1120 sub scsi_inquiry {
1121     my($fh, $noerr) = @_;
1122
1123     my $SG_IO = 0x2285;
1124     my $SG_GET_VERSION_NUM = 0x2282;
1125
1126     my $versionbuf = "\x00" x 8;
1127     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1128     if (!$ret) {
1129         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1130         return undef;
1131     }
1132     my $version = unpack("I", $versionbuf);
1133     if ($version < 30000) {
1134         die "scsi generic interface too old\n"  if !$noerr;
1135         return undef;
1136     }
1137
1138     my $buf = "\x00" x 36;
1139     my $sensebuf = "\x00" x 8;
1140     my $cmd = pack("C x3 C x1", 0x12, 36);
1141
1142     # see /usr/include/scsi/sg.h
1143     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1144
1145     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1146                       length($sensebuf), 0, length($buf), $buf,
1147                       $cmd, $sensebuf, 6000);
1148
1149     $ret = ioctl($fh, $SG_IO, $packet);
1150     if (!$ret) {
1151         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1152         return undef;
1153     }
1154
1155     my @res = unpack($sg_io_hdr_t, $packet);
1156     if ($res[17] || $res[18]) {
1157         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1158         return undef;
1159     }
1160
1161     my $res = {};
1162     (my $byte0, my $byte1, $res->{vendor},
1163      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1164
1165     $res->{removable} = $byte1 & 128 ? 1 : 0;
1166     $res->{type} = $byte0 & 31;
1167
1168     return $res;
1169 }
1170
1171 sub path_is_scsi {
1172     my ($path) = @_;
1173
1174     my $fh = IO::File->new("+<$path") || return undef;
1175     my $res = scsi_inquiry($fh, 1);
1176     close($fh);
1177
1178     return $res;
1179 }
1180
1181 sub machine_type_is_q35 {
1182     my ($conf) = @_;
1183
1184     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1185 }
1186
1187 sub print_tabletdevice_full {
1188     my ($conf) = @_;
1189
1190     my $q35 = machine_type_is_q35($conf);
1191
1192     # we use uhci for old VMs because tablet driver was buggy in older qemu
1193     my $usbbus = $q35 ? "ehci" : "uhci";
1194
1195     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1196 }
1197
1198 sub print_drivedevice_full {
1199     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1200
1201     my $device = '';
1202     my $maxdev = 0;
1203
1204     if ($drive->{interface} eq 'virtio') {
1205         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1206         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1207         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1208     } elsif ($drive->{interface} eq 'scsi') {
1209
1210         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1211         my $unit = $drive->{index} % $maxdev;
1212         my $devicetype = 'hd';
1213         my $path = '';
1214         if (drive_is_cdrom($drive)) {
1215             $devicetype = 'cd';
1216         } else {
1217             if ($drive->{file} =~ m|^/|) {
1218                 $path = $drive->{file};
1219                 if (my $info = path_is_scsi($path)) {
1220                     if ($info->{type} == 0) {
1221                         $devicetype = 'block';
1222                     } elsif ($info->{type} == 1) { # tape
1223                         $devicetype = 'generic';
1224                     }
1225                 }
1226             } else {
1227                  $path = PVE::Storage::path($storecfg, $drive->{file});
1228             }
1229
1230             if($path =~ m/^iscsi\:\/\//){
1231                 $devicetype = 'generic';
1232             }
1233         }
1234
1235         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1236             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1237         } else {
1238             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1239         }
1240
1241     } elsif ($drive->{interface} eq 'ide'){
1242         $maxdev = 2;
1243         my $controller = int($drive->{index} / $maxdev);
1244         my $unit = $drive->{index} % $maxdev;
1245         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1246
1247         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1248         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1249             $device .= ",model=$model";
1250         }
1251     } elsif ($drive->{interface} eq 'sata'){
1252         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1253         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1254         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1255     } elsif ($drive->{interface} eq 'usb') {
1256         die "implement me";
1257         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1258     } else {
1259         die "unsupported interface type";
1260     }
1261
1262     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1263
1264     return $device;
1265 }
1266
1267 sub get_initiator_name {
1268     my $initiator;
1269
1270     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1271     while (defined(my $line = <$fh>)) {
1272         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1273         $initiator = $1;
1274         last;
1275     }
1276     $fh->close();
1277
1278     return $initiator;
1279 }
1280
1281 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1282 sub print_drive_full {
1283     my ($storecfg, $vmid, $drive) = @_;
1284
1285     my $path;
1286     my $volid = $drive->{file};
1287     my $format;
1288
1289     if (drive_is_cdrom($drive)) {
1290         $path = get_iso_path($storecfg, $vmid, $volid);
1291     } else {
1292         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1293         if ($storeid) {
1294             $path = PVE::Storage::path($storecfg, $volid);
1295             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1296             $format = qemu_img_format($scfg, $volname);
1297         } else {
1298             $path = $volid;
1299             $format = "raw";
1300         }
1301    }
1302
1303     my $opts = '';
1304     foreach my $o (@qemu_drive_options) {
1305         next if $o eq 'bootindex';
1306         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1307     }
1308
1309     $opts .= ",format=$format" if $format && !$drive->{format};
1310
1311     foreach my $o (qw(bps bps_rd bps_wr)) {
1312         my $v = $drive->{"m$o"};
1313         $opts .= ",$o=" . int($v*1024*1024) if $v;
1314     }
1315
1316     my $cache_direct = 0;
1317
1318     if (my $cache = $drive->{cache}) {
1319         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1320     } elsif (!drive_is_cdrom($drive)) {
1321         $opts .= ",cache=none";
1322         $cache_direct = 1;
1323     }
1324
1325     # aio native works only with O_DIRECT
1326     if (!$drive->{aio}) {
1327         if($cache_direct) {
1328             $opts .= ",aio=native";
1329         } else {
1330             $opts .= ",aio=threads";
1331         }
1332     }
1333
1334     if (!drive_is_cdrom($drive)) {
1335         my $detectzeroes;
1336         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1337             $detectzeroes = 'off';
1338         } elsif ($drive->{discard}) {
1339             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1340         } else {
1341             # This used to be our default with discard not being specified:
1342             $detectzeroes = 'on';
1343         }
1344         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1345     }
1346
1347     my $pathinfo = $path ? "file=$path," : '';
1348
1349     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1350 }
1351
1352 sub print_netdevice_full {
1353     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1354
1355     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1356
1357     my $device = $net->{model};
1358     if ($net->{model} eq 'virtio') {
1359          $device = 'virtio-net-pci';
1360      };
1361
1362     my $pciaddr = print_pci_addr("$netid", $bridges);
1363     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1364     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1365         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1366         my $vectors = $net->{queues} * 2 + 2;
1367         $tmpstr .= ",vectors=$vectors,mq=on";
1368     }
1369     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1370
1371     if ($use_old_bios_files) {
1372         my $romfile;
1373         if ($device eq 'virtio-net-pci') {
1374             $romfile = 'pxe-virtio.rom';
1375         } elsif ($device eq 'e1000') {
1376             $romfile = 'pxe-e1000.rom';
1377         } elsif ($device eq 'ne2k') {
1378             $romfile = 'pxe-ne2k_pci.rom';
1379         } elsif ($device eq 'pcnet') {
1380             $romfile = 'pxe-pcnet.rom';
1381         } elsif ($device eq 'rtl8139') {
1382             $romfile = 'pxe-rtl8139.rom';
1383         }
1384         $tmpstr .= ",romfile=$romfile" if $romfile;
1385     }
1386
1387     return $tmpstr;
1388 }
1389
1390 sub print_netdev_full {
1391     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1392
1393     my $i = '';
1394     if ($netid =~ m/^net(\d+)$/) {
1395         $i = int($1);
1396     }
1397
1398     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1399
1400     my $ifname = "tap${vmid}i$i";
1401
1402     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1403     die "interface name '$ifname' is too long (max 15 character)\n"
1404         if length($ifname) >= 16;
1405
1406     my $vhostparam = '';
1407     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1408
1409     my $vmname = $conf->{name} || "vm$vmid";
1410
1411     my $netdev = "";
1412     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1413
1414     if ($net->{bridge}) {
1415         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1416     } else {
1417         $netdev = "type=user,id=$netid,hostname=$vmname";
1418     }
1419
1420     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1421
1422     return $netdev;
1423 }
1424
1425 sub drive_is_cdrom {
1426     my ($drive) = @_;
1427
1428     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1429
1430 }
1431
1432 sub parse_numa {
1433     my ($data) = @_;
1434
1435     my $res = {};
1436
1437     foreach my $kvp (split(/,/, $data)) {
1438
1439         if ($kvp =~ m/^memory=(\S+)$/) {
1440             $res->{memory} = $1;
1441         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1442             $res->{policy} = $1;
1443         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1444             $res->{cpus}->{start} = $1;
1445             $res->{cpus}->{end} = $3;
1446         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1447             $res->{hostnodes}->{start} = $1;
1448             $res->{hostnodes}->{end} = $3;
1449         } else {
1450             return undef;
1451         }
1452     }
1453
1454     return $res;
1455 }
1456
1457 sub parse_hostpci {
1458     my ($value) = @_;
1459
1460     return undef if !$value;
1461
1462
1463     my @list = split(/,/, $value);
1464     my $found;
1465
1466     my $res = {};
1467     foreach my $kv (@list) {
1468
1469         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1470             $found = 1;
1471             if(defined($4)){
1472                 push @{$res->{pciid}}, { id => $2 , function => $4};
1473
1474             }else{
1475                 my $pcidevices = lspci($2);
1476                 $res->{pciid} = $pcidevices->{$2};
1477             }
1478         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1479             $res->{rombar} = $1;
1480         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1481             $res->{'x-vga'} = $1;
1482         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1483             $res->{pcie} = 1 if $1 == 1;
1484         } else {
1485             warn "unknown hostpci setting '$kv'\n";
1486         }
1487     }
1488
1489     return undef if !$found;
1490
1491     return $res;
1492 }
1493
1494 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1495 sub parse_net {
1496     my ($data) = @_;
1497
1498     my $res = {};
1499
1500     foreach my $kvp (split(/,/, $data)) {
1501
1502         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1503             my $model = lc($1);
1504             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1505             $res->{model} = $model;
1506             $res->{macaddr} = $mac;
1507         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1508             $res->{bridge} = $1;
1509         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1510             $res->{queues} = $1;
1511         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1512             $res->{rate} = $1;
1513         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1514             $res->{tag} = $1;
1515         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1516             $res->{trunks} = $1;
1517         } elsif ($kvp =~ m/^firewall=([01])$/) {
1518             $res->{firewall} = $1;
1519         } elsif ($kvp =~ m/^link_down=([01])$/) {
1520             $res->{link_down} = $1;
1521         } else {
1522             return undef;
1523         }
1524
1525     }
1526
1527     return undef if !$res->{model};
1528
1529     return $res;
1530 }
1531
1532 sub print_net {
1533     my $net = shift;
1534
1535     my $res = "$net->{model}";
1536     $res .= "=$net->{macaddr}" if $net->{macaddr};
1537     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1538     $res .= ",rate=$net->{rate}" if $net->{rate};
1539     $res .= ",tag=$net->{tag}" if $net->{tag};
1540     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1541     $res .= ",firewall=1" if $net->{firewall};
1542     $res .= ",link_down=1" if $net->{link_down};
1543     $res .= ",queues=$net->{queues}" if $net->{queues};
1544
1545     return $res;
1546 }
1547
1548 sub add_random_macs {
1549     my ($settings) = @_;
1550
1551     foreach my $opt (keys %$settings) {
1552         next if $opt !~ m/^net(\d+)$/;
1553         my $net = parse_net($settings->{$opt});
1554         next if !$net;
1555         $settings->{$opt} = print_net($net);
1556     }
1557 }
1558
1559 sub add_unused_volume {
1560     my ($config, $volid) = @_;
1561
1562     my $key;
1563     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1564         my $test = "unused$ind";
1565         if (my $vid = $config->{$test}) {
1566             return if $vid eq $volid; # do not add duplicates
1567         } else {
1568             $key = $test;
1569         }
1570     }
1571
1572     die "To many unused volume - please delete them first.\n" if !$key;
1573
1574     $config->{$key} = $volid;
1575
1576     return $key;
1577 }
1578
1579 sub vm_is_volid_owner {
1580     my ($storecfg, $vmid, $volid) = @_;
1581
1582     if ($volid !~  m|^/|) {
1583         my ($path, $owner);
1584         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1585         if ($owner && ($owner == $vmid)) {
1586             return 1;
1587         }
1588     }
1589
1590     return undef;
1591 }
1592
1593 sub split_flagged_list {
1594     my $text = shift || '';
1595     $text =~ s/[,;]/ /g;
1596     $text =~ s/^\s+//;
1597     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1598 }
1599
1600 sub join_flagged_list {
1601     my ($how, $lst) = @_;
1602     join $how, map { $lst->{$_} . $_ } keys %$lst;
1603 }
1604
1605 sub vmconfig_delete_pending_option {
1606     my ($conf, $key, $force) = @_;
1607
1608     delete $conf->{pending}->{$key};
1609     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1610     $pending_delete_hash->{$key} = $force ? '!' : '';
1611     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1612 }
1613
1614 sub vmconfig_undelete_pending_option {
1615     my ($conf, $key) = @_;
1616
1617     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1618     delete $pending_delete_hash->{$key};
1619
1620     if (%$pending_delete_hash) {
1621         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1622     } else {
1623         delete $conf->{pending}->{delete};
1624     }
1625 }
1626
1627 sub vmconfig_register_unused_drive {
1628     my ($storecfg, $vmid, $conf, $drive) = @_;
1629
1630     if (!drive_is_cdrom($drive)) {
1631         my $volid = $drive->{file};
1632         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1633             add_unused_volume($conf, $volid, $vmid);
1634         }
1635     }
1636 }
1637
1638 sub vmconfig_cleanup_pending {
1639     my ($conf) = @_;
1640
1641     # remove pending changes when nothing changed
1642     my $changes;
1643     foreach my $opt (keys %{$conf->{pending}}) {
1644         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1645             $changes = 1;
1646             delete $conf->{pending}->{$opt};
1647         }
1648     }
1649
1650     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1651     my $pending_delete_hash = {};
1652     while (my ($opt, $force) = each %$current_delete_hash) {
1653         if (defined($conf->{$opt})) {
1654             $pending_delete_hash->{$opt} = $force;
1655         } else {
1656             $changes = 1;
1657         }
1658     }
1659
1660     if (%$pending_delete_hash) {
1661         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1662     } else {
1663         delete $conf->{pending}->{delete};
1664     }
1665
1666     return $changes;
1667 }
1668
1669 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1670 my $smbios1_desc = {
1671     uuid => {
1672         type => 'string',
1673         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1674         format_description => 'UUID',
1675         optional => 1,
1676     },
1677     version => {
1678         type => 'string',
1679         pattern => '\S+',
1680         format_description => 'str',
1681         optional => 1,
1682     },
1683     serial => {
1684         type => 'string',
1685         pattern => '\S+',
1686         format_description => 'str',
1687         optional => 1,
1688     },
1689     manufacturer => {
1690         type => 'string',
1691         pattern => '\S+',
1692         format_description => 'name',
1693         optional => 1,
1694     },
1695     product => {
1696         type => 'string',
1697         pattern => '\S+',
1698         format_description => 'name',
1699         optional => 1,
1700     },
1701     sku => {
1702         type => 'string',
1703         pattern => '\S+',
1704         format_description => 'str',
1705         optional => 1,
1706     },
1707     family => {
1708         type => 'string',
1709         pattern => '\S+',
1710         format_description => 'str',
1711         optional => 1,
1712     },
1713 };
1714
1715 sub parse_smbios1 {
1716     my ($data) = @_;
1717
1718     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1719     warn $@ if $@;
1720     return $res;
1721 }
1722
1723 sub print_smbios1 {
1724     my ($smbios1) = @_;
1725     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1726 }
1727
1728 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1729
1730 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1731 sub verify_bootdisk {
1732     my ($value, $noerr) = @_;
1733
1734     return $value if valid_drivename($value);
1735
1736     return undef if $noerr;
1737
1738     die "invalid boot disk '$value'\n";
1739 }
1740
1741 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1742 sub verify_numa {
1743     my ($value, $noerr) = @_;
1744
1745     return $value if parse_numa($value);
1746
1747     return undef if $noerr;
1748
1749     die "unable to parse numa options\n";
1750 }
1751
1752 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1753 sub verify_net {
1754     my ($value, $noerr) = @_;
1755
1756     return $value if parse_net($value);
1757
1758     return undef if $noerr;
1759
1760     die "unable to parse network options\n";
1761 }
1762
1763 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1764 sub verify_hostpci {
1765     my ($value, $noerr) = @_;
1766
1767     return $value if parse_hostpci($value);
1768
1769     return undef if $noerr;
1770
1771     die "unable to parse pci id\n";
1772 }
1773
1774 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1775 sub verify_watchdog {
1776     my ($value, $noerr) = @_;
1777
1778     return $value if parse_watchdog($value);
1779
1780     return undef if $noerr;
1781
1782     die "unable to parse watchdog options\n";
1783 }
1784
1785 sub parse_watchdog {
1786     my ($value) = @_;
1787
1788     return undef if !$value;
1789
1790     my $res = {};
1791
1792     foreach my $p (split(/,/, $value)) {
1793         next if $p =~ m/^\s*$/;
1794
1795         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1796             $res->{model} = $2;
1797         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1798             $res->{action} = $2;
1799         } else {
1800             return undef;
1801         }
1802     }
1803
1804     return $res;
1805 }
1806
1807 sub parse_usb_device {
1808     my ($value) = @_;
1809
1810     return undef if !$value;
1811
1812     my $res = {};
1813     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1814         $res->{vendorid} = $2;
1815         $res->{productid} = $4;
1816     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1817         $res->{hostbus} = $1;
1818         $res->{hostport} = $2;
1819     } elsif ($value =~ m/^spice$/i) {
1820         $res->{spice} = 1;
1821     } else {
1822         return undef;
1823     }
1824
1825     return $res;
1826 }
1827
1828 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1829 sub verify_usb_device {
1830     my ($value, $noerr) = @_;
1831
1832     return $value if parse_usb_device($value);
1833
1834     return undef if $noerr;
1835
1836     die "unable to parse usb device\n";
1837 }
1838
1839 # add JSON properties for create and set function
1840 sub json_config_properties {
1841     my $prop = shift;
1842
1843     foreach my $opt (keys %$confdesc) {
1844         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1845         $prop->{$opt} = $confdesc->{$opt};
1846     }
1847
1848     return $prop;
1849 }
1850
1851 sub check_type {
1852     my ($key, $value) = @_;
1853
1854     die "unknown setting '$key'\n" if !$confdesc->{$key};
1855
1856     my $type = $confdesc->{$key}->{type};
1857
1858     if (!defined($value)) {
1859         die "got undefined value\n";
1860     }
1861
1862     if ($value =~ m/[\n\r]/) {
1863         die "property contains a line feed\n";
1864     }
1865
1866     if ($type eq 'boolean') {
1867         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1868         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1869         die "type check ('boolean') failed - got '$value'\n";
1870     } elsif ($type eq 'integer') {
1871         return int($1) if $value =~ m/^(\d+)$/;
1872         die "type check ('integer') failed - got '$value'\n";
1873     } elsif ($type eq 'number') {
1874         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1875         die "type check ('number') failed - got '$value'\n";
1876     } elsif ($type eq 'string') {
1877         if (my $fmt = $confdesc->{$key}->{format}) {
1878             if ($fmt eq 'pve-qm-drive') {
1879                 # special case - we need to pass $key to parse_drive()
1880                 my $drive = parse_drive($key, $value);
1881                 return $value if $drive;
1882                 die "unable to parse drive options\n";
1883             }
1884             PVE::JSONSchema::check_format($fmt, $value);
1885             return $value;
1886         }
1887         $value =~ s/^\"(.*)\"$/$1/;
1888         return $value;
1889     } else {
1890         die "internal error"
1891     }
1892 }
1893
1894 sub lock_config_full {
1895     my ($vmid, $timeout, $code, @param) = @_;
1896
1897     my $filename = config_file_lock($vmid);
1898
1899     my $res = lock_file($filename, $timeout, $code, @param);
1900
1901     die $@ if $@;
1902
1903     return $res;
1904 }
1905
1906 sub lock_config_mode {
1907     my ($vmid, $timeout, $shared, $code, @param) = @_;
1908
1909     my $filename = config_file_lock($vmid);
1910
1911     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1912
1913     die $@ if $@;
1914
1915     return $res;
1916 }
1917
1918 sub lock_config {
1919     my ($vmid, $code, @param) = @_;
1920
1921     return lock_config_full($vmid, 10, $code, @param);
1922 }
1923
1924 sub cfs_config_path {
1925     my ($vmid, $node) = @_;
1926
1927     $node = $nodename if !$node;
1928     return "nodes/$node/qemu-server/$vmid.conf";
1929 }
1930
1931 sub check_iommu_support{
1932     #fixme : need to check IOMMU support
1933     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1934
1935     my $iommu=1;
1936     return $iommu;
1937
1938 }
1939
1940 sub config_file {
1941     my ($vmid, $node) = @_;
1942
1943     my $cfspath = cfs_config_path($vmid, $node);
1944     return "/etc/pve/$cfspath";
1945 }
1946
1947 sub config_file_lock {
1948     my ($vmid) = @_;
1949
1950     return "$lock_dir/lock-$vmid.conf";
1951 }
1952
1953 sub touch_config {
1954     my ($vmid) = @_;
1955
1956     my $conf = config_file($vmid);
1957     utime undef, undef, $conf;
1958 }
1959
1960 sub destroy_vm {
1961     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1962
1963     my $conffile = config_file($vmid);
1964
1965     my $conf = load_config($vmid);
1966
1967     check_lock($conf) if !$skiplock;
1968
1969     # only remove disks owned by this VM
1970     foreach_drive($conf, sub {
1971         my ($ds, $drive) = @_;
1972
1973         return if drive_is_cdrom($drive);
1974
1975         my $volid = $drive->{file};
1976
1977         return if !$volid || $volid =~ m|^/|;
1978
1979         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1980         return if !$path || !$owner || ($owner != $vmid);
1981
1982         PVE::Storage::vdisk_free($storecfg, $volid);
1983     });
1984
1985     if ($keep_empty_config) {
1986         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1987     } else {
1988         unlink $conffile;
1989     }
1990
1991     # also remove unused disk
1992     eval {
1993         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1994
1995         eval {
1996             PVE::Storage::foreach_volid($dl, sub {
1997                 my ($volid, $sid, $volname, $d) = @_;
1998                 PVE::Storage::vdisk_free($storecfg, $volid);
1999             });
2000         };
2001         warn $@ if $@;
2002
2003     };
2004     warn $@ if $@;
2005 }
2006
2007 sub load_config {
2008     my ($vmid, $node) = @_;
2009
2010     my $cfspath = cfs_config_path($vmid, $node);
2011
2012     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2013
2014     die "no such VM ('$vmid')\n" if !defined($conf);
2015
2016     return $conf;
2017 }
2018
2019 sub parse_vm_config {
2020     my ($filename, $raw) = @_;
2021
2022     return undef if !defined($raw);
2023
2024     my $res = {
2025         digest => Digest::SHA::sha1_hex($raw),
2026         snapshots => {},
2027         pending => {},
2028     };
2029
2030     $filename =~ m|/qemu-server/(\d+)\.conf$|
2031         || die "got strange filename '$filename'";
2032
2033     my $vmid = $1;
2034
2035     my $conf = $res;
2036     my $descr;
2037     my $section = '';
2038
2039     my @lines = split(/\n/, $raw);
2040     foreach my $line (@lines) {
2041         next if $line =~ m/^\s*$/;
2042
2043         if ($line =~ m/^\[PENDING\]\s*$/i) {
2044             $section = 'pending';
2045             if (defined($descr)) {
2046                 $descr =~ s/\s+$//;
2047                 $conf->{description} = $descr;
2048             }
2049             $descr = undef;
2050             $conf = $res->{$section} = {};
2051             next;
2052
2053         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2054             $section = $1;
2055             if (defined($descr)) {
2056                 $descr =~ s/\s+$//;
2057                 $conf->{description} = $descr;
2058             }
2059             $descr = undef;
2060             $conf = $res->{snapshots}->{$section} = {};
2061             next;
2062         }
2063
2064         if ($line =~ m/^\#(.*)\s*$/) {
2065             $descr = '' if !defined($descr);
2066             $descr .= PVE::Tools::decode_text($1) . "\n";
2067             next;
2068         }
2069
2070         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2071             $descr = '' if !defined($descr);
2072             $descr .= PVE::Tools::decode_text($2);
2073         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2074             $conf->{snapstate} = $1;
2075         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2076             my $key = $1;
2077             my $value = $2;
2078             $conf->{$key} = $value;
2079         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2080             my $value = $1;
2081             if ($section eq 'pending') {
2082                 $conf->{delete} = $value; # we parse this later
2083             } else {
2084                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2085             }
2086         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2087             my $key = $1;
2088             my $value = $2;
2089             eval { $value = check_type($key, $value); };
2090             if ($@) {
2091                 warn "vm $vmid - unable to parse value of '$key' - $@";
2092             } else {
2093                 my $fmt = $confdesc->{$key}->{format};
2094                 if ($fmt && $fmt eq 'pve-qm-drive') {
2095                     my $v = parse_drive($key, $value);
2096                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2097                         $v->{file} = $volid;
2098                         $value = print_drive($vmid, $v);
2099                     } else {
2100                         warn "vm $vmid - unable to parse value of '$key'\n";
2101                         next;
2102                     }
2103                 }
2104
2105                 if ($key eq 'cdrom') {
2106                     $conf->{ide2} = $value;
2107                 } else {
2108                     $conf->{$key} = $value;
2109                 }
2110             }
2111         }
2112     }
2113
2114     if (defined($descr)) {
2115         $descr =~ s/\s+$//;
2116         $conf->{description} = $descr;
2117     }
2118     delete $res->{snapstate}; # just to be sure
2119
2120     return $res;
2121 }
2122
2123 sub write_vm_config {
2124     my ($filename, $conf) = @_;
2125
2126     delete $conf->{snapstate}; # just to be sure
2127
2128     if ($conf->{cdrom}) {
2129         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2130         $conf->{ide2} = $conf->{cdrom};
2131         delete $conf->{cdrom};
2132     }
2133
2134     # we do not use 'smp' any longer
2135     if ($conf->{sockets}) {
2136         delete $conf->{smp};
2137     } elsif ($conf->{smp}) {
2138         $conf->{sockets} = $conf->{smp};
2139         delete $conf->{cores};
2140         delete $conf->{smp};
2141     }
2142
2143     my $used_volids = {};
2144
2145     my $cleanup_config = sub {
2146         my ($cref, $pending, $snapname) = @_;
2147
2148         foreach my $key (keys %$cref) {
2149             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2150                 $key eq 'snapstate' || $key eq 'pending';
2151             my $value = $cref->{$key};
2152             if ($key eq 'delete') {
2153                 die "propertry 'delete' is only allowed in [PENDING]\n"
2154                     if !$pending;
2155                 # fixme: check syntax?
2156                 next;
2157             }
2158             eval { $value = check_type($key, $value); };
2159             die "unable to parse value of '$key' - $@" if $@;
2160
2161             $cref->{$key} = $value;
2162
2163             if (!$snapname && valid_drivename($key)) {
2164                 my $drive = parse_drive($key, $value);
2165                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2166             }
2167         }
2168     };
2169
2170     &$cleanup_config($conf);
2171
2172     &$cleanup_config($conf->{pending}, 1);
2173
2174     foreach my $snapname (keys %{$conf->{snapshots}}) {
2175         die "internal error" if $snapname eq 'pending';
2176         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2177     }
2178
2179     # remove 'unusedX' settings if we re-add a volume
2180     foreach my $key (keys %$conf) {
2181         my $value = $conf->{$key};
2182         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2183             delete $conf->{$key};
2184         }
2185     }
2186
2187     my $generate_raw_config = sub {
2188         my ($conf, $pending) = @_;
2189
2190         my $raw = '';
2191
2192         # add description as comment to top of file
2193         if (defined(my $descr = $conf->{description})) {
2194             if ($descr) {
2195                 foreach my $cl (split(/\n/, $descr)) {
2196                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2197                 }
2198             } else {
2199                 $raw .= "#\n" if $pending;
2200             }
2201         }
2202
2203         foreach my $key (sort keys %$conf) {
2204             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2205             $raw .= "$key: $conf->{$key}\n";
2206         }
2207         return $raw;
2208     };
2209
2210     my $raw = &$generate_raw_config($conf);
2211
2212     if (scalar(keys %{$conf->{pending}})){
2213         $raw .= "\n[PENDING]\n";
2214         $raw .= &$generate_raw_config($conf->{pending}, 1);
2215     }
2216
2217     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2218         $raw .= "\n[$snapname]\n";
2219         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2220     }
2221
2222     return $raw;
2223 }
2224
2225 sub write_config {
2226     my ($vmid, $conf) = @_;
2227
2228     my $cfspath = cfs_config_path($vmid);
2229
2230     PVE::Cluster::cfs_write_file($cfspath, $conf);
2231 }
2232
2233 sub load_defaults {
2234
2235     my $res = {};
2236
2237     # we use static defaults from our JSON schema configuration
2238     foreach my $key (keys %$confdesc) {
2239         if (defined(my $default = $confdesc->{$key}->{default})) {
2240             $res->{$key} = $default;
2241         }
2242     }
2243
2244     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2245     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2246
2247     return $res;
2248 }
2249
2250 sub config_list {
2251     my $vmlist = PVE::Cluster::get_vmlist();
2252     my $res = {};
2253     return $res if !$vmlist || !$vmlist->{ids};
2254     my $ids = $vmlist->{ids};
2255
2256     foreach my $vmid (keys %$ids) {
2257         my $d = $ids->{$vmid};
2258         next if !$d->{node} || $d->{node} ne $nodename;
2259         next if !$d->{type} || $d->{type} ne 'qemu';
2260         $res->{$vmid}->{exists} = 1;
2261     }
2262     return $res;
2263 }
2264
2265 # test if VM uses local resources (to prevent migration)
2266 sub check_local_resources {
2267     my ($conf, $noerr) = @_;
2268
2269     my $loc_res = 0;
2270
2271     $loc_res = 1 if $conf->{hostusb}; # old syntax
2272     $loc_res = 1 if $conf->{hostpci}; # old syntax
2273
2274     foreach my $k (keys %$conf) {
2275         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2276         # sockets are safe: they will recreated be on the target side post-migrate
2277         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2278         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2279     }
2280
2281     die "VM uses local resources\n" if $loc_res && !$noerr;
2282
2283     return $loc_res;
2284 }
2285
2286 # check if used storages are available on all nodes (use by migrate)
2287 sub check_storage_availability {
2288     my ($storecfg, $conf, $node) = @_;
2289
2290     foreach_drive($conf, sub {
2291         my ($ds, $drive) = @_;
2292
2293         my $volid = $drive->{file};
2294         return if !$volid;
2295
2296         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2297         return if !$sid;
2298
2299         # check if storage is available on both nodes
2300         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2301         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2302    });
2303 }
2304
2305 # list nodes where all VM images are available (used by has_feature API)
2306 sub shared_nodes {
2307     my ($conf, $storecfg) = @_;
2308
2309     my $nodelist = PVE::Cluster::get_nodelist();
2310     my $nodehash = { map { $_ => 1 } @$nodelist };
2311     my $nodename = PVE::INotify::nodename();
2312
2313     foreach_drive($conf, sub {
2314         my ($ds, $drive) = @_;
2315
2316         my $volid = $drive->{file};
2317         return if !$volid;
2318
2319         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2320         if ($storeid) {
2321             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2322             if ($scfg->{disable}) {
2323                 $nodehash = {};
2324             } elsif (my $avail = $scfg->{nodes}) {
2325                 foreach my $node (keys %$nodehash) {
2326                     delete $nodehash->{$node} if !$avail->{$node};
2327                 }
2328             } elsif (!$scfg->{shared}) {
2329                 foreach my $node (keys %$nodehash) {
2330                     delete $nodehash->{$node} if $node ne $nodename
2331                 }
2332             }
2333         }
2334     });
2335
2336     return $nodehash
2337 }
2338
2339 sub check_lock {
2340     my ($conf) = @_;
2341
2342     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2343 }
2344
2345 sub check_cmdline {
2346     my ($pidfile, $pid) = @_;
2347
2348     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2349     if (defined($fh)) {
2350         my $line = <$fh>;
2351         $fh->close;
2352         return undef if !$line;
2353         my @param = split(/\0/, $line);
2354
2355         my $cmd = $param[0];
2356         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2357
2358         for (my $i = 0; $i < scalar (@param); $i++) {
2359             my $p = $param[$i];
2360             next if !$p;
2361             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2362                 my $p = $param[$i+1];
2363                 return 1 if $p && ($p eq $pidfile);
2364                 return undef;
2365             }
2366         }
2367     }
2368     return undef;
2369 }
2370
2371 sub check_running {
2372     my ($vmid, $nocheck, $node) = @_;
2373
2374     my $filename = config_file($vmid, $node);
2375
2376     die "unable to find configuration file for VM $vmid - no such machine\n"
2377         if !$nocheck && ! -f $filename;
2378
2379     my $pidfile = pidfile_name($vmid);
2380
2381     if (my $fd = IO::File->new("<$pidfile")) {
2382         my $st = stat($fd);
2383         my $line = <$fd>;
2384         close($fd);
2385
2386         my $mtime = $st->mtime;
2387         if ($mtime > time()) {
2388             warn "file '$filename' modified in future\n";
2389         }
2390
2391         if ($line =~ m/^(\d+)$/) {
2392             my $pid = $1;
2393             if (check_cmdline($pidfile, $pid)) {
2394                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2395                     return $pid;
2396                 }
2397             }
2398         }
2399     }
2400
2401     return undef;
2402 }
2403
2404 sub vzlist {
2405
2406     my $vzlist = config_list();
2407
2408     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2409
2410     while (defined(my $de = $fd->read)) {
2411         next if $de !~ m/^(\d+)\.pid$/;
2412         my $vmid = $1;
2413         next if !defined($vzlist->{$vmid});
2414         if (my $pid = check_running($vmid)) {
2415             $vzlist->{$vmid}->{pid} = $pid;
2416         }
2417     }
2418
2419     return $vzlist;
2420 }
2421
2422 sub disksize {
2423     my ($storecfg, $conf) = @_;
2424
2425     my $bootdisk = $conf->{bootdisk};
2426     return undef if !$bootdisk;
2427     return undef if !valid_drivename($bootdisk);
2428
2429     return undef if !$conf->{$bootdisk};
2430
2431     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2432     return undef if !defined($drive);
2433
2434     return undef if drive_is_cdrom($drive);
2435
2436     my $volid = $drive->{file};
2437     return undef if !$volid;
2438
2439     return $drive->{size};
2440 }
2441
2442 my $last_proc_pid_stat;
2443
2444 # get VM status information
2445 # This must be fast and should not block ($full == false)
2446 # We only query KVM using QMP if $full == true (this can be slow)
2447 sub vmstatus {
2448     my ($opt_vmid, $full) = @_;
2449
2450     my $res = {};
2451
2452     my $storecfg = PVE::Storage::config();
2453
2454     my $list = vzlist();
2455     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2456
2457     my $cpucount = $cpuinfo->{cpus} || 1;
2458
2459     foreach my $vmid (keys %$list) {
2460         next if $opt_vmid && ($vmid ne $opt_vmid);
2461
2462         my $cfspath = cfs_config_path($vmid);
2463         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2464
2465         my $d = {};
2466         $d->{pid} = $list->{$vmid}->{pid};
2467
2468         # fixme: better status?
2469         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2470
2471         my $size = disksize($storecfg, $conf);
2472         if (defined($size)) {
2473             $d->{disk} = 0; # no info available
2474             $d->{maxdisk} = $size;
2475         } else {
2476             $d->{disk} = 0;
2477             $d->{maxdisk} = 0;
2478         }
2479
2480         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2481         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2482         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2483
2484         $d->{name} = $conf->{name} || "VM $vmid";
2485         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2486
2487         if ($conf->{balloon}) {
2488             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2489             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2490         }
2491
2492         $d->{uptime} = 0;
2493         $d->{cpu} = 0;
2494         $d->{mem} = 0;
2495
2496         $d->{netout} = 0;
2497         $d->{netin} = 0;
2498
2499         $d->{diskread} = 0;
2500         $d->{diskwrite} = 0;
2501
2502         $d->{template} = is_template($conf);
2503
2504         $res->{$vmid} = $d;
2505     }
2506
2507     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2508     foreach my $dev (keys %$netdev) {
2509         next if $dev !~ m/^tap([1-9]\d*)i/;
2510         my $vmid = $1;
2511         my $d = $res->{$vmid};
2512         next if !$d;
2513
2514         $d->{netout} += $netdev->{$dev}->{receive};
2515         $d->{netin} += $netdev->{$dev}->{transmit};
2516
2517         if ($full) {
2518             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2519             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2520         }
2521
2522     }
2523
2524     my $ctime = gettimeofday;
2525
2526     foreach my $vmid (keys %$list) {
2527
2528         my $d = $res->{$vmid};
2529         my $pid = $d->{pid};
2530         next if !$pid;
2531
2532         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2533         next if !$pstat; # not running
2534
2535         my $used = $pstat->{utime} + $pstat->{stime};
2536
2537         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2538
2539         if ($pstat->{vsize}) {
2540             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2541         }
2542
2543         my $old = $last_proc_pid_stat->{$pid};
2544         if (!$old) {
2545             $last_proc_pid_stat->{$pid} = {
2546                 time => $ctime,
2547                 used => $used,
2548                 cpu => 0,
2549             };
2550             next;
2551         }
2552
2553         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2554
2555         if ($dtime > 1000) {
2556             my $dutime = $used -  $old->{used};
2557
2558             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2559             $last_proc_pid_stat->{$pid} = {
2560                 time => $ctime,
2561                 used => $used,
2562                 cpu => $d->{cpu},
2563             };
2564         } else {
2565             $d->{cpu} = $old->{cpu};
2566         }
2567     }
2568
2569     return $res if !$full;
2570
2571     my $qmpclient = PVE::QMPClient->new();
2572
2573     my $ballooncb = sub {
2574         my ($vmid, $resp) = @_;
2575
2576         my $info = $resp->{'return'};
2577         return if !$info->{max_mem};
2578
2579         my $d = $res->{$vmid};
2580
2581         # use memory assigned to VM
2582         $d->{maxmem} = $info->{max_mem};
2583         $d->{balloon} = $info->{actual};
2584
2585         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2586             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2587             $d->{freemem} = $info->{free_mem};
2588         }
2589
2590         $d->{ballooninfo} = $info;
2591     };
2592
2593     my $blockstatscb = sub {
2594         my ($vmid, $resp) = @_;
2595         my $data = $resp->{'return'} || [];
2596         my $totalrdbytes = 0;
2597         my $totalwrbytes = 0;
2598
2599         for my $blockstat (@$data) {
2600             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2601             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2602
2603             $blockstat->{device} =~ s/drive-//;
2604             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2605         }
2606         $res->{$vmid}->{diskread} = $totalrdbytes;
2607         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2608     };
2609
2610     my $statuscb = sub {
2611         my ($vmid, $resp) = @_;
2612
2613         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2614         # this fails if ballon driver is not loaded, so this must be
2615         # the last commnand (following command are aborted if this fails).
2616         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2617
2618         my $status = 'unknown';
2619         if (!defined($status = $resp->{'return'}->{status})) {
2620             warn "unable to get VM status\n";
2621             return;
2622         }
2623
2624         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2625     };
2626
2627     foreach my $vmid (keys %$list) {
2628         next if $opt_vmid && ($vmid ne $opt_vmid);
2629         next if !$res->{$vmid}->{pid}; # not running
2630         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2631     }
2632
2633     $qmpclient->queue_execute(undef, 1);
2634
2635     foreach my $vmid (keys %$list) {
2636         next if $opt_vmid && ($vmid ne $opt_vmid);
2637         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2638     }
2639
2640     return $res;
2641 }
2642
2643 sub foreach_dimm {
2644     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2645
2646     my $dimm_id = 0;
2647     my $current_size = 1024;
2648     my $dimm_size = 512;
2649     return if $current_size == $memory;
2650
2651     for (my $j = 0; $j < 8; $j++) {
2652         for (my $i = 0; $i < 32; $i++) {
2653             my $name = "dimm${dimm_id}";
2654             $dimm_id++;
2655             my $numanode = $i % $sockets;
2656             $current_size += $dimm_size;
2657             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2658             return  $current_size if $current_size >= $memory;
2659         }
2660         $dimm_size *= 2;
2661     }
2662 }
2663
2664 sub foreach_reverse_dimm {
2665     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2666
2667     my $dimm_id = 253;
2668     my $current_size = 4177920;
2669     my $dimm_size = 65536;
2670     return if $current_size == $memory;
2671
2672     for (my $j = 0; $j < 8; $j++) {
2673         for (my $i = 0; $i < 32; $i++) {
2674             my $name = "dimm${dimm_id}";
2675             $dimm_id--;
2676             my $numanode = $i % $sockets;
2677             $current_size -= $dimm_size;
2678             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2679             return  $current_size if $current_size <= $memory;
2680         }
2681         $dimm_size /= 2;
2682     }
2683 }
2684
2685 sub foreach_drive {
2686     my ($conf, $func) = @_;
2687
2688     foreach my $ds (keys %$conf) {
2689         next if !valid_drivename($ds);
2690
2691         my $drive = parse_drive($ds, $conf->{$ds});
2692         next if !$drive;
2693
2694         &$func($ds, $drive);
2695     }
2696 }
2697
2698 sub foreach_volid {
2699     my ($conf, $func) = @_;
2700
2701     my $volhash = {};
2702
2703     my $test_volid = sub {
2704         my ($volid, $is_cdrom) = @_;
2705
2706         return if !$volid;
2707
2708         $volhash->{$volid} = $is_cdrom || 0;
2709     };
2710
2711     foreach_drive($conf, sub {
2712         my ($ds, $drive) = @_;
2713         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2714     });
2715
2716     foreach my $snapname (keys %{$conf->{snapshots}}) {
2717         my $snap = $conf->{snapshots}->{$snapname};
2718         &$test_volid($snap->{vmstate}, 0);
2719         foreach_drive($snap, sub {
2720             my ($ds, $drive) = @_;
2721             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2722         });
2723     }
2724
2725     foreach my $volid (keys %$volhash) {
2726         &$func($volid, $volhash->{$volid});
2727     }
2728 }
2729
2730 sub vga_conf_has_spice {
2731     my ($vga) = @_;
2732
2733     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2734
2735     return $1 || 1;
2736 }
2737
2738 sub config_to_command {
2739     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2740
2741     my $cmd = [];
2742     my $globalFlags = [];
2743     my $machineFlags = [];
2744     my $rtcFlags = [];
2745     my $cpuFlags = [];
2746     my $devices = [];
2747     my $pciaddr = '';
2748     my $bridges = {};
2749     my $kvmver = kvm_user_version();
2750     my $vernum = 0; # unknown
2751     my $ostype = $conf->{ostype};
2752     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2753         $vernum = $1*1000000+$2*1000;
2754     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2755         $vernum = $1*1000000+$2*1000+$3;
2756     }
2757
2758     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2759
2760     my $have_ovz = -f '/proc/vz/vestat';
2761
2762     my $q35 = machine_type_is_q35($conf);
2763     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2764     my $machine_type = $forcemachine || $conf->{machine};
2765     my $use_old_bios_files = undef;
2766     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2767
2768     my $cpuunits = defined($conf->{cpuunits}) ?
2769             $conf->{cpuunits} : $defaults->{cpuunits};
2770
2771     push @$cmd, '/usr/bin/systemd-run';
2772     push @$cmd, '--scope';
2773     push @$cmd, '--slice', "qemu";
2774     push @$cmd, '--unit', $vmid;
2775     # set KillMode=none, so that systemd don't kill those scopes
2776     # at shutdown (pve-manager service should stop the VMs instead)
2777     push @$cmd, '-p', "KillMode=none";
2778     push @$cmd, '-p', "CPUShares=$cpuunits";
2779     if ($conf->{cpulimit}) {
2780         my $cpulimit = int($conf->{cpulimit} * 100);
2781         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2782     }
2783
2784     push @$cmd, '/usr/bin/kvm';
2785
2786     push @$cmd, '-id', $vmid;
2787
2788     my $use_virtio = 0;
2789
2790     my $qmpsocket = qmp_socket($vmid);
2791     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2792     push @$cmd, '-mon', "chardev=qmp,mode=control";
2793
2794
2795     push @$cmd, '-pidfile' , pidfile_name($vmid);
2796
2797     push @$cmd, '-daemonize';
2798
2799     if ($conf->{smbios1}) {
2800         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2801     }
2802
2803     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2804         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2805         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2806         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2807         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2808         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2809         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2810     }
2811
2812     if ($q35) {
2813         # the q35 chipset support native usb2, so we enable usb controller
2814         # by default for this machine type
2815         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2816     } else {
2817         $pciaddr = print_pci_addr("piix3", $bridges);
2818         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2819
2820         my $use_usb2 = 0;
2821         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2822             next if !$conf->{"usb$i"};
2823             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2824             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2825             $use_usb2 = 1;
2826         }
2827         # include usb device config
2828         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2829     }
2830
2831     # add usb3 controller if needed
2832
2833     my $use_usb3 = 0;
2834     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2835         next if !$conf->{"usb$i"};
2836         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2837         next if !$d || !$d->{usb3};
2838         $use_usb3 = 1;
2839     }
2840
2841     $pciaddr = print_pci_addr("xhci", $bridges);
2842     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2843
2844     my $vga = $conf->{vga};
2845
2846     my $qxlnum = vga_conf_has_spice($vga);
2847     $vga = 'qxl' if $qxlnum;
2848
2849     if (!$vga) {
2850         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2851                                 $conf->{ostype} eq 'win7' ||
2852                                 $conf->{ostype} eq 'w2k8')) {
2853             $vga = 'std';
2854         } else {
2855             $vga = 'cirrus';
2856         }
2857     }
2858
2859     # enable absolute mouse coordinates (needed by vnc)
2860     my $tablet;
2861     if (defined($conf->{tablet})) {
2862         $tablet = $conf->{tablet};
2863     } else {
2864         $tablet = $defaults->{tablet};
2865         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2866         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2867     }
2868
2869     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2870
2871     my $kvm_off = 0;
2872     # host pci devices
2873     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2874         my $d = parse_hostpci($conf->{"hostpci$i"});
2875         next if !$d;
2876
2877         my $pcie = $d->{pcie};
2878         if($pcie){
2879             die "q35 machine model is not enabled" if !$q35;
2880             $pciaddr = print_pcie_addr("hostpci$i");
2881         }else{
2882             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2883         }
2884
2885         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2886         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2887         if ($xvga && $xvga ne '') {
2888             $kvm_off = 1;
2889             $vga = 'none';
2890             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2891                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2892             }
2893             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2894                 $xvga = "";
2895             }
2896         }
2897         my $pcidevices = $d->{pciid};
2898         my $multifunction = 1 if @$pcidevices > 1;
2899
2900         my $j=0;
2901         foreach my $pcidevice (@$pcidevices) {
2902
2903             my $id = "hostpci$i";
2904             $id .= ".$j" if $multifunction;
2905             my $addr = $pciaddr;
2906             $addr .= ".$j" if $multifunction;
2907             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2908
2909             if($j == 0){
2910                 $devicestr .= "$rombar$xvga";
2911                 $devicestr .= ",multifunction=on" if $multifunction;
2912             }
2913
2914             push @$devices, '-device', $devicestr;
2915             $j++;
2916         }
2917     }
2918
2919     # usb devices
2920     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2921         next if !$conf->{"usb$i"};
2922         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2923         next if !$d;
2924
2925         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2926         my $usbbus = '';
2927         if (defined($d->{usb3}) && $d->{usb3}) {
2928             $usbbus = ',bus=xhci.0';
2929         }
2930
2931         if (defined($d->{host})) {
2932             $d = parse_usb_device($d->{host});
2933             if (defined($d->{vendorid}) && defined($d->{productid})) {
2934                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2935             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2936                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2937             } elsif (defined($d->{spice}) && $d->{spice}) {
2938                 # usb redir support for spice, currently no usb3
2939                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2940                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2941             }
2942         }
2943     }
2944
2945     # serial devices
2946     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2947         if (my $path = $conf->{"serial$i"}) {
2948             if ($path eq 'socket') {
2949                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2950                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2951                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2952             } else {
2953                 die "no such serial device\n" if ! -c $path;
2954                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2955                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2956             }
2957         }
2958     }
2959
2960     # parallel devices
2961     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2962         if (my $path = $conf->{"parallel$i"}) {
2963             die "no such parallel device\n" if ! -c $path;
2964             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2965             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2966             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2967         }
2968     }
2969
2970     my $vmname = $conf->{name} || "vm$vmid";
2971
2972     push @$cmd, '-name', $vmname;
2973
2974     my $sockets = 1;
2975     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2976     $sockets = $conf->{sockets} if  $conf->{sockets};
2977
2978     my $cores = $conf->{cores} || 1;
2979
2980     my $maxcpus = $sockets * $cores;
2981
2982     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2983
2984     my $allowed_vcpus = $cpuinfo->{cpus};
2985
2986     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2987         if ($allowed_vcpus < $maxcpus);
2988
2989     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2990
2991     push @$cmd, '-nodefaults';
2992
2993     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2994
2995     my $bootindex_hash = {};
2996     my $i = 1;
2997     foreach my $o (split(//, $bootorder)) {
2998         $bootindex_hash->{$o} = $i*100;
2999         $i++;
3000     }
3001
3002     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
3003
3004     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3005
3006     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3007
3008     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3009
3010     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3011         my $socket = vnc_socket($vmid);
3012         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3013     } else {
3014         push @$cmd, '-nographic';
3015     }
3016
3017     # time drift fix
3018     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3019
3020     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3021     my $useLocaltime = $conf->{localtime};
3022
3023     if ($ostype) {
3024         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3025
3026         if ($ostype =~ m/^w/) { # windows
3027             $useLocaltime = 1 if !defined($conf->{localtime});
3028
3029             # use time drift fix when acpi is enabled
3030             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3031                 $tdf = 1 if !defined($conf->{tdf});
3032             }
3033         }
3034
3035         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3036             $ostype eq 'wvista') {
3037             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3038             push @$cmd, '-no-hpet';
3039             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3040                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3041                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3042                 push @$cpuFlags , 'hv_time' if !$nokvm;
3043
3044             } else {
3045                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3046             }
3047         }
3048
3049         if ($ostype eq 'win7' || $ostype eq 'win8') {
3050             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3051         }
3052     }
3053
3054     push @$rtcFlags, 'driftfix=slew' if $tdf;
3055
3056     if ($nokvm) {
3057         push @$machineFlags, 'accel=tcg';
3058     } else {
3059         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3060     }
3061
3062     if ($machine_type) {
3063         push @$machineFlags, "type=${machine_type}";
3064     }
3065
3066     if ($conf->{startdate}) {
3067         push @$rtcFlags, "base=$conf->{startdate}";
3068     } elsif ($useLocaltime) {
3069         push @$rtcFlags, 'base=localtime';
3070     }
3071
3072     my $cpu = $nokvm ? "qemu64" : "kvm64";
3073     if (my $cputype = $conf->{cpu}) {
3074         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3075             or die "Cannot parse cpu description: $cputype\n";
3076         $cpu = $cpuconf->{cputype};
3077         $kvm_off = 1 if $cpuconf->{hidden};
3078     }
3079
3080     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3081
3082     push @$cpuFlags , '-x2apic'
3083         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3084
3085     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3086
3087     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3088
3089     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3090
3091         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3092         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3093     }
3094
3095     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3096
3097     push @$cpuFlags, 'kvm=off' if $kvm_off;
3098
3099     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3100
3101     push @$cmd, '-cpu', $cpu;
3102
3103     my $memory = $conf->{memory} || $defaults->{memory};
3104     my $static_memory = 0;
3105     my $dimm_memory = 0;
3106
3107     if ($hotplug_features->{memory}) {
3108         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3109         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3110         $static_memory = $STATICMEM;
3111         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3112         $dimm_memory = $memory - $static_memory;
3113         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3114
3115     } else {
3116
3117         $static_memory = $memory;
3118         push @$cmd, '-m', $static_memory;
3119     }
3120
3121     if ($conf->{numa}) {
3122
3123         my $numa_totalmemory = undef;
3124         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3125             next if !$conf->{"numa$i"};
3126             my $numa = parse_numa($conf->{"numa$i"});
3127             next if !$numa;
3128             # memory
3129             die "missing numa node$i memory value\n" if !$numa->{memory};
3130             my $numa_memory = $numa->{memory};
3131             $numa_totalmemory += $numa_memory;
3132             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3133
3134             # cpus
3135             my $cpus_start = $numa->{cpus}->{start};
3136             die "missing numa node$i cpus\n" if !defined($cpus_start);
3137             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3138             my $cpus = $cpus_start;
3139             if (defined($cpus_end)) {
3140                 $cpus .= "-$cpus_end";
3141                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3142             }
3143
3144             # hostnodes
3145             my $hostnodes_start = $numa->{hostnodes}->{start};
3146             if (defined($hostnodes_start)) {
3147                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3148                 my $hostnodes = $hostnodes_start;
3149                 if (defined($hostnodes_end)) {
3150                     $hostnodes .= "-$hostnodes_end";
3151                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3152                 }
3153
3154                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3155                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3156                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3157                 }
3158
3159                 # policy
3160                 my $policy = $numa->{policy};
3161                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3162                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3163             }
3164
3165             push @$cmd, '-object', $numa_object;
3166             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3167         }
3168
3169         die "total memory for NUMA nodes must be equal to vm static memory\n"
3170             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3171
3172         #if no custom tology, we split memory and cores across numa nodes
3173         if(!$numa_totalmemory) {
3174
3175             my $numa_memory = ($static_memory / $sockets) . "M";
3176
3177             for (my $i = 0; $i < $sockets; $i++)  {
3178
3179                 my $cpustart = ($cores * $i);
3180                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3181                 my $cpus = $cpustart;
3182                 $cpus .= "-$cpuend" if $cpuend;
3183
3184                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3185                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3186             }
3187         }
3188     }
3189
3190     if ($hotplug_features->{memory}) {
3191         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3192             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3193             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3194             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3195
3196             #if dimm_memory is not aligned to dimm map
3197             if($current_size > $memory) {
3198                  $conf->{memory} = $current_size;
3199                  write_config($vmid, $conf);
3200             }
3201         });
3202     }
3203
3204     push @$cmd, '-S' if $conf->{freeze};
3205
3206     # set keyboard layout
3207     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3208     push @$cmd, '-k', $kb if $kb;
3209
3210     # enable sound
3211     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3212     #push @$cmd, '-soundhw', 'es1370';
3213     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3214
3215     if($conf->{agent}) {
3216         my $qgasocket = qmp_socket($vmid, 1);
3217         my $pciaddr = print_pci_addr("qga0", $bridges);
3218         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3219         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3220         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3221     }
3222
3223     my $spice_port;
3224
3225     if ($qxlnum) {
3226         if ($qxlnum > 1) {
3227             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3228                 for(my $i = 1; $i < $qxlnum; $i++){
3229                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3230                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3231                 }
3232             } else {
3233                 # assume other OS works like Linux
3234                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3235                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3236             }
3237         }
3238
3239         my $pciaddr = print_pci_addr("spice", $bridges);
3240
3241         my $nodename = PVE::INotify::nodename();
3242         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3243         $spice_port = PVE::Tools::next_spice_port($pfamily);
3244
3245         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3246
3247         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3248         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3249         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3250     }
3251
3252     # enable balloon by default, unless explicitly disabled
3253     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3254         $pciaddr = print_pci_addr("balloon0", $bridges);
3255         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3256     }
3257
3258     if ($conf->{watchdog}) {
3259         my $wdopts = parse_watchdog($conf->{watchdog});
3260         $pciaddr = print_pci_addr("watchdog", $bridges);
3261         my $watchdog = $wdopts->{model} || 'i6300esb';
3262         push @$devices, '-device', "$watchdog$pciaddr";
3263         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3264     }
3265
3266     my $vollist = [];
3267     my $scsicontroller = {};
3268     my $ahcicontroller = {};
3269     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3270
3271     # Add iscsi initiator name if available
3272     if (my $initiator = get_initiator_name()) {
3273         push @$devices, '-iscsi', "initiator-name=$initiator";
3274     }
3275
3276     foreach_drive($conf, sub {
3277         my ($ds, $drive) = @_;
3278
3279         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3280             push @$vollist, $drive->{file};
3281         }
3282
3283         $use_virtio = 1 if $ds =~ m/^virtio/;
3284
3285         if (drive_is_cdrom ($drive)) {
3286             if ($bootindex_hash->{d}) {
3287                 $drive->{bootindex} = $bootindex_hash->{d};
3288                 $bootindex_hash->{d} += 1;
3289             }
3290         } else {
3291             if ($bootindex_hash->{c}) {
3292                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3293                 $bootindex_hash->{c} += 1;
3294             }
3295         }
3296
3297         if($drive->{interface} eq 'virtio'){
3298            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3299         }
3300
3301         if ($drive->{interface} eq 'scsi') {
3302
3303             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3304
3305             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3306             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3307
3308             my $iothread = '';
3309             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3310                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3311                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3312             }
3313
3314             my $queues = '';
3315             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3316                 $queues = ",num_queues=$drive->{queues}";
3317             }
3318
3319             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3320             $scsicontroller->{$controller}=1;
3321         }
3322
3323         if ($drive->{interface} eq 'sata') {
3324            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3325            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3326            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3327            $ahcicontroller->{$controller}=1;
3328         }
3329
3330         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3331         push @$devices, '-drive',$drive_cmd;
3332         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3333     });
3334
3335     for (my $i = 0; $i < $MAX_NETS; $i++) {
3336          next if !$conf->{"net$i"};
3337          my $d = parse_net($conf->{"net$i"});
3338          next if !$d;
3339
3340          $use_virtio = 1 if $d->{model} eq 'virtio';
3341
3342          if ($bootindex_hash->{n}) {
3343             $d->{bootindex} = $bootindex_hash->{n};
3344             $bootindex_hash->{n} += 1;
3345          }
3346
3347          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3348          push @$devices, '-netdev', $netdevfull;
3349
3350          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3351          push @$devices, '-device', $netdevicefull;
3352     }
3353
3354     if (!$q35) {
3355         # add pci bridges
3356         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3357            $bridges->{1} = 1;
3358            $bridges->{2} = 1;
3359         }
3360
3361         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3362
3363         while (my ($k, $v) = each %$bridges) {
3364             $pciaddr = print_pci_addr("pci.$k");
3365             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3366         }
3367     }
3368
3369     # add custom args
3370     if ($conf->{args}) {
3371         my $aa = PVE::Tools::split_args($conf->{args});
3372         push @$cmd, @$aa;
3373     }
3374
3375     push @$cmd, @$devices;
3376     push @$cmd, '-rtc', join(',', @$rtcFlags)
3377         if scalar(@$rtcFlags);
3378     push @$cmd, '-machine', join(',', @$machineFlags)
3379         if scalar(@$machineFlags);
3380     push @$cmd, '-global', join(',', @$globalFlags)
3381         if scalar(@$globalFlags);
3382
3383     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3384 }
3385
3386 sub vnc_socket {
3387     my ($vmid) = @_;
3388     return "${var_run_tmpdir}/$vmid.vnc";
3389 }
3390
3391 sub spice_port {
3392     my ($vmid) = @_;
3393
3394     my $res = vm_mon_cmd($vmid, 'query-spice');
3395
3396     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3397 }
3398
3399 sub qmp_socket {
3400     my ($vmid, $qga) = @_;
3401     my $sockettype = $qga ? 'qga' : 'qmp';
3402     return "${var_run_tmpdir}/$vmid.$sockettype";
3403 }
3404
3405 sub pidfile_name {
3406     my ($vmid) = @_;
3407     return "${var_run_tmpdir}/$vmid.pid";
3408 }
3409
3410 sub vm_devices_list {
3411     my ($vmid) = @_;
3412
3413     my $res = vm_mon_cmd($vmid, 'query-pci');
3414     my $devices = {};
3415     foreach my $pcibus (@$res) {
3416         foreach my $device (@{$pcibus->{devices}}) {
3417             next if !$device->{'qdev_id'};
3418             if ($device->{'pci_bridge'}) {
3419                 $devices->{$device->{'qdev_id'}} = 1;
3420                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3421                     next if !$bridge_device->{'qdev_id'};
3422                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3423                     $devices->{$device->{'qdev_id'}}++;
3424                 }
3425             } else {
3426                 $devices->{$device->{'qdev_id'}} = 1;
3427             }
3428         }
3429     }
3430
3431     my $resblock = vm_mon_cmd($vmid, 'query-block');
3432     foreach my $block (@$resblock) {
3433         if($block->{device} =~ m/^drive-(\S+)/){
3434                 $devices->{$1} = 1;
3435         }
3436     }
3437
3438     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3439     foreach my $mice (@$resmice) {
3440         if ($mice->{name} eq 'QEMU HID Tablet') {
3441             $devices->{tablet} = 1;
3442             last;
3443         }
3444     }
3445
3446     return $devices;
3447 }
3448
3449 sub vm_deviceplug {
3450     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3451
3452     my $q35 = machine_type_is_q35($conf);
3453
3454     my $devices_list = vm_devices_list($vmid);
3455     return 1 if defined($devices_list->{$deviceid});
3456
3457     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3458
3459     if ($deviceid eq 'tablet') {
3460
3461         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3462
3463     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3464
3465         qemu_iothread_add($vmid, $deviceid, $device);
3466
3467         qemu_driveadd($storecfg, $vmid, $device);
3468         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3469
3470         qemu_deviceadd($vmid, $devicefull);
3471         eval { qemu_deviceaddverify($vmid, $deviceid); };
3472         if (my $err = $@) {
3473             eval { qemu_drivedel($vmid, $deviceid); };
3474             warn $@ if $@;
3475             die $err;
3476         }
3477
3478     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3479
3480
3481         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3482         my $pciaddr = print_pci_addr($deviceid);
3483         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3484
3485         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3486
3487         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3488             qemu_iothread_add($vmid, $deviceid, $device);
3489             $devicefull .= ",iothread=iothread-$deviceid";
3490         }
3491
3492         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3493             $devicefull .= ",num_queues=$device->{queues}";
3494         }
3495
3496         qemu_deviceadd($vmid, $devicefull);
3497         qemu_deviceaddverify($vmid, $deviceid);
3498
3499     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3500
3501         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3502         qemu_driveadd($storecfg, $vmid, $device);
3503
3504         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3505         eval { qemu_deviceadd($vmid, $devicefull); };
3506         if (my $err = $@) {
3507             eval { qemu_drivedel($vmid, $deviceid); };
3508             warn $@ if $@;
3509             die $err;
3510         }
3511
3512     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3513
3514         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3515
3516         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3517         my $use_old_bios_files = undef;
3518         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3519
3520         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3521         qemu_deviceadd($vmid, $netdevicefull);
3522         eval { qemu_deviceaddverify($vmid, $deviceid); };
3523         if (my $err = $@) {
3524             eval { qemu_netdevdel($vmid, $deviceid); };
3525             warn $@ if $@;
3526             die $err;
3527         }
3528
3529     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3530
3531         my $bridgeid = $2;
3532         my $pciaddr = print_pci_addr($deviceid);
3533         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3534
3535         qemu_deviceadd($vmid, $devicefull);
3536         qemu_deviceaddverify($vmid, $deviceid);
3537
3538     } else {
3539         die "can't hotplug device '$deviceid'\n";
3540     }
3541
3542     return 1;
3543 }
3544
3545 # fixme: this should raise exceptions on error!
3546 sub vm_deviceunplug {
3547     my ($vmid, $conf, $deviceid) = @_;
3548
3549     my $devices_list = vm_devices_list($vmid);
3550     return 1 if !defined($devices_list->{$deviceid});
3551
3552     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3553
3554     if ($deviceid eq 'tablet') {
3555
3556         qemu_devicedel($vmid, $deviceid);
3557
3558     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3559
3560         qemu_devicedel($vmid, $deviceid);
3561         qemu_devicedelverify($vmid, $deviceid);
3562         qemu_drivedel($vmid, $deviceid);
3563         qemu_iothread_del($conf, $vmid, $deviceid);
3564
3565     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3566
3567         qemu_devicedel($vmid, $deviceid);
3568         qemu_devicedelverify($vmid, $deviceid);
3569         qemu_iothread_del($conf, $vmid, $deviceid);
3570
3571     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3572
3573         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3574         my $device = parse_drive($deviceid, $conf->{$deviceid});
3575         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3576
3577         qemu_devicedel($vmid, $deviceid);
3578         qemu_drivedel($vmid, $deviceid);
3579         qemu_deletescsihw($conf, $vmid, $deviceid);
3580
3581     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3582
3583         qemu_devicedel($vmid, $deviceid);
3584         qemu_devicedelverify($vmid, $deviceid);
3585         qemu_netdevdel($vmid, $deviceid);
3586
3587     } else {
3588         die "can't unplug device '$deviceid'\n";
3589     }
3590
3591     return 1;
3592 }
3593
3594 sub qemu_deviceadd {
3595     my ($vmid, $devicefull) = @_;
3596
3597     $devicefull = "driver=".$devicefull;
3598     my %options =  split(/[=,]/, $devicefull);
3599
3600     vm_mon_cmd($vmid, "device_add" , %options);
3601 }
3602
3603 sub qemu_devicedel {
3604     my ($vmid, $deviceid) = @_;
3605
3606     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3607 }
3608
3609 sub qemu_iothread_add {
3610     my($vmid, $deviceid, $device) = @_;
3611
3612     if ($device->{iothread}) {
3613         my $iothreads = vm_iothreads_list($vmid);
3614         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3615     }
3616 }
3617
3618 sub qemu_iothread_del {
3619     my($conf, $vmid, $deviceid) = @_;
3620
3621     my $device = parse_drive($deviceid, $conf->{$deviceid});
3622     if ($device->{iothread}) {
3623         my $iothreads = vm_iothreads_list($vmid);
3624         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3625     }
3626 }
3627
3628 sub qemu_objectadd {
3629     my($vmid, $objectid, $qomtype) = @_;
3630
3631     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3632
3633     return 1;
3634 }
3635
3636 sub qemu_objectdel {
3637     my($vmid, $objectid) = @_;
3638
3639     vm_mon_cmd($vmid, "object-del", id => $objectid);
3640
3641     return 1;
3642 }
3643
3644 sub qemu_driveadd {
3645     my ($storecfg, $vmid, $device) = @_;
3646
3647     my $drive = print_drive_full($storecfg, $vmid, $device);
3648     $drive =~ s/\\/\\\\/g;
3649     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3650
3651     # If the command succeeds qemu prints: "OK"
3652     return 1 if $ret =~ m/OK/s;
3653
3654     die "adding drive failed: $ret\n";
3655 }
3656
3657 sub qemu_drivedel {
3658     my($vmid, $deviceid) = @_;
3659
3660     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3661     $ret =~ s/^\s+//;
3662
3663     return 1 if $ret eq "";
3664
3665     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3666     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3667
3668     die "deleting drive $deviceid failed : $ret\n";
3669 }
3670
3671 sub qemu_deviceaddverify {
3672     my ($vmid, $deviceid) = @_;
3673
3674     for (my $i = 0; $i <= 5; $i++) {
3675          my $devices_list = vm_devices_list($vmid);
3676          return 1 if defined($devices_list->{$deviceid});
3677          sleep 1;
3678     }
3679
3680     die "error on hotplug device '$deviceid'\n";
3681 }
3682
3683
3684 sub qemu_devicedelverify {
3685     my ($vmid, $deviceid) = @_;
3686
3687     # need to verify that the device is correctly removed as device_del
3688     # is async and empty return is not reliable
3689
3690     for (my $i = 0; $i <= 5; $i++) {
3691          my $devices_list = vm_devices_list($vmid);
3692          return 1 if !defined($devices_list->{$deviceid});
3693          sleep 1;
3694     }
3695
3696     die "error on hot-unplugging device '$deviceid'\n";
3697 }
3698
3699 sub qemu_findorcreatescsihw {
3700     my ($storecfg, $conf, $vmid, $device) = @_;
3701
3702     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3703
3704     my $scsihwid="$controller_prefix$controller";
3705     my $devices_list = vm_devices_list($vmid);
3706
3707     if(!defined($devices_list->{$scsihwid})) {
3708         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3709     }
3710
3711     return 1;
3712 }
3713
3714 sub qemu_deletescsihw {
3715     my ($conf, $vmid, $opt) = @_;
3716
3717     my $device = parse_drive($opt, $conf->{$opt});
3718
3719     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3720         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3721         return 1;
3722     }
3723
3724     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3725
3726     my $devices_list = vm_devices_list($vmid);
3727     foreach my $opt (keys %{$devices_list}) {
3728         if (PVE::QemuServer::valid_drivename($opt)) {
3729             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3730             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3731                 return 1;
3732             }
3733         }
3734     }
3735
3736     my $scsihwid="scsihw$controller";
3737
3738     vm_deviceunplug($vmid, $conf, $scsihwid);
3739
3740     return 1;
3741 }
3742
3743 sub qemu_add_pci_bridge {
3744     my ($storecfg, $conf, $vmid, $device) = @_;
3745
3746     my $bridges = {};
3747
3748     my $bridgeid;
3749
3750     print_pci_addr($device, $bridges);
3751
3752     while (my ($k, $v) = each %$bridges) {
3753         $bridgeid = $k;
3754     }
3755     return 1 if !defined($bridgeid) || $bridgeid < 1;
3756
3757     my $bridge = "pci.$bridgeid";
3758     my $devices_list = vm_devices_list($vmid);
3759
3760     if (!defined($devices_list->{$bridge})) {
3761         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3762     }
3763
3764     return 1;
3765 }
3766
3767 sub qemu_set_link_status {
3768     my ($vmid, $device, $up) = @_;
3769
3770     vm_mon_cmd($vmid, "set_link", name => $device,
3771                up => $up ? JSON::true : JSON::false);
3772 }
3773
3774 sub qemu_netdevadd {
3775     my ($vmid, $conf, $device, $deviceid) = @_;
3776
3777     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3778     my %options =  split(/[=,]/, $netdev);
3779
3780     vm_mon_cmd($vmid, "netdev_add",  %options);
3781     return 1;
3782 }
3783
3784 sub qemu_netdevdel {
3785     my ($vmid, $deviceid) = @_;
3786
3787     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3788 }
3789
3790 sub qemu_cpu_hotplug {
3791     my ($vmid, $conf, $vcpus) = @_;
3792
3793     my $sockets = 1;
3794     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3795     $sockets = $conf->{sockets} if  $conf->{sockets};
3796     my $cores = $conf->{cores} || 1;
3797     my $maxcpus = $sockets * $cores;
3798
3799     $vcpus = $maxcpus if !$vcpus;
3800
3801     die "you can't add more vcpus than maxcpus\n"
3802         if $vcpus > $maxcpus;
3803
3804     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3805     die "online cpu unplug is not yet possible\n"
3806         if $vcpus < $currentvcpus;
3807
3808     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3809     die "vcpus in running vm is different than configuration\n"
3810         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3811
3812     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3813         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3814     }
3815 }
3816
3817 sub qemu_memory_hotplug {
3818     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3819
3820     return $value if !check_running($vmid);
3821
3822     my $memory = $conf->{memory} || $defaults->{memory};
3823     $value = $defaults->{memory} if !$value;
3824     return $value if $value == $memory;
3825
3826     my $static_memory = $STATICMEM;
3827     my $dimm_memory = $memory - $static_memory;
3828
3829     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3830     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3831
3832
3833     my $sockets = 1;
3834     $sockets = $conf->{sockets} if $conf->{sockets};
3835
3836     if($value > $memory) {
3837
3838         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3839             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3840
3841                 return if $current_size <= $conf->{memory};
3842
3843                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3844                 if (my $err = $@) {
3845                     eval { qemu_objectdel($vmid, "mem-$name"); };
3846                     die $err;
3847                 }
3848
3849                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3850                 if (my $err = $@) {
3851                     eval { qemu_objectdel($vmid, "mem-$name"); };
3852                     die $err;
3853                 }
3854                 #update conf after each succesful module hotplug
3855                 $conf->{memory} = $current_size;
3856                 write_config($vmid, $conf);
3857         });
3858
3859     } else {
3860
3861         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3862             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3863
3864                 return if $current_size >= $conf->{memory};
3865                 print "try to unplug memory dimm $name\n";
3866
3867                 my $retry = 0;
3868                 while (1) {
3869                     eval { qemu_devicedel($vmid, $name) };
3870                     sleep 3;
3871                     my $dimm_list = qemu_dimm_list($vmid);
3872                     last if !$dimm_list->{$name};
3873                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3874                     $retry++;
3875                 }
3876
3877                 #update conf after each succesful module unplug
3878                 $conf->{memory} = $current_size;
3879
3880                 eval { qemu_objectdel($vmid, "mem-$name"); };
3881                 write_config($vmid, $conf);
3882         });
3883     }
3884 }
3885
3886 sub qemu_dimm_list {
3887     my ($vmid) = @_;
3888
3889     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3890     my $dimms = {};
3891
3892     foreach my $dimm (@$dimmarray) {
3893
3894         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3895         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3896         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3897         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3898         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3899     }
3900     return $dimms;
3901 }
3902
3903 sub qemu_block_set_io_throttle {
3904     my ($vmid, $deviceid,
3905         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3906         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3907
3908     return if !check_running($vmid) ;
3909
3910     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3911         bps => int($bps),
3912         bps_rd => int($bps_rd),
3913         bps_wr => int($bps_wr),
3914         iops => int($iops),
3915         iops_rd => int($iops_rd),
3916         iops_wr => int($iops_wr),
3917         bps_max => int($bps_max),
3918         bps_rd_max => int($bps_rd_max),
3919         bps_wr_max => int($bps_wr_max),
3920         iops_max => int($iops_max),
3921         iops_rd_max => int($iops_rd_max),
3922         iops_wr_max => int($iops_wr_max)
3923     );
3924
3925 }
3926
3927 # old code, only used to shutdown old VM after update
3928 sub __read_avail {
3929     my ($fh, $timeout) = @_;
3930
3931     my $sel = new IO::Select;
3932     $sel->add($fh);
3933
3934     my $res = '';
3935     my $buf;
3936
3937     my @ready;
3938     while (scalar (@ready = $sel->can_read($timeout))) {
3939         my $count;
3940         if ($count = $fh->sysread($buf, 8192)) {
3941             if ($buf =~ /^(.*)\(qemu\) $/s) {
3942                 $res .= $1;
3943                 last;
3944             } else {
3945                 $res .= $buf;
3946             }
3947         } else {
3948             if (!defined($count)) {
3949                 die "$!\n";
3950             }
3951             last;
3952         }
3953     }
3954
3955     die "monitor read timeout\n" if !scalar(@ready);
3956
3957     return $res;
3958 }
3959
3960 # old code, only used to shutdown old VM after update
3961 sub vm_monitor_command {
3962     my ($vmid, $cmdstr, $nocheck) = @_;
3963
3964     my $res;
3965
3966     eval {
3967         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3968
3969         my $sname = "${var_run_tmpdir}/$vmid.mon";
3970
3971         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3972             die "unable to connect to VM $vmid socket - $!\n";
3973
3974         my $timeout = 3;
3975
3976         # hack: migrate sometime blocks the monitor (when migrate_downtime
3977         # is set)
3978         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3979             $timeout = 60*60; # 1 hour
3980         }
3981
3982         # read banner;
3983         my $data = __read_avail($sock, $timeout);
3984
3985         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3986             die "got unexpected qemu monitor banner\n";
3987         }
3988
3989         my $sel = new IO::Select;
3990         $sel->add($sock);
3991
3992         if (!scalar(my @ready = $sel->can_write($timeout))) {
3993             die "monitor write error - timeout";
3994         }
3995
3996         my $fullcmd = "$cmdstr\r";
3997
3998         # syslog('info', "VM $vmid monitor command: $cmdstr");
3999
4000         my $b;
4001         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4002             die "monitor write error - $!";
4003         }
4004
4005         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4006
4007         $timeout = 20;
4008
4009         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4010             $timeout = 60*60; # 1 hour
4011         } elsif ($cmdstr =~ m/^(eject|change)/) {
4012             $timeout = 60; # note: cdrom mount command is slow
4013         }
4014         if ($res = __read_avail($sock, $timeout)) {
4015
4016             my @lines = split("\r?\n", $res);
4017
4018             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4019
4020             $res = join("\n", @lines);
4021             $res .= "\n";
4022         }
4023     };
4024
4025     my $err = $@;
4026
4027     if ($err) {
4028         syslog("err", "VM $vmid monitor command failed - $err");
4029         die $err;
4030     }
4031
4032     return $res;
4033 }
4034
4035 sub qemu_block_resize {
4036     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4037
4038     my $running = check_running($vmid);
4039
4040     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4041
4042     return if !$running;
4043
4044     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4045
4046 }
4047
4048 sub qemu_volume_snapshot {
4049     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4050
4051     my $running = check_running($vmid);
4052
4053     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4054         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4055     } else {
4056         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4057     }
4058 }
4059
4060 sub qemu_volume_snapshot_delete {
4061     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4062
4063     my $running = check_running($vmid);
4064
4065     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4066
4067     return if !$running;
4068
4069     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4070 }
4071
4072 sub set_migration_caps {
4073     my ($vmid) = @_;
4074
4075     my $cap_ref = [];
4076
4077     my $enabled_cap = {
4078         "auto-converge" => 1,
4079         "xbzrle" => 1,
4080         "x-rdma-pin-all" => 0,
4081         "zero-blocks" => 0,
4082         "compress" => 0
4083     };
4084
4085     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4086
4087     for my $supported_capability (@$supported_capabilities) {
4088         push @$cap_ref, {
4089             capability => $supported_capability->{capability},
4090             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4091         };
4092     }
4093
4094     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4095 }
4096
4097 my $fast_plug_option = {
4098     'lock' => 1,
4099     'name' => 1,
4100     'onboot' => 1,
4101     'shares' => 1,
4102     'startup' => 1,
4103     'description' => 1,
4104 };
4105
4106 # hotplug changes in [PENDING]
4107 # $selection hash can be used to only apply specified options, for
4108 # example: { cores => 1 } (only apply changed 'cores')
4109 # $errors ref is used to return error messages
4110 sub vmconfig_hotplug_pending {
4111     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4112
4113     my $defaults = load_defaults();
4114
4115     # commit values which do not have any impact on running VM first
4116     # Note: those option cannot raise errors, we we do not care about
4117     # $selection and always apply them.
4118
4119     my $add_error = sub {
4120         my ($opt, $msg) = @_;
4121         $errors->{$opt} = "hotplug problem - $msg";
4122     };
4123
4124     my $changes = 0;
4125     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4126         if ($fast_plug_option->{$opt}) {
4127             $conf->{$opt} = $conf->{pending}->{$opt};
4128             delete $conf->{pending}->{$opt};
4129             $changes = 1;
4130         }
4131     }
4132
4133     if ($changes) {
4134         write_config($vmid, $conf);
4135         $conf = load_config($vmid); # update/reload
4136     }
4137
4138     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4139
4140     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4141     while (my ($opt, $force) = each %$pending_delete_hash) {
4142         next if $selection && !$selection->{$opt};
4143         eval {
4144             if ($opt eq 'hotplug') {
4145                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4146             } elsif ($opt eq 'tablet') {
4147                 die "skip\n" if !$hotplug_features->{usb};
4148                 if ($defaults->{tablet}) {
4149                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4150                 } else {
4151                     vm_deviceunplug($vmid, $conf, $opt);
4152                 }
4153             } elsif ($opt eq 'vcpus') {
4154                 die "skip\n" if !$hotplug_features->{cpu};
4155                 qemu_cpu_hotplug($vmid, $conf, undef);
4156             } elsif ($opt eq 'balloon') {
4157                 # enable balloon device is not hotpluggable
4158                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4159             } elsif ($fast_plug_option->{$opt}) {
4160                 # do nothing
4161             } elsif ($opt =~ m/^net(\d+)$/) {
4162                 die "skip\n" if !$hotplug_features->{network};
4163                 vm_deviceunplug($vmid, $conf, $opt);
4164             } elsif (valid_drivename($opt)) {
4165                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4166                 vm_deviceunplug($vmid, $conf, $opt);
4167                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4168             } elsif ($opt =~ m/^memory$/) {
4169                 die "skip\n" if !$hotplug_features->{memory};
4170                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4171             } elsif ($opt eq 'cpuunits') {
4172                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4173             } elsif ($opt eq 'cpulimit') {
4174                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4175             } else {
4176                 die "skip\n";
4177             }
4178         };
4179         if (my $err = $@) {
4180             &$add_error($opt, $err) if $err ne "skip\n";
4181         } else {
4182             # save new config if hotplug was successful
4183             delete $conf->{$opt};
4184             vmconfig_undelete_pending_option($conf, $opt);
4185             write_config($vmid, $conf);
4186             $conf = load_config($vmid); # update/reload
4187         }
4188     }
4189
4190     foreach my $opt (keys %{$conf->{pending}}) {
4191         next if $selection && !$selection->{$opt};
4192         my $value = $conf->{pending}->{$opt};
4193         eval {
4194             if ($opt eq 'hotplug') {
4195                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4196             } elsif ($opt eq 'tablet') {
4197                 die "skip\n" if !$hotplug_features->{usb};
4198                 if ($value == 1) {
4199                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4200                 } elsif ($value == 0) {
4201                     vm_deviceunplug($vmid, $conf, $opt);
4202                 }
4203             } elsif ($opt eq 'vcpus') {
4204                 die "skip\n" if !$hotplug_features->{cpu};
4205                 qemu_cpu_hotplug($vmid, $conf, $value);
4206             } elsif ($opt eq 'balloon') {
4207                 # enable/disable balloning device is not hotpluggable
4208                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4209                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4210                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4211
4212                 # allow manual ballooning if shares is set to zero
4213                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4214                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4215                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4216                 }
4217             } elsif ($opt =~ m/^net(\d+)$/) {
4218                 # some changes can be done without hotplug
4219                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4220                                     $vmid, $opt, $value);
4221             } elsif (valid_drivename($opt)) {
4222                 # some changes can be done without hotplug
4223                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4224                                      $vmid, $opt, $value, 1);
4225             } elsif ($opt =~ m/^memory$/) { #dimms
4226                 die "skip\n" if !$hotplug_features->{memory};
4227                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4228             } elsif ($opt eq 'cpuunits') {
4229                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4230             } elsif ($opt eq 'cpulimit') {
4231                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4232                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4233             } else {
4234                 die "skip\n";  # skip non-hot-pluggable options
4235             }
4236         };
4237         if (my $err = $@) {
4238             &$add_error($opt, $err) if $err ne "skip\n";
4239         } else {
4240             # save new config if hotplug was successful
4241             $conf->{$opt} = $value;
4242             delete $conf->{pending}->{$opt};
4243             write_config($vmid, $conf);
4244             $conf = load_config($vmid); # update/reload
4245         }
4246     }
4247 }
4248
4249 sub try_deallocate_drive {
4250     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4251
4252     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4253         my $volid = $drive->{file};
4254         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4255             my $sid = PVE::Storage::parse_volume_id($volid);
4256             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4257
4258             # check if the disk is really unused
4259             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4260                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4261             PVE::Storage::vdisk_free($storecfg, $volid);
4262             return 1;
4263         } else {
4264             # If vm is not owner of this disk remove from config
4265             return 1;
4266         }
4267     }
4268
4269     return undef;
4270 }
4271
4272 sub vmconfig_delete_or_detach_drive {
4273     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4274
4275     my $drive = parse_drive($opt, $conf->{$opt});
4276
4277     my $rpcenv = PVE::RPCEnvironment::get();
4278     my $authuser = $rpcenv->get_user();
4279
4280     if ($force) {
4281         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4282         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4283     } else {
4284         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4285     }
4286 }
4287
4288 sub vmconfig_apply_pending {
4289     my ($vmid, $conf, $storecfg) = @_;
4290
4291     # cold plug
4292
4293     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4294     while (my ($opt, $force) = each %$pending_delete_hash) {
4295         die "internal error" if $opt =~ m/^unused/;
4296         $conf = load_config($vmid); # update/reload
4297         if (!defined($conf->{$opt})) {
4298             vmconfig_undelete_pending_option($conf, $opt);
4299             write_config($vmid, $conf);
4300         } elsif (valid_drivename($opt)) {
4301             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4302             vmconfig_undelete_pending_option($conf, $opt);
4303             delete $conf->{$opt};
4304             write_config($vmid, $conf);
4305         } else {
4306             vmconfig_undelete_pending_option($conf, $opt);
4307             delete $conf->{$opt};
4308             write_config($vmid, $conf);
4309         }
4310     }
4311
4312     $conf = load_config($vmid); # update/reload
4313
4314     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4315         $conf = load_config($vmid); # update/reload
4316
4317         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4318             # skip if nothing changed
4319         } elsif (valid_drivename($opt)) {
4320             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4321                 if defined($conf->{$opt});
4322             $conf->{$opt} = $conf->{pending}->{$opt};
4323         } else {
4324             $conf->{$opt} = $conf->{pending}->{$opt};
4325         }
4326
4327         delete $conf->{pending}->{$opt};
4328         write_config($vmid, $conf);
4329     }
4330 }
4331
4332 my $safe_num_ne = sub {
4333     my ($a, $b) = @_;
4334
4335     return 0 if !defined($a) && !defined($b);
4336     return 1 if !defined($a);
4337     return 1 if !defined($b);
4338
4339     return $a != $b;
4340 };
4341
4342 my $safe_string_ne = sub {
4343     my ($a, $b) = @_;
4344
4345     return 0 if !defined($a) && !defined($b);
4346     return 1 if !defined($a);
4347     return 1 if !defined($b);
4348
4349     return $a ne $b;
4350 };
4351
4352 sub vmconfig_update_net {
4353     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4354
4355     my $newnet = parse_net($value);
4356
4357     if ($conf->{$opt}) {
4358         my $oldnet = parse_net($conf->{$opt});
4359
4360         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4361             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4362             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4363             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4364
4365             # for non online change, we try to hot-unplug
4366             die "skip\n" if !$hotplug;
4367             vm_deviceunplug($vmid, $conf, $opt);
4368         } else {
4369
4370             die "internal error" if $opt !~ m/net(\d+)/;
4371             my $iface = "tap${vmid}i$1";
4372
4373             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4374                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4375             }
4376
4377             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4378                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4379                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4380                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4381                 PVE::Network::tap_unplug($iface);
4382                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4383             }
4384
4385             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4386                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4387             }
4388
4389             return 1;
4390         }
4391     }
4392
4393     if ($hotplug) {
4394         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4395     } else {
4396         die "skip\n";
4397     }
4398 }
4399
4400 sub vmconfig_update_disk {
4401     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4402
4403     # fixme: do we need force?
4404
4405     my $drive = parse_drive($opt, $value);
4406
4407     if ($conf->{$opt}) {
4408
4409         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4410
4411             my $media = $drive->{media} || 'disk';
4412             my $oldmedia = $old_drive->{media} || 'disk';
4413             die "unable to change media type\n" if $media ne $oldmedia;
4414
4415             if (!drive_is_cdrom($old_drive)) {
4416
4417                 if ($drive->{file} ne $old_drive->{file}) {
4418
4419                     die "skip\n" if !$hotplug;
4420
4421                     # unplug and register as unused
4422                     vm_deviceunplug($vmid, $conf, $opt);
4423                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4424
4425                 } else {
4426                     # update existing disk
4427
4428                     # skip non hotpluggable value
4429                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4430                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4431                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4432                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4433                         die "skip\n";
4434                     }
4435
4436                     # apply throttle
4437                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4438                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4439                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4440                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4441                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4442                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4443                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4444                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4445                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4446                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4447                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4448                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4449
4450                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4451                                                    ($drive->{mbps} || 0)*1024*1024,
4452                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4453                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4454                                                    $drive->{iops} || 0,
4455                                                    $drive->{iops_rd} || 0,
4456                                                    $drive->{iops_wr} || 0,
4457                                                    ($drive->{mbps_max} || 0)*1024*1024,
4458                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4459                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4460                                                    $drive->{iops_max} || 0,
4461                                                    $drive->{iops_rd_max} || 0,
4462                                                    $drive->{iops_wr_max} || 0);
4463
4464                     }
4465
4466                     return 1;
4467                 }
4468
4469             } else { # cdrom
4470
4471                 if ($drive->{file} eq 'none') {
4472                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4473                 } else {
4474                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4475                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4476                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4477                 }
4478
4479                 return 1;
4480             }
4481         }
4482     }
4483
4484     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4485     # hotplug new disks
4486     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4487     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4488 }
4489
4490 sub vm_start {
4491     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4492         $forcemachine, $spice_ticket) = @_;
4493
4494     lock_config($vmid, sub {
4495         my $conf = load_config($vmid, $migratedfrom);
4496
4497         die "you can't start a vm if it's a template\n" if is_template($conf);
4498
4499         check_lock($conf) if !$skiplock;
4500
4501         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4502
4503         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4504             vmconfig_apply_pending($vmid, $conf, $storecfg);
4505             $conf = load_config($vmid); # update/reload
4506         }
4507
4508         my $defaults = load_defaults();
4509
4510         # set environment variable useful inside network script
4511         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4512
4513         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4514
4515         my $migrate_port = 0;
4516         my $migrate_uri;
4517         if ($statefile) {
4518             if ($statefile eq 'tcp') {
4519                 my $localip = "localhost";
4520                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4521                 my $nodename = PVE::INotify::nodename();
4522                 if ($datacenterconf->{migration_unsecure}) {
4523                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4524                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4525                 }
4526                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4527                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4528                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4529                 push @$cmd, '-incoming', $migrate_uri;
4530                 push @$cmd, '-S';
4531             } else {
4532                 push @$cmd, '-loadstate', $statefile;
4533             }
4534         } elsif ($paused) {
4535             push @$cmd, '-S';
4536         }
4537
4538         # host pci devices
4539         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4540           my $d = parse_hostpci($conf->{"hostpci$i"});
4541           next if !$d;
4542           my $pcidevices = $d->{pciid};
4543           foreach my $pcidevice (@$pcidevices) {
4544                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4545
4546                 my $info = pci_device_info("0000:$pciid");
4547                 die "IOMMU not present\n" if !check_iommu_support();
4548                 die "no pci device info for device '$pciid'\n" if !$info;
4549                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4550                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4551           }
4552         }
4553
4554         PVE::Storage::activate_volumes($storecfg, $vollist);
4555
4556         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4557                     umask => 0077); };
4558
4559         if (my $err = $@) {
4560             # deactivate volumes if start fails
4561             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4562             die "start failed: $err";
4563         }
4564
4565         print "migration listens on $migrate_uri\n" if $migrate_uri;
4566
4567         if ($statefile && $statefile ne 'tcp')  {
4568             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4569             warn $@ if $@;
4570         }
4571
4572         if ($migratedfrom) {
4573
4574             eval {
4575                 set_migration_caps($vmid);
4576             };
4577             warn $@ if $@;
4578
4579             if ($spice_port) {
4580                 print "spice listens on port $spice_port\n";
4581                 if ($spice_ticket) {
4582                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4583                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4584                 }
4585             }
4586
4587         } else {
4588
4589             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4590                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4591                     if $conf->{balloon};
4592             }
4593
4594             foreach my $opt (keys %$conf) {
4595                 next if $opt !~  m/^net\d+$/;
4596                 my $nicconf = parse_net($conf->{$opt});
4597                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4598             }
4599         }
4600
4601         vm_mon_cmd_nocheck($vmid, 'qom-set',
4602                     path => "machine/peripheral/balloon0",
4603                     property => "guest-stats-polling-interval",
4604                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4605
4606     });
4607 }
4608
4609 sub vm_mon_cmd {
4610     my ($vmid, $execute, %params) = @_;
4611
4612     my $cmd = { execute => $execute, arguments => \%params };
4613     vm_qmp_command($vmid, $cmd);
4614 }
4615
4616 sub vm_mon_cmd_nocheck {
4617     my ($vmid, $execute, %params) = @_;
4618
4619     my $cmd = { execute => $execute, arguments => \%params };
4620     vm_qmp_command($vmid, $cmd, 1);
4621 }
4622
4623 sub vm_qmp_command {
4624     my ($vmid, $cmd, $nocheck) = @_;
4625
4626     my $res;
4627
4628     my $timeout;
4629     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4630         $timeout = $cmd->{arguments}->{timeout};
4631         delete $cmd->{arguments}->{timeout};
4632     }
4633
4634     eval {
4635         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4636         my $sname = qmp_socket($vmid);
4637         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4638             my $qmpclient = PVE::QMPClient->new();
4639
4640             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4641         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4642             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4643                 if scalar(%{$cmd->{arguments}});
4644             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4645         } else {
4646             die "unable to open monitor socket\n";
4647         }
4648     };
4649     if (my $err = $@) {
4650         syslog("err", "VM $vmid qmp command failed - $err");
4651         die $err;
4652     }
4653
4654     return $res;
4655 }
4656
4657 sub vm_human_monitor_command {
4658     my ($vmid, $cmdline) = @_;
4659
4660     my $res;
4661
4662     my $cmd = {
4663         execute => 'human-monitor-command',
4664         arguments => { 'command-line' => $cmdline},
4665     };
4666
4667     return vm_qmp_command($vmid, $cmd);
4668 }
4669
4670 sub vm_commandline {
4671     my ($storecfg, $vmid) = @_;
4672
4673     my $conf = load_config($vmid);
4674
4675     my $defaults = load_defaults();
4676
4677     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4678
4679     return join(' ', @$cmd);
4680 }
4681
4682 sub vm_reset {
4683     my ($vmid, $skiplock) = @_;
4684
4685     lock_config($vmid, sub {
4686
4687         my $conf = load_config($vmid);
4688
4689         check_lock($conf) if !$skiplock;
4690
4691         vm_mon_cmd($vmid, "system_reset");
4692     });
4693 }
4694
4695 sub get_vm_volumes {
4696     my ($conf) = @_;
4697
4698     my $vollist = [];
4699     foreach_volid($conf, sub {
4700         my ($volid, $is_cdrom) = @_;
4701
4702         return if $volid =~ m|^/|;
4703
4704         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4705         return if !$sid;
4706
4707         push @$vollist, $volid;
4708     });
4709
4710     return $vollist;
4711 }
4712
4713 sub vm_stop_cleanup {
4714     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4715
4716     eval {
4717
4718         if (!$keepActive) {
4719             my $vollist = get_vm_volumes($conf);
4720             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4721         }
4722
4723         foreach my $ext (qw(mon qmp pid vnc qga)) {
4724             unlink "/var/run/qemu-server/${vmid}.$ext";
4725         }
4726
4727         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4728     };
4729     warn $@ if $@; # avoid errors - just warn
4730 }
4731
4732 # Note: use $nockeck to skip tests if VM configuration file exists.
4733 # We need that when migration VMs to other nodes (files already moved)
4734 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4735 sub vm_stop {
4736     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4737
4738     $force = 1 if !defined($force) && !$shutdown;
4739
4740     if ($migratedfrom){
4741         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4742         kill 15, $pid if $pid;
4743         my $conf = load_config($vmid, $migratedfrom);
4744         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4745         return;
4746     }
4747
4748     lock_config($vmid, sub {
4749
4750         my $pid = check_running($vmid, $nocheck);
4751         return if !$pid;
4752
4753         my $conf;
4754         if (!$nocheck) {
4755             $conf = load_config($vmid);
4756             check_lock($conf) if !$skiplock;
4757             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4758                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4759                 $timeout = $opts->{down} if $opts->{down};
4760             }
4761         }
4762
4763         $timeout = 60 if !defined($timeout);
4764
4765         eval {
4766             if ($shutdown) {
4767                 if (defined($conf) && $conf->{agent}) {
4768                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4769                 } else {
4770                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4771                 }
4772             } else {
4773                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4774             }
4775         };
4776         my $err = $@;
4777
4778         if (!$err) {
4779             my $count = 0;
4780             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4781                 $count++;
4782                 sleep 1;
4783             }
4784
4785             if ($count >= $timeout) {
4786                 if ($force) {
4787                     warn "VM still running - terminating now with SIGTERM\n";
4788                     kill 15, $pid;
4789                 } else {
4790                     die "VM quit/powerdown failed - got timeout\n";
4791                 }
4792             } else {
4793                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4794                 return;
4795             }
4796         } else {
4797             if ($force) {
4798                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4799                 kill 15, $pid;
4800             } else {
4801                 die "VM quit/powerdown failed\n";
4802             }
4803         }
4804
4805         # wait again
4806         $timeout = 10;
4807
4808         my $count = 0;
4809         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4810             $count++;
4811             sleep 1;
4812         }
4813
4814         if ($count >= $timeout) {
4815             warn "VM still running - terminating now with SIGKILL\n";
4816             kill 9, $pid;
4817             sleep 1;
4818         }
4819
4820         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4821    });
4822 }
4823
4824 sub vm_suspend {
4825     my ($vmid, $skiplock) = @_;
4826
4827     lock_config($vmid, sub {
4828
4829         my $conf = load_config($vmid);
4830
4831         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4832
4833         vm_mon_cmd($vmid, "stop");
4834     });
4835 }
4836
4837 sub vm_resume {
4838     my ($vmid, $skiplock, $nocheck) = @_;
4839
4840     lock_config($vmid, sub {
4841
4842         if (!$nocheck) {
4843
4844             my $conf = load_config($vmid);
4845
4846             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4847
4848             vm_mon_cmd($vmid, "cont");
4849
4850         } else {
4851             vm_mon_cmd_nocheck($vmid, "cont");
4852         }
4853     });
4854 }
4855
4856 sub vm_sendkey {
4857     my ($vmid, $skiplock, $key) = @_;
4858
4859     lock_config($vmid, sub {
4860
4861         my $conf = load_config($vmid);
4862
4863         # there is no qmp command, so we use the human monitor command
4864         vm_human_monitor_command($vmid, "sendkey $key");
4865     });
4866 }
4867
4868 sub vm_destroy {
4869     my ($storecfg, $vmid, $skiplock) = @_;
4870
4871     lock_config($vmid, sub {
4872
4873         my $conf = load_config($vmid);
4874
4875         if (!check_running($vmid)) {
4876             destroy_vm($storecfg, $vmid, undef, $skiplock);
4877         } else {
4878             die "VM $vmid is running - destroy failed\n";
4879         }
4880     });
4881 }
4882
4883 # pci helpers
4884
4885 sub file_write {
4886     my ($filename, $buf) = @_;
4887
4888     my $fh = IO::File->new($filename, "w");
4889     return undef if !$fh;
4890
4891     my $res = print $fh $buf;
4892
4893     $fh->close();
4894
4895     return $res;
4896 }
4897
4898 sub pci_device_info {
4899     my ($name) = @_;
4900
4901     my $res;
4902
4903     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4904     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4905
4906     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4907     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4908
4909     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4910     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4911
4912     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4913     return undef if !defined($product) || $product !~ s/^0x//;
4914
4915     $res = {
4916         name => $name,
4917         vendor => $vendor,
4918         product => $product,
4919         domain => $domain,
4920         bus => $bus,
4921         slot => $slot,
4922         func => $func,
4923         irq => $irq,
4924         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4925     };
4926
4927     return $res;
4928 }
4929
4930 sub pci_dev_reset {
4931     my ($dev) = @_;
4932
4933     my $name = $dev->{name};
4934
4935     my $fn = "$pcisysfs/devices/$name/reset";
4936
4937     return file_write($fn, "1");
4938 }
4939
4940 sub pci_dev_bind_to_vfio {
4941     my ($dev) = @_;
4942
4943     my $name = $dev->{name};
4944
4945     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4946
4947     if (!-d $vfio_basedir) {
4948         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4949     }
4950     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4951
4952     my $testdir = "$vfio_basedir/$name";
4953     return 1 if -d $testdir;
4954
4955     my $data = "$dev->{vendor} $dev->{product}";
4956     return undef if !file_write("$vfio_basedir/new_id", $data);
4957
4958     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4959     if (!file_write($fn, $name)) {
4960         return undef if -f $fn;
4961     }
4962
4963     $fn = "$vfio_basedir/bind";
4964     if (! -d $testdir) {
4965         return undef if !file_write($fn, $name);
4966     }
4967
4968     return -d $testdir;
4969 }
4970
4971 sub pci_dev_group_bind_to_vfio {
4972     my ($pciid) = @_;
4973
4974     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4975
4976     if (!-d $vfio_basedir) {
4977         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4978     }
4979     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4980
4981     # get IOMMU group devices
4982     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4983       my @devs = grep /^0000:/, readdir($D);
4984     closedir($D);
4985
4986     foreach my $pciid (@devs) {
4987         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4988
4989         # pci bridges, switches or root ports are not supported
4990         # they have a pci_bus subdirectory so skip them
4991         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4992
4993         my $info = pci_device_info($1);
4994         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4995     }
4996
4997     return 1;
4998 }
4999
5000 sub print_pci_addr {
5001     my ($id, $bridges) = @_;
5002
5003     my $res = '';
5004     my $devices = {
5005         piix3 => { bus => 0, addr => 1 },
5006         #addr2 : first videocard
5007         balloon0 => { bus => 0, addr => 3 },
5008         watchdog => { bus => 0, addr => 4 },
5009         scsihw0 => { bus => 0, addr => 5 },
5010         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
5011         scsihw1 => { bus => 0, addr => 6 },
5012         ahci0 => { bus => 0, addr => 7 },
5013         qga0 => { bus => 0, addr => 8 },
5014         spice => { bus => 0, addr => 9 },
5015         virtio0 => { bus => 0, addr => 10 },
5016         virtio1 => { bus => 0, addr => 11 },
5017         virtio2 => { bus => 0, addr => 12 },
5018         virtio3 => { bus => 0, addr => 13 },
5019         virtio4 => { bus => 0, addr => 14 },
5020         virtio5 => { bus => 0, addr => 15 },
5021         hostpci0 => { bus => 0, addr => 16 },
5022         hostpci1 => { bus => 0, addr => 17 },
5023         net0 => { bus => 0, addr => 18 },
5024         net1 => { bus => 0, addr => 19 },
5025         net2 => { bus => 0, addr => 20 },
5026         net3 => { bus => 0, addr => 21 },
5027         net4 => { bus => 0, addr => 22 },
5028         net5 => { bus => 0, addr => 23 },
5029         vga1 => { bus => 0, addr => 24 },
5030         vga2 => { bus => 0, addr => 25 },
5031         vga3 => { bus => 0, addr => 26 },
5032         hostpci2 => { bus => 0, addr => 27 },
5033         hostpci3 => { bus => 0, addr => 28 },
5034         #addr29 : usb-host (pve-usb.cfg)
5035         'pci.1' => { bus => 0, addr => 30 },
5036         'pci.2' => { bus => 0, addr => 31 },
5037         'net6' => { bus => 1, addr => 1 },
5038         'net7' => { bus => 1, addr => 2 },
5039         'net8' => { bus => 1, addr => 3 },
5040         'net9' => { bus => 1, addr => 4 },
5041         'net10' => { bus => 1, addr => 5 },
5042         'net11' => { bus => 1, addr => 6 },
5043         'net12' => { bus => 1, addr => 7 },
5044         'net13' => { bus => 1, addr => 8 },
5045         'net14' => { bus => 1, addr => 9 },
5046         'net15' => { bus => 1, addr => 10 },
5047         'net16' => { bus => 1, addr => 11 },
5048         'net17' => { bus => 1, addr => 12 },
5049         'net18' => { bus => 1, addr => 13 },
5050         'net19' => { bus => 1, addr => 14 },
5051         'net20' => { bus => 1, addr => 15 },
5052         'net21' => { bus => 1, addr => 16 },
5053         'net22' => { bus => 1, addr => 17 },
5054         'net23' => { bus => 1, addr => 18 },
5055         'net24' => { bus => 1, addr => 19 },
5056         'net25' => { bus => 1, addr => 20 },
5057         'net26' => { bus => 1, addr => 21 },
5058         'net27' => { bus => 1, addr => 22 },
5059         'net28' => { bus => 1, addr => 23 },
5060         'net29' => { bus => 1, addr => 24 },
5061         'net30' => { bus => 1, addr => 25 },
5062         'net31' => { bus => 1, addr => 26 },
5063         'xhci' => { bus => 1, addr => 27 },
5064         'virtio6' => { bus => 2, addr => 1 },
5065         'virtio7' => { bus => 2, addr => 2 },
5066         'virtio8' => { bus => 2, addr => 3 },
5067         'virtio9' => { bus => 2, addr => 4 },
5068         'virtio10' => { bus => 2, addr => 5 },
5069         'virtio11' => { bus => 2, addr => 6 },
5070         'virtio12' => { bus => 2, addr => 7 },
5071         'virtio13' => { bus => 2, addr => 8 },
5072         'virtio14' => { bus => 2, addr => 9 },
5073         'virtio15' => { bus => 2, addr => 10 },
5074         'virtioscsi0' => { bus => 3, addr => 1 },
5075         'virtioscsi1' => { bus => 3, addr => 2 },
5076         'virtioscsi2' => { bus => 3, addr => 3 },
5077         'virtioscsi3' => { bus => 3, addr => 4 },
5078         'virtioscsi4' => { bus => 3, addr => 5 },
5079         'virtioscsi5' => { bus => 3, addr => 6 },
5080         'virtioscsi6' => { bus => 3, addr => 7 },
5081         'virtioscsi7' => { bus => 3, addr => 8 },
5082         'virtioscsi8' => { bus => 3, addr => 9 },
5083         'virtioscsi9' => { bus => 3, addr => 10 },
5084         'virtioscsi10' => { bus => 3, addr => 11 },
5085         'virtioscsi11' => { bus => 3, addr => 12 },
5086         'virtioscsi12' => { bus => 3, addr => 13 },
5087         'virtioscsi13' => { bus => 3, addr => 14 },
5088         'virtioscsi14' => { bus => 3, addr => 15 },
5089         'virtioscsi15' => { bus => 3, addr => 16 },
5090         'virtioscsi16' => { bus => 3, addr => 17 },
5091         'virtioscsi17' => { bus => 3, addr => 18 },
5092         'virtioscsi18' => { bus => 3, addr => 19 },
5093         'virtioscsi19' => { bus => 3, addr => 20 },
5094         'virtioscsi20' => { bus => 3, addr => 21 },
5095         'virtioscsi21' => { bus => 3, addr => 22 },
5096         'virtioscsi22' => { bus => 3, addr => 23 },
5097         'virtioscsi23' => { bus => 3, addr => 24 },
5098         'virtioscsi24' => { bus => 3, addr => 25 },
5099         'virtioscsi25' => { bus => 3, addr => 26 },
5100         'virtioscsi26' => { bus => 3, addr => 27 },
5101         'virtioscsi27' => { bus => 3, addr => 28 },
5102         'virtioscsi28' => { bus => 3, addr => 29 },
5103         'virtioscsi29' => { bus => 3, addr => 30 },
5104         'virtioscsi30' => { bus => 3, addr => 31 },
5105
5106     };
5107
5108     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5109            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5110            my $bus = $devices->{$id}->{bus};
5111            $res = ",bus=pci.$bus,addr=$addr";
5112            $bridges->{$bus} = 1 if $bridges;
5113     }
5114     return $res;
5115
5116 }
5117
5118 sub print_pcie_addr {
5119     my ($id) = @_;
5120
5121     my $res = '';
5122     my $devices = {
5123         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5124         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5125         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5126         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5127     };
5128
5129     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5130            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5131            my $bus = $devices->{$id}->{bus};
5132            $res = ",bus=$bus,addr=$addr";
5133     }
5134     return $res;
5135
5136 }
5137
5138 # vzdump restore implementaion
5139
5140 sub tar_archive_read_firstfile {
5141     my $archive = shift;
5142
5143     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5144
5145     # try to detect archive type first
5146     my $pid = open (TMP, "tar tf '$archive'|") ||
5147         die "unable to open file '$archive'\n";
5148     my $firstfile = <TMP>;
5149     kill 15, $pid;
5150     close TMP;
5151
5152     die "ERROR: archive contaions no data\n" if !$firstfile;
5153     chomp $firstfile;
5154
5155     return $firstfile;
5156 }
5157
5158 sub tar_restore_cleanup {
5159     my ($storecfg, $statfile) = @_;
5160
5161     print STDERR "starting cleanup\n";
5162
5163     if (my $fd = IO::File->new($statfile, "r")) {
5164         while (defined(my $line = <$fd>)) {
5165             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5166                 my $volid = $2;
5167                 eval {
5168                     if ($volid =~ m|^/|) {
5169                         unlink $volid || die 'unlink failed\n';
5170                     } else {
5171                         PVE::Storage::vdisk_free($storecfg, $volid);
5172                     }
5173                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5174                 };
5175                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5176             } else {
5177                 print STDERR "unable to parse line in statfile - $line";
5178             }
5179         }
5180         $fd->close();
5181     }
5182 }
5183
5184 sub restore_archive {
5185     my ($archive, $vmid, $user, $opts) = @_;
5186
5187     my $format = $opts->{format};
5188     my $comp;
5189
5190     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5191         $format = 'tar' if !$format;
5192         $comp = 'gzip';
5193     } elsif ($archive =~ m/\.tar$/) {
5194         $format = 'tar' if !$format;
5195     } elsif ($archive =~ m/.tar.lzo$/) {
5196         $format = 'tar' if !$format;
5197         $comp = 'lzop';
5198     } elsif ($archive =~ m/\.vma$/) {
5199         $format = 'vma' if !$format;
5200     } elsif ($archive =~ m/\.vma\.gz$/) {
5201         $format = 'vma' if !$format;
5202         $comp = 'gzip';
5203     } elsif ($archive =~ m/\.vma\.lzo$/) {
5204         $format = 'vma' if !$format;
5205         $comp = 'lzop';
5206     } else {
5207         $format = 'vma' if !$format; # default
5208     }
5209
5210     # try to detect archive format
5211     if ($format eq 'tar') {
5212         return restore_tar_archive($archive, $vmid, $user, $opts);
5213     } else {
5214         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5215     }
5216 }
5217
5218 sub restore_update_config_line {
5219     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5220
5221     return if $line =~ m/^\#qmdump\#/;
5222     return if $line =~ m/^\#vzdump\#/;
5223     return if $line =~ m/^lock:/;
5224     return if $line =~ m/^unused\d+:/;
5225     return if $line =~ m/^parent:/;
5226     return if $line =~ m/^template:/; # restored VM is never a template
5227
5228     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5229         # try to convert old 1.X settings
5230         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5231         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5232             my ($model, $macaddr) = split(/\=/, $devconfig);
5233             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5234             my $net = {
5235                 model => $model,
5236                 bridge => "vmbr$ind",
5237                 macaddr => $macaddr,
5238             };
5239             my $netstr = print_net($net);
5240
5241             print $outfd "net$cookie->{netcount}: $netstr\n";
5242             $cookie->{netcount}++;
5243         }
5244     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5245         my ($id, $netstr) = ($1, $2);
5246         my $net = parse_net($netstr);
5247         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5248         $netstr = print_net($net);
5249         print $outfd "$id: $netstr\n";
5250     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5251         my $virtdev = $1;
5252         my $value = $3;
5253         my $di = parse_drive($virtdev, $value);
5254         if (defined($di->{backup}) && !$di->{backup}) {
5255             print $outfd "#$line";
5256         } elsif ($map->{$virtdev}) {
5257             delete $di->{format}; # format can change on restore
5258             $di->{file} = $map->{$virtdev};
5259             $value = print_drive($vmid, $di);
5260             print $outfd "$virtdev: $value\n";
5261         } else {
5262             print $outfd $line;
5263         }
5264     } else {
5265         print $outfd $line;
5266     }
5267 }
5268
5269 sub scan_volids {
5270     my ($cfg, $vmid) = @_;
5271
5272     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5273
5274     my $volid_hash = {};
5275     foreach my $storeid (keys %$info) {
5276         foreach my $item (@{$info->{$storeid}}) {
5277             next if !($item->{volid} && $item->{size});
5278             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5279             $volid_hash->{$item->{volid}} = $item;
5280         }
5281     }
5282
5283     return $volid_hash;
5284 }
5285
5286 sub is_volume_in_use {
5287     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5288
5289     my $path = PVE::Storage::path($storecfg, $volid);
5290
5291     my $scan_config = sub {
5292         my ($cref, $snapname) = @_;
5293
5294         foreach my $key (keys %$cref) {
5295             my $value = $cref->{$key};
5296             if (valid_drivename($key)) {
5297                 next if $skip_drive && $key eq $skip_drive;
5298                 my $drive = parse_drive($key, $value);
5299                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5300                 return 1 if $volid eq $drive->{file};
5301                 if ($drive->{file} =~ m!^/!) {
5302                     return 1 if $drive->{file} eq $path;
5303                 } else {
5304                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5305                     next if !$storeid;
5306                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5307                     next if !$scfg;
5308                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5309                 }
5310             }
5311         }
5312
5313         return 0;
5314     };
5315
5316     return 1 if &$scan_config($conf);
5317
5318     undef $skip_drive;
5319
5320     foreach my $snapname (keys %{$conf->{snapshots}}) {
5321         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5322     }
5323
5324     return 0;
5325 }
5326
5327 sub update_disksize {
5328     my ($vmid, $conf, $volid_hash) = @_;
5329
5330     my $changes;
5331
5332     my $used = {};
5333
5334     # Note: it is allowed to define multiple storages with same path (alias), so
5335     # we need to check both 'volid' and real 'path' (two different volid can point
5336     # to the same path).
5337
5338     my $usedpath = {};
5339
5340     # update size info
5341     foreach my $opt (keys %$conf) {
5342         if (valid_drivename($opt)) {
5343             my $drive = parse_drive($opt, $conf->{$opt});
5344             my $volid = $drive->{file};
5345             next if !$volid;
5346
5347             $used->{$volid} = 1;
5348             if ($volid_hash->{$volid} &&
5349                 (my $path = $volid_hash->{$volid}->{path})) {
5350                 $usedpath->{$path} = 1;
5351             }
5352
5353             next if drive_is_cdrom($drive);
5354             next if !$volid_hash->{$volid};
5355
5356             $drive->{size} = $volid_hash->{$volid}->{size};
5357             my $new = print_drive($vmid, $drive);
5358             if ($new ne $conf->{$opt}) {
5359                 $changes = 1;
5360                 $conf->{$opt} = $new;
5361             }
5362         }
5363     }
5364
5365     # remove 'unusedX' entry if volume is used
5366     foreach my $opt (keys %$conf) {
5367         next if $opt !~ m/^unused\d+$/;
5368         my $volid = $conf->{$opt};
5369         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5370         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5371             $changes = 1;
5372             delete $conf->{$opt};
5373         }
5374     }
5375
5376     foreach my $volid (sort keys %$volid_hash) {
5377         next if $volid =~ m/vm-$vmid-state-/;
5378         next if $used->{$volid};
5379         my $path = $volid_hash->{$volid}->{path};
5380         next if !$path; # just to be sure
5381         next if $usedpath->{$path};
5382         $changes = 1;
5383         add_unused_volume($conf, $volid);
5384         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5385     }
5386
5387     return $changes;
5388 }
5389
5390 sub rescan {
5391     my ($vmid, $nolock) = @_;
5392
5393     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5394
5395     my $volid_hash = scan_volids($cfg, $vmid);
5396
5397     my $updatefn =  sub {
5398         my ($vmid) = @_;
5399
5400         my $conf = load_config($vmid);
5401
5402         check_lock($conf);
5403
5404         my $vm_volids = {};
5405         foreach my $volid (keys %$volid_hash) {
5406             my $info = $volid_hash->{$volid};
5407             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5408         }
5409
5410         my $changes = update_disksize($vmid, $conf, $vm_volids);
5411
5412         write_config($vmid, $conf) if $changes;
5413     };
5414
5415     if (defined($vmid)) {
5416         if ($nolock) {
5417             &$updatefn($vmid);
5418         } else {
5419             lock_config($vmid, $updatefn, $vmid);
5420         }
5421     } else {
5422         my $vmlist = config_list();
5423         foreach my $vmid (keys %$vmlist) {
5424             if ($nolock) {
5425                 &$updatefn($vmid);
5426             } else {
5427                 lock_config($vmid, $updatefn, $vmid);
5428             }
5429         }
5430     }
5431 }
5432
5433 sub restore_vma_archive {
5434     my ($archive, $vmid, $user, $opts, $comp) = @_;
5435
5436     my $input = $archive eq '-' ? "<&STDIN" : undef;
5437     my $readfrom = $archive;
5438
5439     my $uncomp = '';
5440     if ($comp) {
5441         $readfrom = '-';
5442         my $qarchive = PVE::Tools::shellquote($archive);
5443         if ($comp eq 'gzip') {
5444             $uncomp = "zcat $qarchive|";
5445         } elsif ($comp eq 'lzop') {
5446             $uncomp = "lzop -d -c $qarchive|";
5447         } else {
5448             die "unknown compression method '$comp'\n";
5449         }
5450
5451     }
5452
5453     my $tmpdir = "/var/tmp/vzdumptmp$$";
5454     rmtree $tmpdir;
5455
5456     # disable interrupts (always do cleanups)
5457     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5458         warn "got interrupt - ignored\n";
5459     };
5460
5461     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5462     POSIX::mkfifo($mapfifo, 0600);
5463     my $fifofh;
5464
5465     my $openfifo = sub {
5466         open($fifofh, '>', $mapfifo) || die $!;
5467     };
5468
5469     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5470
5471     my $oldtimeout;
5472     my $timeout = 5;
5473
5474     my $devinfo = {};
5475
5476     my $rpcenv = PVE::RPCEnvironment::get();
5477
5478     my $conffile = config_file($vmid);
5479     my $tmpfn = "$conffile.$$.tmp";
5480
5481     # Note: $oldconf is undef if VM does not exists
5482     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5483
5484     my $print_devmap = sub {
5485         my $virtdev_hash = {};
5486
5487         my $cfgfn = "$tmpdir/qemu-server.conf";
5488
5489         # we can read the config - that is already extracted
5490         my $fh = IO::File->new($cfgfn, "r") ||
5491             "unable to read qemu-server.conf - $!\n";
5492
5493         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5494         if (-f $fwcfgfn) {
5495             my $pve_firewall_dir = '/etc/pve/firewall';
5496             mkdir $pve_firewall_dir; # make sure the dir exists
5497             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5498         }
5499
5500         while (defined(my $line = <$fh>)) {
5501             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5502                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5503                 die "archive does not contain data for drive '$virtdev'\n"
5504                     if !$devinfo->{$devname};
5505                 if (defined($opts->{storage})) {
5506                     $storeid = $opts->{storage} || 'local';
5507                 } elsif (!$storeid) {
5508                     $storeid = 'local';
5509                 }
5510                 $format = 'raw' if !$format;
5511                 $devinfo->{$devname}->{devname} = $devname;
5512                 $devinfo->{$devname}->{virtdev} = $virtdev;
5513                 $devinfo->{$devname}->{format} = $format;
5514                 $devinfo->{$devname}->{storeid} = $storeid;
5515
5516                 # check permission on storage
5517                 my $pool = $opts->{pool}; # todo: do we need that?
5518                 if ($user ne 'root@pam') {
5519                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5520                 }
5521
5522                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5523             }
5524         }
5525
5526         foreach my $devname (keys %$devinfo) {
5527             die "found no device mapping information for device '$devname'\n"
5528                 if !$devinfo->{$devname}->{virtdev};
5529         }
5530
5531         my $cfg = cfs_read_file('storage.cfg');
5532
5533         # create empty/temp config
5534         if ($oldconf) {
5535             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5536             foreach_drive($oldconf, sub {
5537                 my ($ds, $drive) = @_;
5538
5539                 return if drive_is_cdrom($drive);
5540
5541                 my $volid = $drive->{file};
5542
5543                 return if !$volid || $volid =~ m|^/|;
5544
5545                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5546                 return if !$path || !$owner || ($owner != $vmid);
5547
5548                 # Note: only delete disk we want to restore
5549                 # other volumes will become unused
5550                 if ($virtdev_hash->{$ds}) {
5551                     PVE::Storage::vdisk_free($cfg, $volid);
5552                 }
5553             });
5554
5555             # delete vmstate files
5556             # since after the restore we have no snapshots anymore
5557             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5558                 my $snap = $oldconf->{snapshots}->{$snapname};
5559                 if ($snap->{vmstate}) {
5560                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5561                     if (my $err = $@) {
5562                         warn $err;
5563                     }
5564                 }
5565             }
5566         }
5567
5568         my $map = {};
5569         foreach my $virtdev (sort keys %$virtdev_hash) {
5570             my $d = $virtdev_hash->{$virtdev};
5571             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5572             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5573
5574             # test if requested format is supported
5575             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5576             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5577             $d->{format} = $defFormat if !$supported;
5578
5579             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5580                                                   $d->{format}, undef, $alloc_size);
5581             print STDERR "new volume ID is '$volid'\n";
5582             $d->{volid} = $volid;
5583             my $path = PVE::Storage::path($cfg, $volid);
5584
5585             PVE::Storage::activate_volumes($cfg,[$volid]);
5586
5587             my $write_zeros = 1;
5588             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5589                 $write_zeros = 0;
5590             }
5591
5592             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5593
5594             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5595             $map->{$virtdev} = $volid;
5596         }
5597
5598         $fh->seek(0, 0) || die "seek failed - $!\n";
5599
5600         my $outfd = new IO::File ($tmpfn, "w") ||
5601             die "unable to write config for VM $vmid\n";
5602
5603         my $cookie = { netcount => 0 };
5604         while (defined(my $line = <$fh>)) {
5605             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5606         }
5607
5608         $fh->close();
5609         $outfd->close();
5610     };
5611
5612     eval {
5613         # enable interrupts
5614         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5615             die "interrupted by signal\n";
5616         };
5617         local $SIG{ALRM} = sub { die "got timeout\n"; };
5618
5619         $oldtimeout = alarm($timeout);
5620
5621         my $parser = sub {
5622             my $line = shift;
5623
5624             print "$line\n";
5625
5626             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5627                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5628                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5629             } elsif ($line =~ m/^CTIME: /) {
5630                 # we correctly received the vma config, so we can disable
5631                 # the timeout now for disk allocation (set to 10 minutes, so
5632                 # that we always timeout if something goes wrong)
5633                 alarm(600);
5634                 &$print_devmap();
5635                 print $fifofh "done\n";
5636                 my $tmp = $oldtimeout || 0;
5637                 $oldtimeout = undef;
5638                 alarm($tmp);
5639                 close($fifofh);
5640             }
5641         };
5642
5643         print "restore vma archive: $cmd\n";
5644         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5645     };
5646     my $err = $@;
5647
5648     alarm($oldtimeout) if $oldtimeout;
5649
5650     my $vollist = [];
5651     foreach my $devname (keys %$devinfo) {
5652         my $volid = $devinfo->{$devname}->{volid};
5653         push @$vollist, $volid if $volid;
5654     }
5655
5656     my $cfg = cfs_read_file('storage.cfg');
5657     PVE::Storage::deactivate_volumes($cfg, $vollist);
5658
5659     unlink $mapfifo;
5660
5661     if ($err) {
5662         rmtree $tmpdir;
5663         unlink $tmpfn;
5664
5665         foreach my $devname (keys %$devinfo) {
5666             my $volid = $devinfo->{$devname}->{volid};
5667             next if !$volid;
5668             eval {
5669                 if ($volid =~ m|^/|) {
5670                     unlink $volid || die 'unlink failed\n';
5671                 } else {
5672                     PVE::Storage::vdisk_free($cfg, $volid);
5673                 }
5674                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5675             };
5676             print STDERR "unable to cleanup '$volid' - $@" if $@;
5677         }
5678         die $err;
5679     }
5680
5681     rmtree $tmpdir;
5682
5683     rename($tmpfn, $conffile) ||
5684         die "unable to commit configuration file '$conffile'\n";
5685
5686     PVE::Cluster::cfs_update(); # make sure we read new file
5687
5688     eval { rescan($vmid, 1); };
5689     warn $@ if $@;
5690 }
5691
5692 sub restore_tar_archive {
5693     my ($archive, $vmid, $user, $opts) = @_;
5694
5695     if ($archive ne '-') {
5696         my $firstfile = tar_archive_read_firstfile($archive);
5697         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5698             if $firstfile ne 'qemu-server.conf';
5699     }
5700
5701     my $storecfg = cfs_read_file('storage.cfg');
5702
5703     # destroy existing data - keep empty config
5704     my $vmcfgfn = config_file($vmid);
5705     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5706
5707     my $tocmd = "/usr/lib/qemu-server/qmextract";
5708
5709     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5710     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5711     $tocmd .= ' --prealloc' if $opts->{prealloc};
5712     $tocmd .= ' --info' if $opts->{info};
5713
5714     # tar option "xf" does not autodetect compression when read from STDIN,
5715     # so we pipe to zcat
5716     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5717         PVE::Tools::shellquote("--to-command=$tocmd");
5718
5719     my $tmpdir = "/var/tmp/vzdumptmp$$";
5720     mkpath $tmpdir;
5721
5722     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5723     local $ENV{VZDUMP_VMID} = $vmid;
5724     local $ENV{VZDUMP_USER} = $user;
5725
5726     my $conffile = config_file($vmid);
5727     my $tmpfn = "$conffile.$$.tmp";
5728
5729     # disable interrupts (always do cleanups)
5730     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5731         print STDERR "got interrupt - ignored\n";
5732     };
5733
5734     eval {
5735         # enable interrupts
5736         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5737             die "interrupted by signal\n";
5738         };
5739
5740         if ($archive eq '-') {
5741             print "extracting archive from STDIN\n";
5742             run_command($cmd, input => "<&STDIN");
5743         } else {
5744             print "extracting archive '$archive'\n";
5745             run_command($cmd);
5746         }
5747
5748         return if $opts->{info};
5749
5750         # read new mapping
5751         my $map = {};
5752         my $statfile = "$tmpdir/qmrestore.stat";
5753         if (my $fd = IO::File->new($statfile, "r")) {
5754             while (defined (my $line = <$fd>)) {
5755                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5756                     $map->{$1} = $2 if $1;
5757                 } else {
5758                     print STDERR "unable to parse line in statfile - $line\n";
5759                 }
5760             }
5761             $fd->close();
5762         }
5763
5764         my $confsrc = "$tmpdir/qemu-server.conf";
5765
5766         my $srcfd = new IO::File($confsrc, "r") ||
5767             die "unable to open file '$confsrc'\n";
5768
5769         my $outfd = new IO::File ($tmpfn, "w") ||
5770             die "unable to write config for VM $vmid\n";
5771
5772         my $cookie = { netcount => 0 };
5773         while (defined (my $line = <$srcfd>)) {
5774             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5775         }
5776
5777         $srcfd->close();
5778         $outfd->close();
5779     };
5780     my $err = $@;
5781
5782     if ($err) {
5783
5784         unlink $tmpfn;
5785
5786         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5787
5788         die $err;
5789     }
5790
5791     rmtree $tmpdir;
5792
5793     rename $tmpfn, $conffile ||
5794         die "unable to commit configuration file '$conffile'\n";
5795
5796     PVE::Cluster::cfs_update(); # make sure we read new file
5797
5798     eval { rescan($vmid, 1); };
5799     warn $@ if $@;
5800 };
5801
5802
5803 # Internal snapshots
5804
5805 # NOTE: Snapshot create/delete involves several non-atomic
5806 # action, and can take a long time.
5807 # So we try to avoid locking the file and use 'lock' variable
5808 # inside the config file instead.
5809
5810 my $snapshot_copy_config = sub {
5811     my ($source, $dest) = @_;
5812
5813     foreach my $k (keys %$source) {
5814         next if $k eq 'snapshots';
5815         next if $k eq 'snapstate';
5816         next if $k eq 'snaptime';
5817         next if $k eq 'vmstate';
5818         next if $k eq 'lock';
5819         next if $k eq 'digest';
5820         next if $k eq 'description';
5821         next if $k =~ m/^unused\d+$/;
5822
5823         $dest->{$k} = $source->{$k};
5824     }
5825 };
5826
5827 my $snapshot_apply_config = sub {
5828     my ($conf, $snap) = @_;
5829
5830     # copy snapshot list
5831     my $newconf = {
5832         snapshots => $conf->{snapshots},
5833     };
5834
5835     # keep description and list of unused disks
5836     foreach my $k (keys %$conf) {
5837         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5838         $newconf->{$k} = $conf->{$k};
5839     }
5840
5841     &$snapshot_copy_config($snap, $newconf);
5842
5843     return $newconf;
5844 };
5845
5846 sub foreach_writable_storage {
5847     my ($conf, $func) = @_;
5848
5849     my $sidhash = {};
5850
5851     foreach my $ds (keys %$conf) {
5852         next if !valid_drivename($ds);
5853
5854         my $drive = parse_drive($ds, $conf->{$ds});
5855         next if !$drive;
5856         next if drive_is_cdrom($drive);
5857
5858         my $volid = $drive->{file};
5859
5860         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5861         $sidhash->{$sid} = $sid if $sid;
5862     }
5863
5864     foreach my $sid (sort keys %$sidhash) {
5865         &$func($sid);
5866     }
5867 }
5868
5869 my $alloc_vmstate_volid = sub {
5870     my ($storecfg, $vmid, $conf, $snapname) = @_;
5871
5872     # Note: we try to be smart when selecting a $target storage
5873
5874     my $target;
5875
5876     # search shared storage first
5877     foreach_writable_storage($conf, sub {
5878         my ($sid) = @_;
5879         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5880         return if !$scfg->{shared};
5881
5882         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5883     });
5884
5885     if (!$target) {
5886         # now search local storage
5887         foreach_writable_storage($conf, sub {
5888             my ($sid) = @_;
5889             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5890             return if $scfg->{shared};
5891
5892             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5893         });
5894     }
5895
5896     $target = 'local' if !$target;
5897
5898     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5899     # we abort live save after $conf->{memory}, so we need at max twice that space
5900     my $size = $conf->{memory}*2 + $driver_state_size;
5901
5902     my $name = "vm-$vmid-state-$snapname";
5903     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5904     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5905     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5906
5907     return $volid;
5908 };
5909
5910 sub snapshot_save_vmstate {
5911     my ($vmid, $conf, $snapname, $storecfg) = @_;
5912
5913     my $snap = $conf->{snapshots}->{$snapname};
5914
5915     $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5916     # always overwrite machine if we save vmstate. This makes sure we
5917     # can restore it later using correct machine type
5918     $snap->{machine} = get_current_qemu_machine($vmid);
5919 }
5920
5921 sub snapshot_prepare {
5922     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5923
5924     my $snap;
5925
5926     my $updatefn =  sub {
5927
5928         my $conf = load_config($vmid);
5929
5930         die "you can't take a snapshot if it's a template\n"
5931             if is_template($conf);
5932
5933         check_lock($conf);
5934
5935         $conf->{lock} = 'snapshot';
5936
5937         die "snapshot name '$snapname' already used\n"
5938             if defined($conf->{snapshots}->{$snapname});
5939
5940         my $storecfg = PVE::Storage::config();
5941         die "snapshot feature is not available\n"
5942             if !has_feature('snapshot', $conf, $storecfg, undef, undef, $snapname eq 'vzdump');
5943
5944         $snap = $conf->{snapshots}->{$snapname} = {};
5945
5946         if ($save_vmstate && check_running($vmid)) {
5947             snapshot_save_vmstate($vmid, $conf, $snapname, $storecfg);
5948         }
5949
5950         &$snapshot_copy_config($conf, $snap);
5951
5952         $snap->{snapstate} = "prepare";
5953         $snap->{snaptime} = time();
5954         $snap->{description} = $comment if $comment;
5955
5956         write_config($vmid, $conf);
5957     };
5958
5959     lock_config($vmid, $updatefn);
5960
5961     return $snap;
5962 }
5963
5964 sub snapshot_commit {
5965     my ($vmid, $snapname) = @_;
5966
5967     my $updatefn = sub {
5968
5969         my $conf = load_config($vmid);
5970
5971         die "missing snapshot lock\n"
5972             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5973
5974         my $snap = $conf->{snapshots}->{$snapname};
5975
5976         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5977
5978         die "wrong snapshot state\n"
5979             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5980
5981         delete $snap->{snapstate};
5982         delete $conf->{lock};
5983
5984         $conf->{parent} = $snapname;
5985
5986         write_config($vmid, $conf);
5987     };
5988
5989     lock_config($vmid, $updatefn);
5990 }
5991
5992 sub snapshot_rollback {
5993     my ($vmid, $snapname) = @_;
5994
5995     my $prepare = 1;
5996
5997     my $storecfg = PVE::Storage::config();
5998
5999     my $conf = load_config($vmid);
6000
6001     my $get_snapshot_config = sub {
6002
6003         die "you can't rollback if vm is a template\n" if is_template($conf);
6004
6005         my $res = $conf->{snapshots}->{$snapname};
6006
6007         die "snapshot '$snapname' does not exist\n" if !defined($res);
6008
6009         return $res;
6010     };
6011
6012     my $snap = &$get_snapshot_config();
6013
6014     foreach_drive($snap, sub {
6015         my ($ds, $drive) = @_;
6016
6017         return if drive_is_cdrom($drive);
6018
6019         my $volid = $drive->{file};
6020
6021         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
6022     });
6023
6024     my $updatefn = sub {
6025
6026         $conf = load_config($vmid);
6027
6028         $snap = &$get_snapshot_config();
6029
6030         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6031             if $snap->{snapstate};
6032
6033         if ($prepare) {
6034             check_lock($conf);
6035             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6036         }
6037
6038         die "unable to rollback vm $vmid: vm is running\n"
6039             if check_running($vmid);
6040
6041         if ($prepare) {
6042             $conf->{lock} = 'rollback';
6043         } else {
6044             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6045             delete $conf->{lock};
6046         }
6047
6048         my $forcemachine;
6049
6050         if (!$prepare) {
6051             my $has_machine_config = defined($conf->{machine});
6052
6053             # copy snapshot config to current config
6054             $conf = &$snapshot_apply_config($conf, $snap);
6055             $conf->{parent} = $snapname;
6056
6057             # Note: old code did not store 'machine', so we try to be smart
6058             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6059             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6060             # we remove the 'machine' configuration if not explicitly specified
6061             # in the original config.
6062             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6063         }
6064
6065         write_config($vmid, $conf);
6066
6067         if (!$prepare && $snap->{vmstate}) {
6068             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6069             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6070         }
6071     };
6072
6073     lock_config($vmid, $updatefn);
6074
6075     foreach_drive($snap, sub {
6076         my ($ds, $drive) = @_;
6077
6078         return if drive_is_cdrom($drive);
6079
6080         my $volid = $drive->{file};
6081         my $device = "drive-$ds";
6082
6083         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6084     });
6085
6086     $prepare = 0;
6087     lock_config($vmid, $updatefn);
6088 }
6089
6090 my $savevm_wait = sub {
6091     my ($vmid) = @_;
6092
6093     for(;;) {
6094         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6095         if (!$stat->{status}) {
6096             die "savevm not active\n";
6097         } elsif ($stat->{status} eq 'active') {
6098             sleep(1);
6099             next;
6100         } elsif ($stat->{status} eq 'completed') {
6101             last;
6102         } else {
6103             die "query-savevm returned status '$stat->{status}'\n";
6104         }
6105     }
6106 };
6107
6108 sub do_snapshots_with_qemu {
6109     my ($storecfg, $volid) = @_;
6110
6111     my $storage_name = PVE::Storage::parse_volume_id($volid);
6112
6113     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6114         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6115         return 1;
6116     }
6117
6118     if ($volid =~ m/\.(qcow2|qed)$/){
6119         return 1;
6120     }
6121
6122     return undef;
6123 }
6124
6125 sub qga_check_running {
6126     my ($vmid) = @_;
6127
6128     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6129     if ($@) {
6130         warn "Qemu Guest Agent are not running - $@";
6131         return 0;
6132     }
6133     return 1;
6134 }
6135
6136 sub check_freeze_needed {
6137     my ($vmid, $config, $save_vmstate) = @_;
6138
6139     my $running = check_running($vmid);
6140     if ($save_vmstate) {
6141         return ($running, $running && $config->{agent} && qga_check_running($vmid));
6142     } else {
6143         return ($running, 0);
6144     }
6145 }
6146
6147 sub snapshot_create {
6148     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6149
6150     my $snap = snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6151
6152     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6153
6154     my $config = load_config($vmid);
6155
6156     my ($running, $freezefs) = check_freeze_needed($vmid, $config, $snap->{vmstate});
6157
6158     my $drivehash = {};
6159
6160     eval {
6161         if ($freezefs) {
6162             eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6163             warn "guest-fsfreeze-freeze problems - $@" if $@;
6164         }
6165
6166         # create internal snapshots of all drives
6167
6168         my $storecfg = PVE::Storage::config();
6169
6170         if ($running) {
6171             if ($snap->{vmstate}) {
6172                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6173                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6174                 &$savevm_wait($vmid);
6175             } else {
6176                 vm_mon_cmd($vmid, "savevm-start");
6177             }
6178         };
6179
6180         foreach_drive($snap, sub {
6181             my ($ds, $drive) = @_;
6182
6183             return if drive_is_cdrom($drive);
6184
6185             my $volid = $drive->{file};
6186             my $device = "drive-$ds";
6187
6188             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6189             $drivehash->{$ds} = 1;
6190        });
6191     };
6192     my $err = $@;
6193
6194     if ($running) {
6195         eval { vm_mon_cmd($vmid, "savevm-end")  };
6196         warn $@ if $@;
6197
6198         if ($freezefs) {
6199             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6200             warn "guest-fsfreeze-thaw problems - $@" if $@;
6201         }
6202
6203         # savevm-end is async, we need to wait
6204         for (;;) {
6205             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6206             if (!$stat->{bytes}) {
6207                 last;
6208             } else {
6209                 print "savevm not yet finished\n";
6210                 sleep(1);
6211                 next;
6212             }
6213         }
6214     }
6215
6216     if ($err) {
6217         warn "snapshot create failed: starting cleanup\n";
6218         eval { snapshot_delete($vmid, $snapname, 1, $drivehash); };
6219         warn $@ if $@;
6220         die $err;
6221     }
6222
6223     snapshot_commit($vmid, $snapname);
6224 }
6225
6226 # Note: $drivehash is only set when called from snapshot_create.
6227 sub snapshot_delete {
6228     my ($vmid, $snapname, $force, $drivehash) = @_;
6229
6230     my $prepare = 1;
6231
6232     my $snap;
6233     my $unused = [];
6234
6235     my $unlink_parent = sub {
6236         my ($confref, $new_parent) = @_;
6237
6238         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6239             if ($new_parent) {
6240                 $confref->{parent} = $new_parent;
6241             } else {
6242                 delete $confref->{parent};
6243             }
6244         }
6245     };
6246
6247     my $updatefn =  sub {
6248         my ($remove_drive) = @_;
6249
6250         my $conf = load_config($vmid);
6251
6252         if (!$drivehash) {
6253             check_lock($conf);
6254             die "you can't delete a snapshot if vm is a template\n"
6255                 if is_template($conf);
6256         }
6257
6258         $snap = $conf->{snapshots}->{$snapname};
6259
6260         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6261
6262         # remove parent refs
6263         if (!$prepare) {
6264             &$unlink_parent($conf, $snap->{parent});
6265             foreach my $sn (keys %{$conf->{snapshots}}) {
6266                 next if $sn eq $snapname;
6267                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6268             }
6269         }
6270
6271         if ($remove_drive) {
6272             if ($remove_drive eq 'vmstate') {
6273                 delete $snap->{$remove_drive};
6274             } else {
6275                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6276                 my $volid = $drive->{file};
6277                 delete $snap->{$remove_drive};
6278                 add_unused_volume($conf, $volid);
6279             }
6280         }
6281
6282         if ($prepare) {
6283             $snap->{snapstate} = 'delete';
6284         } else {
6285             delete $conf->{snapshots}->{$snapname};
6286             delete $conf->{lock} if $drivehash;
6287             foreach my $volid (@$unused) {
6288                 add_unused_volume($conf, $volid);
6289             }
6290         }
6291
6292         write_config($vmid, $conf);
6293     };
6294
6295     lock_config($vmid, $updatefn);
6296
6297     # now remove vmstate file
6298
6299     my $storecfg = PVE::Storage::config();
6300
6301     if ($snap->{vmstate}) {
6302         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6303         if (my $err = $@) {
6304             die $err if !$force;
6305             warn $err;
6306         }
6307         # save changes (remove vmstate from snapshot)
6308         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6309     };
6310
6311     # now remove all internal snapshots
6312     foreach_drive($snap, sub {
6313         my ($ds, $drive) = @_;
6314
6315         return if drive_is_cdrom($drive);
6316
6317         my $volid = $drive->{file};
6318         my $device = "drive-$ds";
6319
6320         if (!$drivehash || $drivehash->{$ds}) {
6321             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6322             if (my $err = $@) {
6323                 die $err if !$force;
6324                 warn $err;
6325             }
6326         }
6327
6328         # save changes (remove drive fron snapshot)
6329         lock_config($vmid, $updatefn, $ds) if !$force;
6330         push @$unused, $volid;
6331     });
6332
6333     # now cleanup config
6334     $prepare = 0;
6335     lock_config($vmid, $updatefn);
6336 }
6337
6338 sub has_feature {
6339     my ($feature, $conf, $storecfg, $snapname, $running, $backup_only) = @_;
6340
6341     my $err;
6342     foreach_drive($conf, sub {
6343         my ($ds, $drive) = @_;
6344
6345         return if drive_is_cdrom($drive);
6346         return if $backup_only && !$drive->{backup};
6347         my $volid = $drive->{file};
6348         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6349     });
6350
6351     return $err ? 0 : 1;
6352 }
6353
6354 sub template_create {
6355     my ($vmid, $conf, $disk) = @_;
6356
6357     my $storecfg = PVE::Storage::config();
6358
6359     foreach_drive($conf, sub {
6360         my ($ds, $drive) = @_;
6361
6362         return if drive_is_cdrom($drive);
6363         return if $disk && $ds ne $disk;
6364
6365         my $volid = $drive->{file};
6366         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6367
6368         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6369         $drive->{file} = $voliddst;
6370         $conf->{$ds} = print_drive($vmid, $drive);
6371         write_config($vmid, $conf);
6372     });
6373 }
6374
6375 sub is_template {
6376     my ($conf) = @_;
6377
6378     return 1 if defined $conf->{template} && $conf->{template} == 1;
6379 }
6380
6381 sub qemu_img_convert {
6382     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6383
6384     my $storecfg = PVE::Storage::config();
6385     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6386     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6387
6388     if ($src_storeid && $dst_storeid) {
6389
6390         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6391
6392         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6393         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6394
6395         my $src_format = qemu_img_format($src_scfg, $src_volname);
6396         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6397
6398         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6399         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6400
6401         my $cmd = [];
6402         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6403         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6404         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6405
6406         my $parser = sub {
6407             my $line = shift;
6408             if($line =~ m/\((\S+)\/100\%\)/){
6409                 my $percent = $1;
6410                 my $transferred = int($size * $percent / 100);
6411                 my $remaining = $size - $transferred;
6412
6413                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6414             }
6415
6416         };
6417
6418         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6419         my $err = $@;
6420         die "copy failed: $err" if $err;
6421     }
6422 }
6423
6424 sub qemu_img_format {
6425     my ($scfg, $volname) = @_;
6426
6427     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6428         return $1;
6429     } else {
6430         return "raw";
6431     }
6432 }
6433
6434 sub qemu_drive_mirror {
6435     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6436
6437     my $storecfg = PVE::Storage::config();
6438     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6439
6440     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6441
6442     my $format = qemu_img_format($dst_scfg, $dst_volname);
6443
6444     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6445
6446     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6447     $opts->{format} = $format if $format;
6448
6449     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6450
6451     eval {
6452     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6453         while (1) {
6454             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6455             my $stat = @$stats[0];
6456             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6457             die "error job is not mirroring" if $stat->{type} ne "mirror";
6458
6459             my $busy = $stat->{busy};
6460             my $ready = $stat->{ready};
6461
6462             if (my $total = $stat->{len}) {
6463                 my $transferred = $stat->{offset} || 0;
6464                 my $remaining = $total - $transferred;
6465                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6466
6467                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6468             }
6469
6470
6471             if ($stat->{ready} eq 'true') {
6472
6473                 last if $vmiddst != $vmid;
6474
6475                 # try to switch the disk if source and destination are on the same guest
6476                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6477                 last if !$@;
6478                 die $@ if $@ !~ m/cannot be completed/;
6479             }
6480             sleep 1;
6481         }
6482
6483
6484     };
6485     my $err = $@;
6486
6487     my $cancel_job = sub {
6488         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6489         while (1) {
6490             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6491             my $stat = @$stats[0];
6492             last if !$stat;
6493             sleep 1;
6494         }
6495     };
6496
6497     if ($err) {
6498         eval { &$cancel_job(); };
6499         die "mirroring error: $err";
6500     }
6501
6502     if ($vmiddst != $vmid) {
6503         # if we clone a disk for a new target vm, we don't switch the disk
6504         &$cancel_job(); # so we call block-job-cancel
6505     }
6506 }
6507
6508 sub clone_disk {
6509     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6510         $newvmid, $storage, $format, $full, $newvollist) = @_;
6511
6512     my $newvolid;
6513
6514     if (!$full) {
6515         print "create linked clone of drive $drivename ($drive->{file})\n";
6516         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6517         push @$newvollist, $newvolid;
6518     } else {
6519         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6520         $storeid = $storage if $storage;
6521
6522         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6523         if (!$format) {
6524             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6525             $format = qemu_img_format($scfg, $volname);
6526         }
6527
6528         # test if requested format is supported - else use default
6529         my $supported = grep { $_ eq $format } @$validFormats;
6530         $format = $defFormat if !$supported;
6531
6532         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6533
6534         print "create full clone of drive $drivename ($drive->{file})\n";
6535         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6536         push @$newvollist, $newvolid;
6537
6538         PVE::Storage::activate_volumes($storecfg, $newvollist);
6539
6540         if (!$running || $snapname) {
6541             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6542         } else {
6543             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6544         }
6545     }
6546
6547     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6548
6549     my $disk = $drive;
6550     $disk->{format} = undef;
6551     $disk->{file} = $newvolid;
6552     $disk->{size} = $size;
6553
6554     return $disk;
6555 }
6556
6557 # this only works if VM is running
6558 sub get_current_qemu_machine {
6559     my ($vmid) = @_;
6560
6561     my $cmd = { execute => 'query-machines', arguments => {} };
6562     my $res = vm_qmp_command($vmid, $cmd);
6563
6564     my ($current, $default);
6565     foreach my $e (@$res) {
6566         $default = $e->{name} if $e->{'is-default'};
6567         $current = $e->{name} if $e->{'is-current'};
6568     }
6569
6570     # fallback to the default machine if current is not supported by qemu
6571     return $current || $default || 'pc';
6572 }
6573
6574 sub qemu_machine_feature_enabled {
6575     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6576
6577     my $current_major;
6578     my $current_minor;
6579
6580     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6581
6582         $current_major = $3;
6583         $current_minor = $4;
6584
6585     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6586
6587         $current_major = $1;
6588         $current_minor = $2;
6589     }
6590
6591     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6592
6593
6594 }
6595
6596 sub qemu_machine_pxe {
6597     my ($vmid, $conf, $machine) = @_;
6598
6599     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6600
6601     foreach my $opt (keys %$conf) {
6602         next if $opt !~ m/^net(\d+)$/;
6603         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6604         next if !$net;
6605         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6606         return $machine.".pxe" if $romfile =~ m/pxe/;
6607         last;
6608     }
6609
6610     return $machine;
6611 }
6612
6613 sub qemu_use_old_bios_files {
6614     my ($machine_type) = @_;
6615
6616     return if !$machine_type;
6617
6618     my $use_old_bios_files = undef;
6619
6620     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6621         $machine_type = $1;
6622         $use_old_bios_files = 1;
6623     } else {
6624         my $kvmver = kvm_user_version();
6625         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6626         # load new efi bios files on migration. So this hack is required to allow
6627         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6628         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6629         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6630     }
6631
6632     return ($use_old_bios_files, $machine_type);
6633 }
6634
6635 sub lspci {
6636
6637     my $devices = {};
6638
6639     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6640             my (undef, $id, $function) = @_;
6641             my $res = { id => $id, function => $function};
6642             push @{$devices->{$id}}, $res;
6643     });
6644
6645     return $devices;
6646 }
6647
6648 sub vm_iothreads_list {
6649     my ($vmid) = @_;
6650
6651     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6652
6653     my $iothreads = {};
6654     foreach my $iothread (@$res) {
6655         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6656     }
6657
6658     return $iothreads;
6659 }
6660
6661 sub scsihw_infos {
6662     my ($conf, $drive) = @_;
6663
6664     my $maxdev = 0;
6665
6666     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6667         $maxdev = 7;
6668     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6669         $maxdev = 1;
6670     } else {
6671         $maxdev = 256;
6672     }
6673
6674     my $controller = int($drive->{index} / $maxdev);
6675     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6676
6677     return ($maxdev, $controller, $controller_prefix);
6678 }
6679
6680 # bash completion helper
6681
6682 sub complete_backup_archives {
6683     my ($cmdname, $pname, $cvalue) = @_;
6684
6685     my $cfg = PVE::Storage::config();
6686
6687     my $storeid;
6688
6689     if ($cvalue =~ m/^([^:]+):/) {
6690         $storeid = $1;
6691     }
6692
6693     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6694
6695     my $res = [];
6696     foreach my $id (keys %$data) {
6697         foreach my $item (@{$data->{$id}}) {
6698             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6699             push @$res, $item->{volid} if defined($item->{volid});
6700         }
6701     }
6702
6703     return $res;
6704 }
6705
6706 my $complete_vmid_full = sub {
6707     my ($running) = @_;
6708
6709     my $idlist = vmstatus();
6710
6711     my $res = [];
6712
6713     foreach my $id (keys %$idlist) {
6714         my $d = $idlist->{$id};
6715         if (defined($running)) {
6716             next if $d->{template};
6717             next if $running && $d->{status} ne 'running';
6718             next if !$running && $d->{status} eq 'running';
6719         }
6720         push @$res, $id;
6721
6722     }
6723     return $res;
6724 };
6725
6726 sub complete_vmid {
6727     return &$complete_vmid_full();
6728 }
6729
6730 sub complete_vmid_stopped {
6731     return &$complete_vmid_full(0);
6732 }
6733
6734 sub complete_vmid_running {
6735     return &$complete_vmid_full(1);
6736 }
6737
6738 sub complete_storage {
6739
6740     my $cfg = PVE::Storage::config();
6741     my $ids = $cfg->{ids};
6742
6743     my $res = [];
6744     foreach my $sid (keys %$ids) {
6745         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6746         next if !$ids->{$sid}->{content}->{images};
6747         push @$res, $sid;
6748     }
6749
6750     return $res;
6751 }
6752
6753 1;