PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $cpudesc = {
  94     cputype => {
  95         description => "Emulated CPU type.",
  96         type => 'string',
  97         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  98         default => 'kvm64',
  99         default_key => 1,
 100     },
 101     hidden => {
 102         description => "Do not identify as a KVM virtual machine.",
 103         type => 'boolean',
 104         optional => 1,
 105         default => 0
 106     },
 107 };
 108
 109 my $confdesc = {
 110     onboot => {
 111         optional => 1,
 112         type => 'boolean',
 113         description => "Specifies whether a VM will be started during system bootup.",
 114         default => 0,
 115     },
 116     autostart => {
 117         optional => 1,
 118         type => 'boolean',
 119         description => "Automatic restart after crash (currently ignored).",
 120         default => 0,
 121     },
 122     hotplug => {
 123         optional => 1,
 124         type => 'string', format => 'pve-hotplug-features',
 125         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 126         default => 'network,disk,usb',
 127     },
 128     reboot => {
 129         optional => 1,
 130         type => 'boolean',
 131         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 132         default => 1,
 133     },
 134     lock => {
 135         optional => 1,
 136         type => 'string',
 137         description => "Lock/unlock the VM.",
 138         enum => [qw(migrate backup snapshot rollback)],
 139     },
 140     cpulimit => {
 141         optional => 1,
 142         type => 'number',
 143         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 144         minimum => 0,
 145         maximum => 128,
 146         default => 0,
 147     },
 148     cpuunits => {
 149         optional => 1,
 150         type => 'integer',
 151         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 152         minimum => 0,
 153         maximum => 500000,
 154         default => 1000,
 155     },
 156     memory => {
 157         optional => 1,
 158         type => 'integer',
 159         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 160         minimum => 16,
 161         default => 512,
 162     },
 163     balloon => {
 164         optional => 1,
 165         type => 'integer',
 166         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 167         minimum => 0,
 168     },
 169     shares => {
 170         optional => 1,
 171         type => 'integer',
 172         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 173         minimum => 0,
 174         maximum => 50000,
 175         default => 1000,
 176     },
 177     keyboard => {
 178         optional => 1,
 179         type => 'string',
 180         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 181         enum => PVE::Tools::kvmkeymaplist(),
 182         default => 'en-us',
 183     },
 184     name => {
 185         optional => 1,
 186         type => 'string', format => 'dns-name',
 187         description => "Set a name for the VM. Only used on the configuration web interface.",
 188     },
 189     scsihw => {
 190         optional => 1,
 191         type => 'string',
 192         description => "scsi controller model",
 193         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 194         default => 'lsi',
 195     },
 196     description => {
 197         optional => 1,
 198         type => 'string',
 199         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 200     },
 201     ostype => {
 202         optional => 1,
 203         type => 'string',
 204         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 205         description => <<EODESC,
 206 Used to enable special optimization/features for specific
 207 operating systems:
 208
 209 other  => unspecified OS
 210 wxp    => Microsoft Windows XP
 211 w2k    => Microsoft Windows 2000
 212 w2k3   => Microsoft Windows 2003
 213 w2k8   => Microsoft Windows 2008
 214 wvista => Microsoft Windows Vista
 215 win7   => Microsoft Windows 7
 216 win8   => Microsoft Windows 8/2012
 217 l24    => Linux 2.4 Kernel
 218 l26    => Linux 2.6/3.X Kernel
 219 solaris => solaris/opensolaris/openindiania kernel
 220
 221 other|l24|l26|solaris                       ... no special behaviour
 222 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 223 EODESC
 224     },
 225     boot => {
 226         optional => 1,
 227         type => 'string',
 228         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 229         pattern => '[acdn]{1,4}',
 230         default => 'cdn',
 231     },
 232     bootdisk => {
 233         optional => 1,
 234         type => 'string', format => 'pve-qm-bootdisk',
 235         description => "Enable booting from specified disk.",
 236         pattern => '(ide|sata|scsi|virtio)\d+',
 237     },
 238     smp => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "The number of CPUs. Please use option -sockets instead.",
 242         minimum => 1,
 243         default => 1,
 244     },
 245     sockets => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "The number of CPU sockets.",
 249         minimum => 1,
 250         default => 1,
 251     },
 252     cores => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "The number of cores per socket.",
 256         minimum => 1,
 257         default => 1,
 258     },
 259     numa => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Enable/disable Numa.",
 263         default => 0,
 264     },
 265     vcpus => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Number of hotplugged vcpus.",
 269         minimum => 1,
 270         default => 0,
 271     },
 272     acpi => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable ACPI.",
 276         default => 1,
 277     },
 278     agent => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Enable/disable Qemu GuestAgent.",
 282         default => 0,
 283     },
 284     kvm => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Enable/disable KVM hardware virtualization.",
 288         default => 1,
 289     },
 290     tdf => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Enable/disable time drift fix.",
 294         default => 0,
 295     },
 296     localtime => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 300     },
 301     freeze => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 305     },
 306     vga => {
 307         optional => 1,
 308         type => 'string',
 309         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 310         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 311     },
 312     watchdog => {
 313         optional => 1,
 314         type => 'string', format => 'pve-qm-watchdog',
 315         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 316         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 317     },
 318     startdate => {
 319         optional => 1,
 320         type => 'string',
 321         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 322         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 323         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 324         default => 'now',
 325     },
 326     startup =>  get_standard_option('pve-startup-order'),
 327     template => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Enable/disable Template.",
 331         default => 0,
 332     },
 333     args => {
 334         optional => 1,
 335         type => 'string',
 336         description => <<EODESCR,
 337 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 338
 339 args: -no-reboot -no-hpet
 340 EODESCR
 341     },
 342     tablet => {
 343         optional => 1,
 344         type => 'boolean',
 345         default => 1,
 346         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 347     },
 348     migrate_speed => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 352         minimum => 0,
 353         default => 0,
 354     },
 355     migrate_downtime => {
 356         optional => 1,
 357         type => 'number',
 358         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 359         minimum => 0,
 360         default => 0.1,
 361     },
 362     cdrom => {
 363         optional => 1,
 364         type => 'string', format => 'pve-qm-drive',
 365         typetext => 'volume',
 366         description => "This is an alias for option -ide2",
 367     },
 368     cpu => {
 369         optional => 1,
 370         description => "Emulated CPU type.",
 371         type => 'string',
 372         format => $cpudesc,
 373     },
 374     parent => get_standard_option('pve-snapshot-name', {
 375         optional => 1,
 376         description => "Parent snapshot name. This is used internally, and should not be modified.",
 377     }),
 378     snaptime => {
 379         optional => 1,
 380         description => "Timestamp for snapshots.",
 381         type => 'integer',
 382         minimum => 0,
 383     },
 384     vmstate => {
 385         optional => 1,
 386         type => 'string', format => 'pve-volume-id',
 387         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 388     },
 389     machine => {
 390         description => "Specific the Qemu machine type.",
 391         type => 'string',
 392         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 393         maxLength => 40,
 394         optional => 1,
 395     },
 396     smbios1 => {
 397         description => "Specify SMBIOS type 1 fields.",
 398         type => 'string', format => 'pve-qm-smbios1',
 399         maxLength => 256,
 400         optional => 1,
 401     },
 402     protection => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 406         default => 0,
 407     },
 408     bios => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [ qw(seabios ovmf) ],
 412         description => "Select BIOS implementation.",
 413         default => 'seabios',
 414     },
 415 };
 416
 417 # what about other qemu settings ?
 418 #cpu => 'string',
 419 #machine => 'string',
 420 #fda => 'file',
 421 #fdb => 'file',
 422 #mtdblock => 'file',
 423 #sd => 'file',
 424 #pflash => 'file',
 425 #snapshot => 'bool',
 426 #bootp => 'file',
 427 ##tftp => 'dir',
 428 ##smb => 'dir',
 429 #kernel => 'file',
 430 #append => 'string',
 431 #initrd => 'file',
 432 ##soundhw => 'string',
 433
 434 while (my ($k, $v) = each %$confdesc) {
 435     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 436 }
 437
 438 my $MAX_IDE_DISKS = 4;
 439 my $MAX_SCSI_DISKS = 14;
 440 my $MAX_VIRTIO_DISKS = 16;
 441 my $MAX_SATA_DISKS = 6;
 442 my $MAX_USB_DEVICES = 5;
 443 my $MAX_NETS = 32;
 444 my $MAX_UNUSED_DISKS = 8;
 445 my $MAX_HOSTPCI_DEVICES = 4;
 446 my $MAX_SERIAL_PORTS = 4;
 447 my $MAX_PARALLEL_PORTS = 3;
 448 my $MAX_NUMA = 8;
 449 my $MAX_MEM = 4194304;
 450 my $STATICMEM = 1024;
 451
 452 my $numadesc = {
 453     optional => 1,
 454     type => 'string', format => 'pve-qm-numanode',
 455     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 456     description => "numa topology",
 457 };
 458 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 459
 460 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 461     $confdesc->{"numa$i"} = $numadesc;
 462 }
 463
 464 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 465                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 466                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 467 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 468
 469 my $netdesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-net',
 472     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 473     description => <<EODESCR,
 474 Specify network devices.
 475
 476 MODEL is one of: $nic_model_list_txt
 477
 478 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 479 automatically generated if not specified.
 480
 481 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 482
 483 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 484
 485 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 486
 487 10.0.2.2   Gateway
 488 10.0.2.3   DNS Server
 489 10.0.2.4   SMB Server
 490
 491 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 492
 493 EODESCR
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 496
 497 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 498     $confdesc->{"net$i"} = $netdesc;
 499 }
 500
 501 my $drivename_hash;
 502
 503 my %drivedesc_base = (
 504     volume => { alias => 'file' },
 505     file => {
 506         type => 'pve-volume-id',
 507         default_key => 1,
 508         format_description => 'volume',
 509         description => "The drive's backing volume.",
 510     },
 511     media => {
 512         type => 'string',
 513         format_description => 'cdrom|disk',
 514         enum => [qw(cdrom disk)],
 515         description => "The drive's media type.",
 516         default => 'disk',
 517         optional => 1
 518     },
 519     cyls => {
 520         type => 'integer',
 521         format_description => 'count',
 522         description => "Force the drive's physical geometry to have a specific cylinder count.",
 523         optional => 1
 524     },
 525     heads => {
 526         type => 'integer',
 527         format_description => 'count',
 528         description => "Force the drive's physical geometry to have a specific head count.",
 529         optional => 1
 530     },
 531     secs => {
 532         type => 'integer',
 533         format_description => 'count',
 534         description => "Force the drive's physical geometry to have a specific sector count.",
 535         optional => 1
 536     },
 537     trans => {
 538         type => 'string',
 539         format_description => 'none|lba|auto',
 540         enum => [qw(none lba auto)],
 541         description => "Force disk geometry bios translation mode.",
 542         optional => 1,
 543     },
 544     snapshot => {
 545         type => 'boolean',
 546         format_description => 'on|off',
 547         description => "Whether the drive should be included when making snapshots.",
 548         optional => 1,
 549     },
 550     cache => {
 551         type => 'string',
 552         format_description => 'none|writethrough|writeback|unsafe|directsync',
 553         enum => [qw(none writethrough writeback unsafe directsync)],
 554         description => "The drive's cache mode",
 555         optional => 1,
 556     },
 557     format => {
 558         type => 'string',
 559         format_description => 'drive format',
 560         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 561         description => "The drive's backing file's data format.",
 562         optional => 1,
 563     },
 564     size => {
 565         type => 'string',
 566         format => 'disk-size',
 567         description => "Disk size. This is purely informational and has no effect.",
 568         optional => 1,
 569     },
 570     backup => {
 571         type => 'boolean',
 572         format_description => 'on|off',
 573         description => "Whether the drive should be included when making backups.",
 574         optional => 1,
 575     },
 576     werror => {
 577         type => 'string',
 578         format_description => 'enospc|ignore|report|stop',
 579         enum => [qw(enospc ignore report stop)],
 580         description => 'Write error action.',
 581         optional => 1,
 582     },
 583     aio => {
 584         type => 'string',
 585         format_description => 'native|threads',
 586         enum => [qw(native threads)],
 587         description => 'AIO type to use.',
 588         optional => 1,
 589     },
 590     discard => {
 591         type => 'string',
 592         format_description => 'ignore|on',
 593         enum => [qw(ignore on)],
 594         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 595         optional => 1,
 596     },
 597     detect_zeroes => {
 598         type => 'boolean',
 599         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 600         optional => 1,
 601     },
 602     serial => {
 603         type => 'string',
 604         format_description => 'serial',
 605         description => "The drive's reported serial number.",
 606         optional => 1,
 607     }
 608 );
 609
 610 my %rerror_fmt = (
 611     rerror => {
 612         type => 'string',
 613         format_description => 'ignore|report|stop',
 614         enum => [qw(ignore report stop)],
 615         description => 'Read error action.',
 616         optional => 1,
 617     },
 618 );
 619
 620 my %iothread_fmt = ( iothread => {
 621         type => 'boolean',
 622         format_description => 'off|on',
 623         description => "Whether to use iothreads for this drive",
 624         optional => 1,
 625 });
 626
 627 my %model_fmt = (
 628     model => {
 629         type => 'string',
 630         format_description => 'model',
 631         description => "The drive's reported model name.",
 632         optional => 1,
 633     },
 634 );
 635
 636 my %queues_fmt = (
 637     queues => {
 638         type => 'integer',
 639         format_description => 'nbqueues',
 640         description => "Number of queues.",
 641         minimum => 2,
 642         optional => 1
 643     }
 644 );
 645
 646 my $add_throttle_desc = sub {
 647     my ($key, $type, $what, $size, $longsize) = @_;
 648     $drivedesc_base{$key} = {
 649         type => $type,
 650         format_description => $size,
 651         description => "Maximum $what speed in $longsize per second.",
 652         optional => 1,
 653     };
 654 };
 655 # throughput: (leaky bucket)
 656 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 657 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 658 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 659 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 660 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 662 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 663 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 664 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 665
 666 # pools: (pool of IO before throttling starts taking effect)
 667 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 668 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 670 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 671 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 672 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 673
 674 my $ide_fmt = {
 675     %drivedesc_base,
 676     %rerror_fmt,
 677     %model_fmt,
 678 };
 679
 680 my $idedesc = {
 681     optional => 1,
 682     type => 'string', format => $ide_fmt,
 683     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 684 };
 685 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 686
 687 my $scsi_fmt = {
 688     %drivedesc_base,
 689     %iothread_fmt,
 690     %queues_fmt,
 691 };
 692 my $scsidesc = {
 693     optional => 1,
 694     type => 'string', format => $scsi_fmt,
 695     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 696 };
 697 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 698
 699 my $sata_fmt = {
 700     %drivedesc_base,
 701     %rerror_fmt,
 702 };
 703 my $satadesc = {
 704     optional => 1,
 705     type => 'string', format => $sata_fmt,
 706     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 707 };
 708 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 709
 710 my $virtio_fmt = {
 711     %drivedesc_base,
 712     %iothread_fmt,
 713     %rerror_fmt,
 714 };
 715 my $virtiodesc = {
 716     optional => 1,
 717     type => 'string', format => $virtio_fmt,
 718     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 719 };
 720 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 721
 722 my $alldrive_fmt = {
 723     %drivedesc_base,
 724     %rerror_fmt,
 725     %iothread_fmt,
 726     %model_fmt,
 727     %queues_fmt,
 728 };
 729
 730 my $usbdesc = {
 731     optional => 1,
 732     type => 'string', format => 'pve-qm-usb-device',
 733     typetext => 'host=HOSTUSBDEVICE|spice',
 734     description => <<EODESCR,
 735 Configure an USB device (n is 0 to 4). This can be used to
 736 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 737
 738 'bus-port(.port)*' (decimal numbers) or
 739 'vendor_id:product_id' (hexadeciaml numbers)
 740
 741 You can use the 'lsusb -t' command to list existing usb devices.
 742
 743 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 744
 745 The value 'spice' can be used to add a usb redirection devices for spice.
 746
 747 EODESCR
 748 };
 749 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 750
 751 my $hostpcidesc = {
 752         optional => 1,
 753         type => 'string', format => 'pve-qm-hostpci',
 754         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 755         description => <<EODESCR,
 756 Map host pci devices. HOSTPCIDEVICE syntax is:
 757
 758 'bus:dev.func' (hexadecimal numbers)
 759
 760 You can us the 'lspci' command to list existing pci devices.
 761
 762 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 763
 764 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 765
 766 Experimental: user reported problems with this option.
 767 EODESCR
 768 };
 769 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 770
 771 my $serialdesc = {
 772         optional => 1,
 773         type => 'string',
 774         pattern => '(/dev/.+|socket)',
 775         description =>  <<EODESCR,
 776 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 777
 778 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 779
 780 Experimental: user reported problems with this option.
 781 EODESCR
 782 };
 783
 784 my $paralleldesc= {
 785         optional => 1,
 786         type => 'string',
 787         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 788         description =>  <<EODESCR,
 789 Map host parallel devices (n is 0 to 2).
 790
 791 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 792
 793 Experimental: user reported problems with this option.
 794 EODESCR
 795 };
 796
 797 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 798     $confdesc->{"parallel$i"} = $paralleldesc;
 799 }
 800
 801 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 802     $confdesc->{"serial$i"} = $serialdesc;
 803 }
 804
 805 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 806     $confdesc->{"hostpci$i"} = $hostpcidesc;
 807 }
 808
 809 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 810     $drivename_hash->{"ide$i"} = 1;
 811     $confdesc->{"ide$i"} = $idedesc;
 812 }
 813
 814 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 815     $drivename_hash->{"sata$i"} = 1;
 816     $confdesc->{"sata$i"} = $satadesc;
 817 }
 818
 819 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 820     $drivename_hash->{"scsi$i"} = 1;
 821     $confdesc->{"scsi$i"} = $scsidesc ;
 822 }
 823
 824 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 825     $drivename_hash->{"virtio$i"} = 1;
 826     $confdesc->{"virtio$i"} = $virtiodesc;
 827 }
 828
 829 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 830     $confdesc->{"usb$i"} = $usbdesc;
 831 }
 832
 833 my $unuseddesc = {
 834     optional => 1,
 835     type => 'string', format => 'pve-volume-id',
 836     description => "Reference to unused volumes.",
 837 };
 838
 839 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 840     $confdesc->{"unused$i"} = $unuseddesc;
 841 }
 842
 843 my $kvm_api_version = 0;
 844
 845 sub kvm_version {
 846
 847     return $kvm_api_version if $kvm_api_version;
 848
 849     my $fh = IO::File->new("</dev/kvm") ||
 850         return 0;
 851
 852     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 853         $kvm_api_version = $v;
 854     }
 855
 856     $fh->close();
 857
 858     return  $kvm_api_version;
 859 }
 860
 861 my $kvm_user_version;
 862
 863 sub kvm_user_version {
 864
 865     return $kvm_user_version if $kvm_user_version;
 866
 867     $kvm_user_version = 'unknown';
 868
 869     my $tmp = `kvm -help 2>/dev/null`;
 870
 871     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 872         $kvm_user_version = $2;
 873     }
 874
 875     return $kvm_user_version;
 876
 877 }
 878
 879 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 880
 881 sub disknames {
 882     # order is important - used to autoselect boot disk
 883     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 884             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 885             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 886             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 887 }
 888
 889 sub valid_drivename {
 890     my $dev = shift;
 891
 892     return defined($drivename_hash->{$dev});
 893 }
 894
 895 sub option_exists {
 896     my $key = shift;
 897     return defined($confdesc->{$key});
 898 }
 899
 900 sub nic_models {
 901     return $nic_model_list;
 902 }
 903
 904 sub os_list_description {
 905
 906     return {
 907         other => 'Other',
 908         wxp => 'Windows XP',
 909         w2k => 'Windows 2000',
 910         w2k3 =>, 'Windows 2003',
 911         w2k8 => 'Windows 2008',
 912         wvista => 'Windows Vista',
 913         win7 => 'Windows 7',
 914         win8 => 'Windows 8/2012',
 915         l24 => 'Linux 2.4',
 916         l26 => 'Linux 2.6',
 917     };
 918 }
 919
 920 my $cdrom_path;
 921
 922 sub get_cdrom_path {
 923
 924     return  $cdrom_path if $cdrom_path;
 925
 926     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 927     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 928     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 929 }
 930
 931 sub get_iso_path {
 932     my ($storecfg, $vmid, $cdrom) = @_;
 933
 934     if ($cdrom eq 'cdrom') {
 935         return get_cdrom_path();
 936     } elsif ($cdrom eq 'none') {
 937         return '';
 938     } elsif ($cdrom =~ m|^/|) {
 939         return $cdrom;
 940     } else {
 941         return PVE::Storage::path($storecfg, $cdrom);
 942     }
 943 }
 944
 945 # try to convert old style file names to volume IDs
 946 sub filename_to_volume_id {
 947     my ($vmid, $file, $media) = @_;
 948
 949     if (!($file eq 'none' || $file eq 'cdrom' ||
 950           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 951
 952         return undef if $file =~ m|/|;
 953
 954         if ($media && $media eq 'cdrom') {
 955             $file = "local:iso/$file";
 956         } else {
 957             $file = "local:$vmid/$file";
 958         }
 959     }
 960
 961     return $file;
 962 }
 963
 964 sub verify_media_type {
 965     my ($opt, $vtype, $media) = @_;
 966
 967     return if !$media;
 968
 969     my $etype;
 970     if ($media eq 'disk') {
 971         $etype = 'images';
 972     } elsif ($media eq 'cdrom') {
 973         $etype = 'iso';
 974     } else {
 975         die "internal error";
 976     }
 977
 978     return if ($vtype eq $etype);
 979
 980     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 981 }
 982
 983 sub cleanup_drive_path {
 984     my ($opt, $storecfg, $drive) = @_;
 985
 986     # try to convert filesystem paths to volume IDs
 987
 988     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 989         ($drive->{file} !~ m|^/dev/.+|) &&
 990         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 991         ($drive->{file} !~ m/^\d+$/)) {
 992         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 993         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 994         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 995         verify_media_type($opt, $vtype, $drive->{media});
 996         $drive->{file} = $volid;
 997     }
 998
 999     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1000 }
1001
1002 sub create_conf_nolock {
1003     my ($vmid, $settings) = @_;
1004
1005     my $filename = config_file($vmid);
1006
1007     die "configuration file '$filename' already exists\n" if -f $filename;
1008
1009     my $defaults = load_defaults();
1010
1011     $settings->{name} = "vm$vmid" if !$settings->{name};
1012     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
1013
1014     my $data = '';
1015     foreach my $opt (keys %$settings) {
1016         next if !$confdesc->{$opt};
1017
1018         my $value = $settings->{$opt};
1019         next if !$value;
1020
1021         $data .= "$opt: $value\n";
1022     }
1023
1024     PVE::Tools::file_set_contents($filename, $data);
1025 }
1026
1027 sub parse_hotplug_features {
1028     my ($data) = @_;
1029
1030     my $res = {};
1031
1032     return $res if $data eq '0';
1033
1034     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1035
1036     foreach my $feature (PVE::Tools::split_list($data)) {
1037         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1038             $res->{$1} = 1;
1039         } else {
1040             warn "ignoring unknown hotplug feature '$feature'\n";
1041         }
1042     }
1043     return $res;
1044 }
1045
1046 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1047 sub pve_verify_hotplug_features {
1048     my ($value, $noerr) = @_;
1049
1050     return $value if parse_hotplug_features($value);
1051
1052     return undef if $noerr;
1053
1054     die "unable to parse hotplug option\n";
1055 }
1056
1057 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1058 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1059 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1060 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1061 #        [,iothread=on][,serial=serial][,model=model]
1062
1063 sub parse_drive {
1064     my ($key, $data) = @_;
1065
1066     my ($interface, $index);
1067
1068     if ($key =~ m/^([^\d]+)(\d+)$/) {
1069         $interface = $1;
1070         $index = $2;
1071     } else {
1072         return undef;
1073     }
1074
1075     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1076                                      : $confdesc->{$key}->{format};
1077     if (!$desc) {
1078         warn "invalid drive key: $key\n";
1079         return undef;
1080     }
1081     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1082     return undef if !$res;
1083     $res->{interface} = $interface;
1084     $res->{index} = $index;
1085
1086     my $error = 0;
1087     foreach my $opt (qw(bps bps_rd bps_wr)) {
1088         if (my $bps = defined(delete $res->{$opt})) {
1089             if (defined($res->{"m$opt"})) {
1090                 warn "both $opt and m$opt specified\n";
1091                 ++$error;
1092                 next;
1093             }
1094             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1095         }
1096     }
1097     return undef if $error;
1098
1099     return undef if $res->{mbps_rd} && $res->{mbps};
1100     return undef if $res->{mbps_wr} && $res->{mbps};
1101     return undef if $res->{iops_rd} && $res->{iops};
1102     return undef if $res->{iops_wr} && $res->{iops};
1103
1104     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1105         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1106         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1107         return undef if $res->{interface} eq 'virtio';
1108     }
1109
1110     if (my $size = $res->{size}) {
1111         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1112     }
1113
1114     return $res;
1115 }
1116
1117 sub print_drive {
1118     my ($vmid, $drive) = @_;
1119     my $data = { %$drive };
1120     delete $data->{$_} for qw(index interface);
1121     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1122 }
1123
1124 sub scsi_inquiry {
1125     my($fh, $noerr) = @_;
1126
1127     my $SG_IO = 0x2285;
1128     my $SG_GET_VERSION_NUM = 0x2282;
1129
1130     my $versionbuf = "\x00" x 8;
1131     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1132     if (!$ret) {
1133         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1134         return undef;
1135     }
1136     my $version = unpack("I", $versionbuf);
1137     if ($version < 30000) {
1138         die "scsi generic interface too old\n"  if !$noerr;
1139         return undef;
1140     }
1141
1142     my $buf = "\x00" x 36;
1143     my $sensebuf = "\x00" x 8;
1144     my $cmd = pack("C x3 C x1", 0x12, 36);
1145
1146     # see /usr/include/scsi/sg.h
1147     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1148
1149     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1150                       length($sensebuf), 0, length($buf), $buf,
1151                       $cmd, $sensebuf, 6000);
1152
1153     $ret = ioctl($fh, $SG_IO, $packet);
1154     if (!$ret) {
1155         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1156         return undef;
1157     }
1158
1159     my @res = unpack($sg_io_hdr_t, $packet);
1160     if ($res[17] || $res[18]) {
1161         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1162         return undef;
1163     }
1164
1165     my $res = {};
1166     (my $byte0, my $byte1, $res->{vendor},
1167      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1168
1169     $res->{removable} = $byte1 & 128 ? 1 : 0;
1170     $res->{type} = $byte0 & 31;
1171
1172     return $res;
1173 }
1174
1175 sub path_is_scsi {
1176     my ($path) = @_;
1177
1178     my $fh = IO::File->new("+<$path") || return undef;
1179     my $res = scsi_inquiry($fh, 1);
1180     close($fh);
1181
1182     return $res;
1183 }
1184
1185 sub machine_type_is_q35 {
1186     my ($conf) = @_;
1187
1188     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1189 }
1190
1191 sub print_tabletdevice_full {
1192     my ($conf) = @_;
1193
1194     my $q35 = machine_type_is_q35($conf);
1195
1196     # we use uhci for old VMs because tablet driver was buggy in older qemu
1197     my $usbbus = $q35 ? "ehci" : "uhci";
1198
1199     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1200 }
1201
1202 sub print_drivedevice_full {
1203     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1204
1205     my $device = '';
1206     my $maxdev = 0;
1207
1208     if ($drive->{interface} eq 'virtio') {
1209         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1210         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1211         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1212     } elsif ($drive->{interface} eq 'scsi') {
1213
1214         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1215         my $unit = $drive->{index} % $maxdev;
1216         my $devicetype = 'hd';
1217         my $path = '';
1218         if (drive_is_cdrom($drive)) {
1219               $devicetype = 'cd';
1220         } else {
1221               if ($drive->{file} =~ m|^/|) {
1222                   $path = $drive->{file};
1223               } else {
1224                   $path = PVE::Storage::path($storecfg, $drive->{file});
1225               }
1226
1227               if($path =~ m/^iscsi\:\/\//){
1228                   $devicetype = 'generic';
1229               } else {
1230                   if (my $info = path_is_scsi($path)) {
1231                       if ($info->{type} == 0) {
1232                           $devicetype = 'block';
1233                       } elsif ($info->{type} == 1) { # tape
1234                           $devicetype = 'generic';
1235                       }
1236                   }
1237               }
1238          }
1239
1240         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1241             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1242         } else {
1243             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1244         }
1245
1246     } elsif ($drive->{interface} eq 'ide'){
1247         $maxdev = 2;
1248         my $controller = int($drive->{index} / $maxdev);
1249         my $unit = $drive->{index} % $maxdev;
1250         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1251
1252         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1253         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1254             $device .= ",model=$model";
1255         }
1256     } elsif ($drive->{interface} eq 'sata'){
1257         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1258         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1259         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1260     } elsif ($drive->{interface} eq 'usb') {
1261         die "implement me";
1262         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1263     } else {
1264         die "unsupported interface type";
1265     }
1266
1267     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1268
1269     return $device;
1270 }
1271
1272 sub get_initiator_name {
1273     my $initiator;
1274
1275     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1276     while (defined(my $line = <$fh>)) {
1277         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1278         $initiator = $1;
1279         last;
1280     }
1281     $fh->close();
1282
1283     return $initiator;
1284 }
1285
1286 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1287 sub print_drive_full {
1288     my ($storecfg, $vmid, $drive) = @_;
1289
1290     my $path;
1291     my $volid = $drive->{file};
1292     my $format;
1293
1294     if (drive_is_cdrom($drive)) {
1295         $path = get_iso_path($storecfg, $vmid, $volid);
1296     } else {
1297         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1298         if ($storeid) {
1299             $path = PVE::Storage::path($storecfg, $volid);
1300             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1301             $format = qemu_img_format($scfg, $volname);
1302         } else {
1303             $path = $volid;
1304             $format = "raw";
1305         }
1306    }
1307
1308     my $opts = '';
1309     foreach my $o (@qemu_drive_options) {
1310         next if $o eq 'bootindex';
1311         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1312     }
1313
1314     $opts .= ",format=$format" if $format && !$drive->{format};
1315
1316     foreach my $o (qw(bps bps_rd bps_wr)) {
1317         my $v = $drive->{"m$o"};
1318         $opts .= ",$o=" . int($v*1024*1024) if $v;
1319     }
1320
1321     my $cache_direct = 0;
1322
1323     if (my $cache = $drive->{cache}) {
1324         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1325     } elsif (!drive_is_cdrom($drive)) {
1326         $opts .= ",cache=none";
1327         $cache_direct = 1;
1328     }
1329
1330     # aio native works only with O_DIRECT
1331     if (!$drive->{aio}) {
1332         if($cache_direct) {
1333             $opts .= ",aio=native";
1334         } else {
1335             $opts .= ",aio=threads";
1336         }
1337     }
1338
1339     if (!drive_is_cdrom($drive)) {
1340         my $detectzeroes;
1341         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1342             $detectzeroes = 'off';
1343         } elsif ($drive->{discard}) {
1344             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1345         } else {
1346             # This used to be our default with discard not being specified:
1347             $detectzeroes = 'on';
1348         }
1349         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1350     }
1351
1352     my $pathinfo = $path ? "file=$path," : '';
1353
1354     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1355 }
1356
1357 sub print_netdevice_full {
1358     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1359
1360     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1361
1362     my $device = $net->{model};
1363     if ($net->{model} eq 'virtio') {
1364          $device = 'virtio-net-pci';
1365      };
1366
1367     my $pciaddr = print_pci_addr("$netid", $bridges);
1368     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1369     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1370         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1371         my $vectors = $net->{queues} * 2 + 2;
1372         $tmpstr .= ",vectors=$vectors,mq=on";
1373     }
1374     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1375
1376     if ($use_old_bios_files) {
1377         my $romfile;
1378         if ($device eq 'virtio-net-pci') {
1379             $romfile = 'pxe-virtio.rom';
1380         } elsif ($device eq 'e1000') {
1381             $romfile = 'pxe-e1000.rom';
1382         } elsif ($device eq 'ne2k') {
1383             $romfile = 'pxe-ne2k_pci.rom';
1384         } elsif ($device eq 'pcnet') {
1385             $romfile = 'pxe-pcnet.rom';
1386         } elsif ($device eq 'rtl8139') {
1387             $romfile = 'pxe-rtl8139.rom';
1388         }
1389         $tmpstr .= ",romfile=$romfile" if $romfile;
1390     }
1391
1392     return $tmpstr;
1393 }
1394
1395 sub print_netdev_full {
1396     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1397
1398     my $i = '';
1399     if ($netid =~ m/^net(\d+)$/) {
1400         $i = int($1);
1401     }
1402
1403     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1404
1405     my $ifname = "tap${vmid}i$i";
1406
1407     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1408     die "interface name '$ifname' is too long (max 15 character)\n"
1409         if length($ifname) >= 16;
1410
1411     my $vhostparam = '';
1412     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1413
1414     my $vmname = $conf->{name} || "vm$vmid";
1415
1416     my $netdev = "";
1417     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1418
1419     if ($net->{bridge}) {
1420         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1421     } else {
1422         $netdev = "type=user,id=$netid,hostname=$vmname";
1423     }
1424
1425     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1426
1427     return $netdev;
1428 }
1429
1430 sub drive_is_cdrom {
1431     my ($drive) = @_;
1432
1433     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1434
1435 }
1436
1437 sub parse_numa {
1438     my ($data) = @_;
1439
1440     my $res = {};
1441
1442     foreach my $kvp (split(/,/, $data)) {
1443
1444         if ($kvp =~ m/^memory=(\S+)$/) {
1445             $res->{memory} = $1;
1446         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1447             $res->{policy} = $1;
1448         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1449             $res->{cpus}->{start} = $1;
1450             $res->{cpus}->{end} = $3;
1451         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1452             $res->{hostnodes}->{start} = $1;
1453             $res->{hostnodes}->{end} = $3;
1454         } else {
1455             return undef;
1456         }
1457     }
1458
1459     return $res;
1460 }
1461
1462 sub parse_hostpci {
1463     my ($value) = @_;
1464
1465     return undef if !$value;
1466
1467
1468     my @list = split(/,/, $value);
1469     my $found;
1470
1471     my $res = {};
1472     foreach my $kv (@list) {
1473
1474         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1475             $found = 1;
1476             if(defined($4)){
1477                 push @{$res->{pciid}}, { id => $2 , function => $4};
1478
1479             }else{
1480                 my $pcidevices = lspci($2);
1481                 $res->{pciid} = $pcidevices->{$2};
1482             }
1483         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1484             $res->{rombar} = $1;
1485         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1486             $res->{'x-vga'} = $1;
1487         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1488             $res->{pcie} = 1 if $1 == 1;
1489         } else {
1490             warn "unknown hostpci setting '$kv'\n";
1491         }
1492     }
1493
1494     return undef if !$found;
1495
1496     return $res;
1497 }
1498
1499 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1500 sub parse_net {
1501     my ($data) = @_;
1502
1503     my $res = {};
1504
1505     foreach my $kvp (split(/,/, $data)) {
1506
1507         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1508             my $model = lc($1);
1509             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1510             $res->{model} = $model;
1511             $res->{macaddr} = $mac;
1512         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1513             $res->{bridge} = $1;
1514         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1515             $res->{queues} = $1;
1516         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1517             $res->{rate} = $1;
1518         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1519             $res->{tag} = $1;
1520         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1521             $res->{trunks} = $1;
1522         } elsif ($kvp =~ m/^firewall=([01])$/) {
1523             $res->{firewall} = $1;
1524         } elsif ($kvp =~ m/^link_down=([01])$/) {
1525             $res->{link_down} = $1;
1526         } else {
1527             return undef;
1528         }
1529
1530     }
1531
1532     return undef if !$res->{model};
1533
1534     return $res;
1535 }
1536
1537 sub print_net {
1538     my $net = shift;
1539
1540     my $res = "$net->{model}";
1541     $res .= "=$net->{macaddr}" if $net->{macaddr};
1542     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1543     $res .= ",rate=$net->{rate}" if $net->{rate};
1544     $res .= ",tag=$net->{tag}" if $net->{tag};
1545     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1546     $res .= ",firewall=1" if $net->{firewall};
1547     $res .= ",link_down=1" if $net->{link_down};
1548     $res .= ",queues=$net->{queues}" if $net->{queues};
1549
1550     return $res;
1551 }
1552
1553 sub add_random_macs {
1554     my ($settings) = @_;
1555
1556     foreach my $opt (keys %$settings) {
1557         next if $opt !~ m/^net(\d+)$/;
1558         my $net = parse_net($settings->{$opt});
1559         next if !$net;
1560         $settings->{$opt} = print_net($net);
1561     }
1562 }
1563
1564 sub add_unused_volume {
1565     my ($config, $volid) = @_;
1566
1567     my $key;
1568     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1569         my $test = "unused$ind";
1570         if (my $vid = $config->{$test}) {
1571             return if $vid eq $volid; # do not add duplicates
1572         } else {
1573             $key = $test;
1574         }
1575     }
1576
1577     die "To many unused volume - please delete them first.\n" if !$key;
1578
1579     $config->{$key} = $volid;
1580
1581     return $key;
1582 }
1583
1584 sub vm_is_volid_owner {
1585     my ($storecfg, $vmid, $volid) = @_;
1586
1587     if ($volid !~  m|^/|) {
1588         my ($path, $owner);
1589         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1590         if ($owner && ($owner == $vmid)) {
1591             return 1;
1592         }
1593     }
1594
1595     return undef;
1596 }
1597
1598 sub split_flagged_list {
1599     my $text = shift || '';
1600     $text =~ s/[,;]/ /g;
1601     $text =~ s/^\s+//;
1602     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1603 }
1604
1605 sub join_flagged_list {
1606     my ($how, $lst) = @_;
1607     join $how, map { $lst->{$_} . $_ } keys %$lst;
1608 }
1609
1610 sub vmconfig_delete_pending_option {
1611     my ($conf, $key, $force) = @_;
1612
1613     delete $conf->{pending}->{$key};
1614     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1615     $pending_delete_hash->{$key} = $force ? '!' : '';
1616     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1617 }
1618
1619 sub vmconfig_undelete_pending_option {
1620     my ($conf, $key) = @_;
1621
1622     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1623     delete $pending_delete_hash->{$key};
1624
1625     if (%$pending_delete_hash) {
1626         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1627     } else {
1628         delete $conf->{pending}->{delete};
1629     }
1630 }
1631
1632 sub vmconfig_register_unused_drive {
1633     my ($storecfg, $vmid, $conf, $drive) = @_;
1634
1635     if (!drive_is_cdrom($drive)) {
1636         my $volid = $drive->{file};
1637         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1638             add_unused_volume($conf, $volid, $vmid);
1639         }
1640     }
1641 }
1642
1643 sub vmconfig_cleanup_pending {
1644     my ($conf) = @_;
1645
1646     # remove pending changes when nothing changed
1647     my $changes;
1648     foreach my $opt (keys %{$conf->{pending}}) {
1649         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1650             $changes = 1;
1651             delete $conf->{pending}->{$opt};
1652         }
1653     }
1654
1655     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1656     my $pending_delete_hash = {};
1657     while (my ($opt, $force) = each %$current_delete_hash) {
1658         if (defined($conf->{$opt})) {
1659             $pending_delete_hash->{$opt} = $force;
1660         } else {
1661             $changes = 1;
1662         }
1663     }
1664
1665     if (%$pending_delete_hash) {
1666         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1667     } else {
1668         delete $conf->{pending}->{delete};
1669     }
1670
1671     return $changes;
1672 }
1673
1674 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1675 my $smbios1_desc = {
1676     uuid => {
1677         type => 'string',
1678         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1679         format_description => 'UUID',
1680         optional => 1,
1681     },
1682     version => {
1683         type => 'string',
1684         pattern => '\S+',
1685         format_description => 'str',
1686         optional => 1,
1687     },
1688     serial => {
1689         type => 'string',
1690         pattern => '\S+',
1691         format_description => 'str',
1692         optional => 1,
1693     },
1694     manufacturer => {
1695         type => 'string',
1696         pattern => '\S+',
1697         format_description => 'name',
1698         optional => 1,
1699     },
1700     product => {
1701         type => 'string',
1702         pattern => '\S+',
1703         format_description => 'name',
1704         optional => 1,
1705     },
1706     sku => {
1707         type => 'string',
1708         pattern => '\S+',
1709         format_description => 'str',
1710         optional => 1,
1711     },
1712     family => {
1713         type => 'string',
1714         pattern => '\S+',
1715         format_description => 'str',
1716         optional => 1,
1717     },
1718 };
1719
1720 sub parse_smbios1 {
1721     my ($data) = @_;
1722
1723     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1724     warn $@ if $@;
1725     return $res;
1726 }
1727
1728 sub print_smbios1 {
1729     my ($smbios1) = @_;
1730     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1731 }
1732
1733 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1734
1735 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1736 sub verify_bootdisk {
1737     my ($value, $noerr) = @_;
1738
1739     return $value if valid_drivename($value);
1740
1741     return undef if $noerr;
1742
1743     die "invalid boot disk '$value'\n";
1744 }
1745
1746 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1747 sub verify_numa {
1748     my ($value, $noerr) = @_;
1749
1750     return $value if parse_numa($value);
1751
1752     return undef if $noerr;
1753
1754     die "unable to parse numa options\n";
1755 }
1756
1757 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1758 sub verify_net {
1759     my ($value, $noerr) = @_;
1760
1761     return $value if parse_net($value);
1762
1763     return undef if $noerr;
1764
1765     die "unable to parse network options\n";
1766 }
1767
1768 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1769 sub verify_hostpci {
1770     my ($value, $noerr) = @_;
1771
1772     return $value if parse_hostpci($value);
1773
1774     return undef if $noerr;
1775
1776     die "unable to parse pci id\n";
1777 }
1778
1779 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1780 sub verify_watchdog {
1781     my ($value, $noerr) = @_;
1782
1783     return $value if parse_watchdog($value);
1784
1785     return undef if $noerr;
1786
1787     die "unable to parse watchdog options\n";
1788 }
1789
1790 sub parse_watchdog {
1791     my ($value) = @_;
1792
1793     return undef if !$value;
1794
1795     my $res = {};
1796
1797     foreach my $p (split(/,/, $value)) {
1798         next if $p =~ m/^\s*$/;
1799
1800         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1801             $res->{model} = $2;
1802         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1803             $res->{action} = $2;
1804         } else {
1805             return undef;
1806         }
1807     }
1808
1809     return $res;
1810 }
1811
1812 sub parse_usb_device {
1813     my ($value) = @_;
1814
1815     return undef if !$value;
1816
1817     my @dl = split(/,/, $value);
1818     my $found;
1819
1820     my $res = {};
1821     foreach my $v (@dl) {
1822         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1823             $found = 1;
1824             $res->{vendorid} = $2;
1825             $res->{productid} = $4;
1826         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1827             $found = 1;
1828             $res->{hostbus} = $1;
1829             $res->{hostport} = $2;
1830         } elsif ($v =~ m/^spice$/) {
1831             $found = 1;
1832             $res->{spice} = 1;
1833         } else {
1834             return undef;
1835         }
1836     }
1837     return undef if !$found;
1838
1839     return $res;
1840 }
1841
1842 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1843 sub verify_usb_device {
1844     my ($value, $noerr) = @_;
1845
1846     return $value if parse_usb_device($value);
1847
1848     return undef if $noerr;
1849
1850     die "unable to parse usb device\n";
1851 }
1852
1853 # add JSON properties for create and set function
1854 sub json_config_properties {
1855     my $prop = shift;
1856
1857     foreach my $opt (keys %$confdesc) {
1858         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1859         $prop->{$opt} = $confdesc->{$opt};
1860     }
1861
1862     return $prop;
1863 }
1864
1865 sub check_type {
1866     my ($key, $value) = @_;
1867
1868     die "unknown setting '$key'\n" if !$confdesc->{$key};
1869
1870     my $type = $confdesc->{$key}->{type};
1871
1872     if (!defined($value)) {
1873         die "got undefined value\n";
1874     }
1875
1876     if ($value =~ m/[\n\r]/) {
1877         die "property contains a line feed\n";
1878     }
1879
1880     if ($type eq 'boolean') {
1881         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1882         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1883         die "type check ('boolean') failed - got '$value'\n";
1884     } elsif ($type eq 'integer') {
1885         return int($1) if $value =~ m/^(\d+)$/;
1886         die "type check ('integer') failed - got '$value'\n";
1887     } elsif ($type eq 'number') {
1888         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1889         die "type check ('number') failed - got '$value'\n";
1890     } elsif ($type eq 'string') {
1891         if (my $fmt = $confdesc->{$key}->{format}) {
1892             if ($fmt eq 'pve-qm-drive') {
1893                 # special case - we need to pass $key to parse_drive()
1894                 my $drive = parse_drive($key, $value);
1895                 return $value if $drive;
1896                 die "unable to parse drive options\n";
1897             }
1898             PVE::JSONSchema::check_format($fmt, $value);
1899             return $value;
1900         }
1901         $value =~ s/^\"(.*)\"$/$1/;
1902         return $value;
1903     } else {
1904         die "internal error"
1905     }
1906 }
1907
1908 sub lock_config_full {
1909     my ($vmid, $timeout, $code, @param) = @_;
1910
1911     my $filename = config_file_lock($vmid);
1912
1913     my $res = lock_file($filename, $timeout, $code, @param);
1914
1915     die $@ if $@;
1916
1917     return $res;
1918 }
1919
1920 sub lock_config_mode {
1921     my ($vmid, $timeout, $shared, $code, @param) = @_;
1922
1923     my $filename = config_file_lock($vmid);
1924
1925     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1926
1927     die $@ if $@;
1928
1929     return $res;
1930 }
1931
1932 sub lock_config {
1933     my ($vmid, $code, @param) = @_;
1934
1935     return lock_config_full($vmid, 10, $code, @param);
1936 }
1937
1938 sub cfs_config_path {
1939     my ($vmid, $node) = @_;
1940
1941     $node = $nodename if !$node;
1942     return "nodes/$node/qemu-server/$vmid.conf";
1943 }
1944
1945 sub check_iommu_support{
1946     #fixme : need to check IOMMU support
1947     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1948
1949     my $iommu=1;
1950     return $iommu;
1951
1952 }
1953
1954 sub config_file {
1955     my ($vmid, $node) = @_;
1956
1957     my $cfspath = cfs_config_path($vmid, $node);
1958     return "/etc/pve/$cfspath";
1959 }
1960
1961 sub config_file_lock {
1962     my ($vmid) = @_;
1963
1964     return "$lock_dir/lock-$vmid.conf";
1965 }
1966
1967 sub touch_config {
1968     my ($vmid) = @_;
1969
1970     my $conf = config_file($vmid);
1971     utime undef, undef, $conf;
1972 }
1973
1974 sub destroy_vm {
1975     my ($storecfg, $vmid, $keep_empty_config) = @_;
1976
1977     my $conffile = config_file($vmid);
1978
1979     my $conf = load_config($vmid);
1980
1981     check_lock($conf);
1982
1983     # only remove disks owned by this VM
1984     foreach_drive($conf, sub {
1985         my ($ds, $drive) = @_;
1986
1987         return if drive_is_cdrom($drive);
1988
1989         my $volid = $drive->{file};
1990
1991         return if !$volid || $volid =~ m|^/|;
1992
1993         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1994         return if !$path || !$owner || ($owner != $vmid);
1995
1996         PVE::Storage::vdisk_free($storecfg, $volid);
1997     });
1998
1999     if ($keep_empty_config) {
2000         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2001     } else {
2002         unlink $conffile;
2003     }
2004
2005     # also remove unused disk
2006     eval {
2007         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2008
2009         eval {
2010             PVE::Storage::foreach_volid($dl, sub {
2011                 my ($volid, $sid, $volname, $d) = @_;
2012                 PVE::Storage::vdisk_free($storecfg, $volid);
2013             });
2014         };
2015         warn $@ if $@;
2016
2017     };
2018     warn $@ if $@;
2019 }
2020
2021 sub load_config {
2022     my ($vmid, $node) = @_;
2023
2024     my $cfspath = cfs_config_path($vmid, $node);
2025
2026     my $conf = PVE::Cluster::cfs_read_file($cfspath);
2027
2028     die "no such VM ('$vmid')\n" if !defined($conf);
2029
2030     return $conf;
2031 }
2032
2033 sub parse_vm_config {
2034     my ($filename, $raw) = @_;
2035
2036     return undef if !defined($raw);
2037
2038     my $res = {
2039         digest => Digest::SHA::sha1_hex($raw),
2040         snapshots => {},
2041         pending => {},
2042     };
2043
2044     $filename =~ m|/qemu-server/(\d+)\.conf$|
2045         || die "got strange filename '$filename'";
2046
2047     my $vmid = $1;
2048
2049     my $conf = $res;
2050     my $descr;
2051     my $section = '';
2052
2053     my @lines = split(/\n/, $raw);
2054     foreach my $line (@lines) {
2055         next if $line =~ m/^\s*$/;
2056
2057         if ($line =~ m/^\[PENDING\]\s*$/i) {
2058             $section = 'pending';
2059             if (defined($descr)) {
2060                 $descr =~ s/\s+$//;
2061                 $conf->{description} = $descr;
2062             }
2063             $descr = undef;
2064             $conf = $res->{$section} = {};
2065             next;
2066
2067         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2068             $section = $1;
2069             if (defined($descr)) {
2070                 $descr =~ s/\s+$//;
2071                 $conf->{description} = $descr;
2072             }
2073             $descr = undef;
2074             $conf = $res->{snapshots}->{$section} = {};
2075             next;
2076         }
2077
2078         if ($line =~ m/^\#(.*)\s*$/) {
2079             $descr = '' if !defined($descr);
2080             $descr .= PVE::Tools::decode_text($1) . "\n";
2081             next;
2082         }
2083
2084         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2085             $descr = '' if !defined($descr);
2086             $descr .= PVE::Tools::decode_text($2);
2087         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2088             $conf->{snapstate} = $1;
2089         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2090             my $key = $1;
2091             my $value = $2;
2092             $conf->{$key} = $value;
2093         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2094             my $value = $1;
2095             if ($section eq 'pending') {
2096                 $conf->{delete} = $value; # we parse this later
2097             } else {
2098                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2099             }
2100         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2101             my $key = $1;
2102             my $value = $2;
2103             eval { $value = check_type($key, $value); };
2104             if ($@) {
2105                 warn "vm $vmid - unable to parse value of '$key' - $@";
2106             } else {
2107                 my $fmt = $confdesc->{$key}->{format};
2108                 if ($fmt && $fmt eq 'pve-qm-drive') {
2109                     my $v = parse_drive($key, $value);
2110                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2111                         $v->{file} = $volid;
2112                         $value = print_drive($vmid, $v);
2113                     } else {
2114                         warn "vm $vmid - unable to parse value of '$key'\n";
2115                         next;
2116                     }
2117                 }
2118
2119                 if ($key eq 'cdrom') {
2120                     $conf->{ide2} = $value;
2121                 } else {
2122                     $conf->{$key} = $value;
2123                 }
2124             }
2125         }
2126     }
2127
2128     if (defined($descr)) {
2129         $descr =~ s/\s+$//;
2130         $conf->{description} = $descr;
2131     }
2132     delete $res->{snapstate}; # just to be sure
2133
2134     return $res;
2135 }
2136
2137 sub write_vm_config {
2138     my ($filename, $conf) = @_;
2139
2140     delete $conf->{snapstate}; # just to be sure
2141
2142     if ($conf->{cdrom}) {
2143         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2144         $conf->{ide2} = $conf->{cdrom};
2145         delete $conf->{cdrom};
2146     }
2147
2148     # we do not use 'smp' any longer
2149     if ($conf->{sockets}) {
2150         delete $conf->{smp};
2151     } elsif ($conf->{smp}) {
2152         $conf->{sockets} = $conf->{smp};
2153         delete $conf->{cores};
2154         delete $conf->{smp};
2155     }
2156
2157     my $used_volids = {};
2158
2159     my $cleanup_config = sub {
2160         my ($cref, $pending, $snapname) = @_;
2161
2162         foreach my $key (keys %$cref) {
2163             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2164                 $key eq 'snapstate' || $key eq 'pending';
2165             my $value = $cref->{$key};
2166             if ($key eq 'delete') {
2167                 die "propertry 'delete' is only allowed in [PENDING]\n"
2168                     if !$pending;
2169                 # fixme: check syntax?
2170                 next;
2171             }
2172             eval { $value = check_type($key, $value); };
2173             die "unable to parse value of '$key' - $@" if $@;
2174
2175             $cref->{$key} = $value;
2176
2177             if (!$snapname && valid_drivename($key)) {
2178                 my $drive = parse_drive($key, $value);
2179                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2180             }
2181         }
2182     };
2183
2184     &$cleanup_config($conf);
2185
2186     &$cleanup_config($conf->{pending}, 1);
2187
2188     foreach my $snapname (keys %{$conf->{snapshots}}) {
2189         die "internal error" if $snapname eq 'pending';
2190         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2191     }
2192
2193     # remove 'unusedX' settings if we re-add a volume
2194     foreach my $key (keys %$conf) {
2195         my $value = $conf->{$key};
2196         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2197             delete $conf->{$key};
2198         }
2199     }
2200
2201     my $generate_raw_config = sub {
2202         my ($conf, $pending) = @_;
2203
2204         my $raw = '';
2205
2206         # add description as comment to top of file
2207         if (defined(my $descr = $conf->{description})) {
2208             if ($descr) {
2209                 foreach my $cl (split(/\n/, $descr)) {
2210                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2211                 }
2212             } else {
2213                 $raw .= "#\n" if $pending;
2214             }
2215         }
2216
2217         foreach my $key (sort keys %$conf) {
2218             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2219             $raw .= "$key: $conf->{$key}\n";
2220         }
2221         return $raw;
2222     };
2223
2224     my $raw = &$generate_raw_config($conf);
2225
2226     if (scalar(keys %{$conf->{pending}})){
2227         $raw .= "\n[PENDING]\n";
2228         $raw .= &$generate_raw_config($conf->{pending}, 1);
2229     }
2230
2231     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2232         $raw .= "\n[$snapname]\n";
2233         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2234     }
2235
2236     return $raw;
2237 }
2238
2239 sub update_config_nolock {
2240     my ($vmid, $conf, $skiplock) = @_;
2241
2242     check_lock($conf) if !$skiplock;
2243
2244     my $cfspath = cfs_config_path($vmid);
2245
2246     PVE::Cluster::cfs_write_file($cfspath, $conf);
2247 }
2248
2249 sub update_config {
2250     my ($vmid, $conf, $skiplock) = @_;
2251
2252     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2253 }
2254
2255 sub load_defaults {
2256
2257     my $res = {};
2258
2259     # we use static defaults from our JSON schema configuration
2260     foreach my $key (keys %$confdesc) {
2261         if (defined(my $default = $confdesc->{$key}->{default})) {
2262             $res->{$key} = $default;
2263         }
2264     }
2265
2266     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2267     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2268
2269     return $res;
2270 }
2271
2272 sub config_list {
2273     my $vmlist = PVE::Cluster::get_vmlist();
2274     my $res = {};
2275     return $res if !$vmlist || !$vmlist->{ids};
2276     my $ids = $vmlist->{ids};
2277
2278     foreach my $vmid (keys %$ids) {
2279         my $d = $ids->{$vmid};
2280         next if !$d->{node} || $d->{node} ne $nodename;
2281         next if !$d->{type} || $d->{type} ne 'qemu';
2282         $res->{$vmid}->{exists} = 1;
2283     }
2284     return $res;
2285 }
2286
2287 # test if VM uses local resources (to prevent migration)
2288 sub check_local_resources {
2289     my ($conf, $noerr) = @_;
2290
2291     my $loc_res = 0;
2292
2293     $loc_res = 1 if $conf->{hostusb}; # old syntax
2294     $loc_res = 1 if $conf->{hostpci}; # old syntax
2295
2296     foreach my $k (keys %$conf) {
2297         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2298         # sockets are safe: they will recreated be on the target side post-migrate
2299         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2300         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2301     }
2302
2303     die "VM uses local resources\n" if $loc_res && !$noerr;
2304
2305     return $loc_res;
2306 }
2307
2308 # check if used storages are available on all nodes (use by migrate)
2309 sub check_storage_availability {
2310     my ($storecfg, $conf, $node) = @_;
2311
2312     foreach_drive($conf, sub {
2313         my ($ds, $drive) = @_;
2314
2315         my $volid = $drive->{file};
2316         return if !$volid;
2317
2318         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2319         return if !$sid;
2320
2321         # check if storage is available on both nodes
2322         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2323         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2324    });
2325 }
2326
2327 # list nodes where all VM images are available (used by has_feature API)
2328 sub shared_nodes {
2329     my ($conf, $storecfg) = @_;
2330
2331     my $nodelist = PVE::Cluster::get_nodelist();
2332     my $nodehash = { map { $_ => 1 } @$nodelist };
2333     my $nodename = PVE::INotify::nodename();
2334
2335     foreach_drive($conf, sub {
2336         my ($ds, $drive) = @_;
2337
2338         my $volid = $drive->{file};
2339         return if !$volid;
2340
2341         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2342         if ($storeid) {
2343             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2344             if ($scfg->{disable}) {
2345                 $nodehash = {};
2346             } elsif (my $avail = $scfg->{nodes}) {
2347                 foreach my $node (keys %$nodehash) {
2348                     delete $nodehash->{$node} if !$avail->{$node};
2349                 }
2350             } elsif (!$scfg->{shared}) {
2351                 foreach my $node (keys %$nodehash) {
2352                     delete $nodehash->{$node} if $node ne $nodename
2353                 }
2354             }
2355         }
2356     });
2357
2358     return $nodehash
2359 }
2360
2361 sub check_lock {
2362     my ($conf) = @_;
2363
2364     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2365 }
2366
2367 sub check_cmdline {
2368     my ($pidfile, $pid) = @_;
2369
2370     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2371     if (defined($fh)) {
2372         my $line = <$fh>;
2373         $fh->close;
2374         return undef if !$line;
2375         my @param = split(/\0/, $line);
2376
2377         my $cmd = $param[0];
2378         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2379
2380         for (my $i = 0; $i < scalar (@param); $i++) {
2381             my $p = $param[$i];
2382             next if !$p;
2383             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2384                 my $p = $param[$i+1];
2385                 return 1 if $p && ($p eq $pidfile);
2386                 return undef;
2387             }
2388         }
2389     }
2390     return undef;
2391 }
2392
2393 sub check_running {
2394     my ($vmid, $nocheck, $node) = @_;
2395
2396     my $filename = config_file($vmid, $node);
2397
2398     die "unable to find configuration file for VM $vmid - no such machine\n"
2399         if !$nocheck && ! -f $filename;
2400
2401     my $pidfile = pidfile_name($vmid);
2402
2403     if (my $fd = IO::File->new("<$pidfile")) {
2404         my $st = stat($fd);
2405         my $line = <$fd>;
2406         close($fd);
2407
2408         my $mtime = $st->mtime;
2409         if ($mtime > time()) {
2410             warn "file '$filename' modified in future\n";
2411         }
2412
2413         if ($line =~ m/^(\d+)$/) {
2414             my $pid = $1;
2415             if (check_cmdline($pidfile, $pid)) {
2416                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2417                     return $pid;
2418                 }
2419             }
2420         }
2421     }
2422
2423     return undef;
2424 }
2425
2426 sub vzlist {
2427
2428     my $vzlist = config_list();
2429
2430     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2431
2432     while (defined(my $de = $fd->read)) {
2433         next if $de !~ m/^(\d+)\.pid$/;
2434         my $vmid = $1;
2435         next if !defined($vzlist->{$vmid});
2436         if (my $pid = check_running($vmid)) {
2437             $vzlist->{$vmid}->{pid} = $pid;
2438         }
2439     }
2440
2441     return $vzlist;
2442 }
2443
2444 sub disksize {
2445     my ($storecfg, $conf) = @_;
2446
2447     my $bootdisk = $conf->{bootdisk};
2448     return undef if !$bootdisk;
2449     return undef if !valid_drivename($bootdisk);
2450
2451     return undef if !$conf->{$bootdisk};
2452
2453     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2454     return undef if !defined($drive);
2455
2456     return undef if drive_is_cdrom($drive);
2457
2458     my $volid = $drive->{file};
2459     return undef if !$volid;
2460
2461     return $drive->{size};
2462 }
2463
2464 my $last_proc_pid_stat;
2465
2466 # get VM status information
2467 # This must be fast and should not block ($full == false)
2468 # We only query KVM using QMP if $full == true (this can be slow)
2469 sub vmstatus {
2470     my ($opt_vmid, $full) = @_;
2471
2472     my $res = {};
2473
2474     my $storecfg = PVE::Storage::config();
2475
2476     my $list = vzlist();
2477     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2478
2479     my $cpucount = $cpuinfo->{cpus} || 1;
2480
2481     foreach my $vmid (keys %$list) {
2482         next if $opt_vmid && ($vmid ne $opt_vmid);
2483
2484         my $cfspath = cfs_config_path($vmid);
2485         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2486
2487         my $d = {};
2488         $d->{pid} = $list->{$vmid}->{pid};
2489
2490         # fixme: better status?
2491         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2492
2493         my $size = disksize($storecfg, $conf);
2494         if (defined($size)) {
2495             $d->{disk} = 0; # no info available
2496             $d->{maxdisk} = $size;
2497         } else {
2498             $d->{disk} = 0;
2499             $d->{maxdisk} = 0;
2500         }
2501
2502         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2503         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2504         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2505
2506         $d->{name} = $conf->{name} || "VM $vmid";
2507         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2508
2509         if ($conf->{balloon}) {
2510             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2511             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2512         }
2513
2514         $d->{uptime} = 0;
2515         $d->{cpu} = 0;
2516         $d->{mem} = 0;
2517
2518         $d->{netout} = 0;
2519         $d->{netin} = 0;
2520
2521         $d->{diskread} = 0;
2522         $d->{diskwrite} = 0;
2523
2524         $d->{template} = is_template($conf);
2525
2526         $res->{$vmid} = $d;
2527     }
2528
2529     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2530     foreach my $dev (keys %$netdev) {
2531         next if $dev !~ m/^tap([1-9]\d*)i/;
2532         my $vmid = $1;
2533         my $d = $res->{$vmid};
2534         next if !$d;
2535
2536         $d->{netout} += $netdev->{$dev}->{receive};
2537         $d->{netin} += $netdev->{$dev}->{transmit};
2538
2539         if ($full) {
2540             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2541             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2542         }
2543
2544     }
2545
2546     my $ctime = gettimeofday;
2547
2548     foreach my $vmid (keys %$list) {
2549
2550         my $d = $res->{$vmid};
2551         my $pid = $d->{pid};
2552         next if !$pid;
2553
2554         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2555         next if !$pstat; # not running
2556
2557         my $used = $pstat->{utime} + $pstat->{stime};
2558
2559         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2560
2561         if ($pstat->{vsize}) {
2562             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2563         }
2564
2565         my $old = $last_proc_pid_stat->{$pid};
2566         if (!$old) {
2567             $last_proc_pid_stat->{$pid} = {
2568                 time => $ctime,
2569                 used => $used,
2570                 cpu => 0,
2571             };
2572             next;
2573         }
2574
2575         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2576
2577         if ($dtime > 1000) {
2578             my $dutime = $used -  $old->{used};
2579
2580             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2581             $last_proc_pid_stat->{$pid} = {
2582                 time => $ctime,
2583                 used => $used,
2584                 cpu => $d->{cpu},
2585             };
2586         } else {
2587             $d->{cpu} = $old->{cpu};
2588         }
2589     }
2590
2591     return $res if !$full;
2592
2593     my $qmpclient = PVE::QMPClient->new();
2594
2595     my $ballooncb = sub {
2596         my ($vmid, $resp) = @_;
2597
2598         my $info = $resp->{'return'};
2599         return if !$info->{max_mem};
2600
2601         my $d = $res->{$vmid};
2602
2603         # use memory assigned to VM
2604         $d->{maxmem} = $info->{max_mem};
2605         $d->{balloon} = $info->{actual};
2606
2607         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2608             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2609             $d->{freemem} = $info->{free_mem};
2610         }
2611
2612         $d->{ballooninfo} = $info;
2613     };
2614
2615     my $blockstatscb = sub {
2616         my ($vmid, $resp) = @_;
2617         my $data = $resp->{'return'} || [];
2618         my $totalrdbytes = 0;
2619         my $totalwrbytes = 0;
2620
2621         for my $blockstat (@$data) {
2622             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2623             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2624
2625             $blockstat->{device} =~ s/drive-//;
2626             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2627         }
2628         $res->{$vmid}->{diskread} = $totalrdbytes;
2629         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2630     };
2631
2632     my $statuscb = sub {
2633         my ($vmid, $resp) = @_;
2634
2635         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2636         # this fails if ballon driver is not loaded, so this must be
2637         # the last commnand (following command are aborted if this fails).
2638         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2639
2640         my $status = 'unknown';
2641         if (!defined($status = $resp->{'return'}->{status})) {
2642             warn "unable to get VM status\n";
2643             return;
2644         }
2645
2646         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2647     };
2648
2649     foreach my $vmid (keys %$list) {
2650         next if $opt_vmid && ($vmid ne $opt_vmid);
2651         next if !$res->{$vmid}->{pid}; # not running
2652         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2653     }
2654
2655     $qmpclient->queue_execute(undef, 1);
2656
2657     foreach my $vmid (keys %$list) {
2658         next if $opt_vmid && ($vmid ne $opt_vmid);
2659         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2660     }
2661
2662     return $res;
2663 }
2664
2665 sub foreach_dimm {
2666     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2667
2668     my $dimm_id = 0;
2669     my $current_size = 1024;
2670     my $dimm_size = 512;
2671     return if $current_size == $memory;
2672
2673     for (my $j = 0; $j < 8; $j++) {
2674         for (my $i = 0; $i < 32; $i++) {
2675             my $name = "dimm${dimm_id}";
2676             $dimm_id++;
2677             my $numanode = $i % $sockets;
2678             $current_size += $dimm_size;
2679             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2680             return  $current_size if $current_size >= $memory;
2681         }
2682         $dimm_size *= 2;
2683     }
2684 }
2685
2686 sub foreach_reverse_dimm {
2687     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2688
2689     my $dimm_id = 253;
2690     my $current_size = 4177920;
2691     my $dimm_size = 65536;
2692     return if $current_size == $memory;
2693
2694     for (my $j = 0; $j < 8; $j++) {
2695         for (my $i = 0; $i < 32; $i++) {
2696             my $name = "dimm${dimm_id}";
2697             $dimm_id--;
2698             my $numanode = $i % $sockets;
2699             $current_size -= $dimm_size;
2700             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2701             return  $current_size if $current_size <= $memory;
2702         }
2703         $dimm_size /= 2;
2704     }
2705 }
2706
2707 sub foreach_drive {
2708     my ($conf, $func) = @_;
2709
2710     foreach my $ds (keys %$conf) {
2711         next if !valid_drivename($ds);
2712
2713         my $drive = parse_drive($ds, $conf->{$ds});
2714         next if !$drive;
2715
2716         &$func($ds, $drive);
2717     }
2718 }
2719
2720 sub foreach_volid {
2721     my ($conf, $func) = @_;
2722
2723     my $volhash = {};
2724
2725     my $test_volid = sub {
2726         my ($volid, $is_cdrom) = @_;
2727
2728         return if !$volid;
2729
2730         $volhash->{$volid} = $is_cdrom || 0;
2731     };
2732
2733     foreach_drive($conf, sub {
2734         my ($ds, $drive) = @_;
2735         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2736     });
2737
2738     foreach my $snapname (keys %{$conf->{snapshots}}) {
2739         my $snap = $conf->{snapshots}->{$snapname};
2740         &$test_volid($snap->{vmstate}, 0);
2741         foreach_drive($snap, sub {
2742             my ($ds, $drive) = @_;
2743             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2744         });
2745     }
2746
2747     foreach my $volid (keys %$volhash) {
2748         &$func($volid, $volhash->{$volid});
2749     }
2750 }
2751
2752 sub vga_conf_has_spice {
2753     my ($vga) = @_;
2754
2755     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2756
2757     return $1 || 1;
2758 }
2759
2760 sub config_to_command {
2761     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2762
2763     my $cmd = [];
2764     my $globalFlags = [];
2765     my $machineFlags = [];
2766     my $rtcFlags = [];
2767     my $cpuFlags = [];
2768     my $devices = [];
2769     my $pciaddr = '';
2770     my $bridges = {};
2771     my $kvmver = kvm_user_version();
2772     my $vernum = 0; # unknown
2773     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2774         $vernum = $1*1000000+$2*1000;
2775     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2776         $vernum = $1*1000000+$2*1000+$3;
2777     }
2778
2779     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2780
2781     my $have_ovz = -f '/proc/vz/vestat';
2782
2783     my $q35 = machine_type_is_q35($conf);
2784     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2785     my $machine_type = $forcemachine || $conf->{machine};
2786     my $use_old_bios_files = undef;
2787     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2788
2789     my $cpuunits = defined($conf->{cpuunits}) ?
2790             $conf->{cpuunits} : $defaults->{cpuunits};
2791
2792     push @$cmd, '/usr/bin/systemd-run';
2793     push @$cmd, '--scope';
2794     push @$cmd, '--slice', "qemu";
2795     push @$cmd, '--unit', $vmid;
2796     # set KillMode=none, so that systemd don't kill those scopes
2797     # at shutdown (pve-manager service should stop the VMs instead)
2798     push @$cmd, '-p', "KillMode=none";
2799     push @$cmd, '-p', "CPUShares=$cpuunits";
2800     if ($conf->{cpulimit}) {
2801         my $cpulimit = int($conf->{cpulimit} * 100);
2802         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2803     }
2804
2805     push @$cmd, '/usr/bin/kvm';
2806
2807     push @$cmd, '-id', $vmid;
2808
2809     my $use_virtio = 0;
2810
2811     my $qmpsocket = qmp_socket($vmid);
2812     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2813     push @$cmd, '-mon', "chardev=qmp,mode=control";
2814
2815     my $socket = vnc_socket($vmid);
2816     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2817
2818     push @$cmd, '-pidfile' , pidfile_name($vmid);
2819
2820     push @$cmd, '-daemonize';
2821
2822     if ($conf->{smbios1}) {
2823         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2824     }
2825
2826     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2827         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2828         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2829         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2830         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2831         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2832         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2833     }
2834
2835     if ($q35) {
2836         # the q35 chipset support native usb2, so we enable usb controller
2837         # by default for this machine type
2838         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2839     } else {
2840         $pciaddr = print_pci_addr("piix3", $bridges);
2841         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2842
2843         my $use_usb2 = 0;
2844         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2845             next if !$conf->{"usb$i"};
2846             $use_usb2 = 1;
2847         }
2848         # include usb device config
2849         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2850     }
2851
2852     my $vga = $conf->{vga};
2853
2854     my $qxlnum = vga_conf_has_spice($vga);
2855     $vga = 'qxl' if $qxlnum;
2856
2857     if (!$vga) {
2858         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2859                                 $conf->{ostype} eq 'win7' ||
2860                                 $conf->{ostype} eq 'w2k8')) {
2861             $vga = 'std';
2862         } else {
2863             $vga = 'cirrus';
2864         }
2865     }
2866
2867     # enable absolute mouse coordinates (needed by vnc)
2868     my $tablet;
2869     if (defined($conf->{tablet})) {
2870         $tablet = $conf->{tablet};
2871     } else {
2872         $tablet = $defaults->{tablet};
2873         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2874         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2875     }
2876
2877     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2878
2879     my $kvm_off = 0;
2880     my $nohyperv;
2881     # host pci devices
2882     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2883         my $d = parse_hostpci($conf->{"hostpci$i"});
2884         next if !$d;
2885
2886         my $pcie = $d->{pcie};
2887         if($pcie){
2888             die "q35 machine model is not enabled" if !$q35;
2889             $pciaddr = print_pcie_addr("hostpci$i");
2890         }else{
2891             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2892         }
2893
2894         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2895         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2896         if ($xvga && $xvga ne '') {
2897             $kvm_off = 1;
2898             $vga = 'none';
2899             $nohyperv = 1;
2900             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2901                 $xvga = "";
2902             }
2903         }
2904         my $pcidevices = $d->{pciid};
2905         my $multifunction = 1 if @$pcidevices > 1;
2906
2907         my $j=0;
2908         foreach my $pcidevice (@$pcidevices) {
2909
2910             my $id = "hostpci$i";
2911             $id .= ".$j" if $multifunction;
2912             my $addr = $pciaddr;
2913             $addr .= ".$j" if $multifunction;
2914             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2915
2916             if($j == 0){
2917                 $devicestr .= "$rombar$xvga";
2918                 $devicestr .= ",multifunction=on" if $multifunction;
2919             }
2920
2921             push @$devices, '-device', $devicestr;
2922             $j++;
2923         }
2924     }
2925
2926     # usb devices
2927     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2928         my $d = parse_usb_device($conf->{"usb$i"});
2929         next if !$d;
2930         if ($d->{vendorid} && $d->{productid}) {
2931             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2932         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2933             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2934         } elsif ($d->{spice}) {
2935             # usb redir support for spice
2936             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2937             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2938         }
2939     }
2940
2941     # serial devices
2942     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2943         if (my $path = $conf->{"serial$i"}) {
2944             if ($path eq 'socket') {
2945                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2946                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2947                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2948             } else {
2949                 die "no such serial device\n" if ! -c $path;
2950                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2951                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2952             }
2953         }
2954     }
2955
2956     # parallel devices
2957     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2958         if (my $path = $conf->{"parallel$i"}) {
2959             die "no such parallel device\n" if ! -c $path;
2960             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2961             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2962             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2963         }
2964     }
2965
2966     my $vmname = $conf->{name} || "vm$vmid";
2967
2968     push @$cmd, '-name', $vmname;
2969
2970     my $sockets = 1;
2971     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2972     $sockets = $conf->{sockets} if  $conf->{sockets};
2973
2974     my $cores = $conf->{cores} || 1;
2975
2976     my $maxcpus = $sockets * $cores;
2977
2978     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2979
2980     my $allowed_vcpus = $cpuinfo->{cpus};
2981
2982     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2983         if ($allowed_vcpus < $maxcpus);
2984
2985     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2986
2987     push @$cmd, '-nodefaults';
2988
2989     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2990
2991     my $bootindex_hash = {};
2992     my $i = 1;
2993     foreach my $o (split(//, $bootorder)) {
2994         $bootindex_hash->{$o} = $i*100;
2995         $i++;
2996     }
2997
2998     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2999
3000     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3001
3002     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3003
3004     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3005
3006     # time drift fix
3007     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3008
3009     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3010     my $useLocaltime = $conf->{localtime};
3011
3012     if (my $ost = $conf->{ostype}) {
3013         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3014
3015         if ($ost =~ m/^w/) { # windows
3016             $useLocaltime = 1 if !defined($conf->{localtime});
3017
3018             # use time drift fix when acpi is enabled
3019             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3020                 $tdf = 1 if !defined($conf->{tdf});
3021             }
3022         }
3023
3024         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
3025             $ost eq 'wvista') {
3026             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3027             push @$cmd, '-no-hpet';
3028             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3029                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
3030                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
3031                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
3032
3033             } else {
3034                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
3035             }
3036         }
3037
3038         if ($ost eq 'win7' || $ost eq 'win8') {
3039             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
3040         }
3041     }
3042
3043     push @$rtcFlags, 'driftfix=slew' if $tdf;
3044
3045     if ($nokvm) {
3046         push @$machineFlags, 'accel=tcg';
3047     } else {
3048         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3049     }
3050
3051     if ($machine_type) {
3052         push @$machineFlags, "type=${machine_type}";
3053     }
3054
3055     if ($conf->{startdate}) {
3056         push @$rtcFlags, "base=$conf->{startdate}";
3057     } elsif ($useLocaltime) {
3058         push @$rtcFlags, 'base=localtime';
3059     }
3060
3061     my $cpu = $nokvm ? "qemu64" : "kvm64";
3062     if (my $cputype = $conf->{cpu}) {
3063         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3064             or die "Cannot parse cpu description: $cputype\n";
3065         $cpu = $cpuconf->{cputype};
3066         $kvm_off = 1 if $cpuconf->{hidden};
3067     }
3068
3069     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3070
3071     push @$cpuFlags , '-x2apic'
3072         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3073
3074     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3075
3076     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3077
3078     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3079
3080         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3081         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3082     }
3083
3084     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3085
3086     push @$cpuFlags, 'kvm=off' if $kvm_off;
3087
3088     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3089
3090     push @$cmd, '-cpu', $cpu;
3091
3092     my $memory = $conf->{memory} || $defaults->{memory};
3093     my $static_memory = 0;
3094     my $dimm_memory = 0;
3095
3096     if ($hotplug_features->{memory}) {
3097         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3098         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3099         $static_memory = $STATICMEM;
3100         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3101         $dimm_memory = $memory - $static_memory;
3102         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3103
3104     } else {
3105
3106         $static_memory = $memory;
3107         push @$cmd, '-m', $static_memory;
3108     }
3109
3110     if ($conf->{numa}) {
3111
3112         my $numa_totalmemory = undef;
3113         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3114             next if !$conf->{"numa$i"};
3115             my $numa = parse_numa($conf->{"numa$i"});
3116             next if !$numa;
3117             # memory
3118             die "missing numa node$i memory value\n" if !$numa->{memory};
3119             my $numa_memory = $numa->{memory};
3120             $numa_totalmemory += $numa_memory;
3121             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3122
3123             # cpus
3124             my $cpus_start = $numa->{cpus}->{start};
3125             die "missing numa node$i cpus\n" if !defined($cpus_start);
3126             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3127             my $cpus = $cpus_start;
3128             if (defined($cpus_end)) {
3129                 $cpus .= "-$cpus_end";
3130                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3131             }
3132
3133             # hostnodes
3134             my $hostnodes_start = $numa->{hostnodes}->{start};
3135             if (defined($hostnodes_start)) {
3136                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3137                 my $hostnodes = $hostnodes_start;
3138                 if (defined($hostnodes_end)) {
3139                     $hostnodes .= "-$hostnodes_end";
3140                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3141                 }
3142
3143                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3144                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3145                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3146                 }
3147
3148                 # policy
3149                 my $policy = $numa->{policy};
3150                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3151                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3152             }
3153
3154             push @$cmd, '-object', $numa_object;
3155             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3156         }
3157
3158         die "total memory for NUMA nodes must be equal to vm static memory\n"
3159             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3160
3161         #if no custom tology, we split memory and cores across numa nodes
3162         if(!$numa_totalmemory) {
3163
3164             my $numa_memory = ($static_memory / $sockets) . "M";
3165
3166             for (my $i = 0; $i < $sockets; $i++)  {
3167
3168                 my $cpustart = ($cores * $i);
3169                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3170                 my $cpus = $cpustart;
3171                 $cpus .= "-$cpuend" if $cpuend;
3172
3173                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3174                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3175             }
3176         }
3177     }
3178
3179     if ($hotplug_features->{memory}) {
3180         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3181             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3182             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3183             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3184
3185             #if dimm_memory is not aligned to dimm map
3186             if($current_size > $memory) {
3187                  $conf->{memory} = $current_size;
3188                  update_config_nolock($vmid, $conf, 1);
3189             }
3190         });
3191     }
3192
3193     push @$cmd, '-S' if $conf->{freeze};
3194
3195     # set keyboard layout
3196     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3197     push @$cmd, '-k', $kb if $kb;
3198
3199     # enable sound
3200     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3201     #push @$cmd, '-soundhw', 'es1370';
3202     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3203
3204     if($conf->{agent}) {
3205         my $qgasocket = qmp_socket($vmid, 1);
3206         my $pciaddr = print_pci_addr("qga0", $bridges);
3207         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3208         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3209         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3210     }
3211
3212     my $spice_port;
3213
3214     if ($qxlnum) {
3215         if ($qxlnum > 1) {
3216             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3217                 for(my $i = 1; $i < $qxlnum; $i++){
3218                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3219                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3220                 }
3221             } else {
3222                 # assume other OS works like Linux
3223                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3224                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3225             }
3226         }
3227
3228         my $pciaddr = print_pci_addr("spice", $bridges);
3229
3230         my $nodename = PVE::INotify::nodename();
3231         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3232         $spice_port = PVE::Tools::next_spice_port($pfamily);
3233
3234         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3235
3236         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3237         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3238         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3239     }
3240
3241     # enable balloon by default, unless explicitly disabled
3242     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3243         $pciaddr = print_pci_addr("balloon0", $bridges);
3244         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3245     }
3246
3247     if ($conf->{watchdog}) {
3248         my $wdopts = parse_watchdog($conf->{watchdog});
3249         $pciaddr = print_pci_addr("watchdog", $bridges);
3250         my $watchdog = $wdopts->{model} || 'i6300esb';
3251         push @$devices, '-device', "$watchdog$pciaddr";
3252         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3253     }
3254
3255     my $vollist = [];
3256     my $scsicontroller = {};
3257     my $ahcicontroller = {};
3258     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3259
3260     # Add iscsi initiator name if available
3261     if (my $initiator = get_initiator_name()) {
3262         push @$devices, '-iscsi', "initiator-name=$initiator";
3263     }
3264
3265     foreach_drive($conf, sub {
3266         my ($ds, $drive) = @_;
3267
3268         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3269             push @$vollist, $drive->{file};
3270         }
3271
3272         $use_virtio = 1 if $ds =~ m/^virtio/;
3273
3274         if (drive_is_cdrom ($drive)) {
3275             if ($bootindex_hash->{d}) {
3276                 $drive->{bootindex} = $bootindex_hash->{d};
3277                 $bootindex_hash->{d} += 1;
3278             }
3279         } else {
3280             if ($bootindex_hash->{c}) {
3281                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3282                 $bootindex_hash->{c} += 1;
3283             }
3284         }
3285
3286         if($drive->{interface} eq 'virtio'){
3287            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3288         }
3289
3290         if ($drive->{interface} eq 'scsi') {
3291
3292             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3293
3294             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3295             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3296
3297             my $iothread = '';
3298             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3299                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3300                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3301             }
3302
3303             my $queues = '';
3304             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3305                 $queues = ",num_queues=$drive->{queues}";
3306             }
3307
3308             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3309             $scsicontroller->{$controller}=1;
3310         }
3311
3312         if ($drive->{interface} eq 'sata') {
3313            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3314            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3315            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3316            $ahcicontroller->{$controller}=1;
3317         }
3318
3319         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3320         push @$devices, '-drive',$drive_cmd;
3321         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3322     });
3323
3324     for (my $i = 0; $i < $MAX_NETS; $i++) {
3325          next if !$conf->{"net$i"};
3326          my $d = parse_net($conf->{"net$i"});
3327          next if !$d;
3328
3329          $use_virtio = 1 if $d->{model} eq 'virtio';
3330
3331          if ($bootindex_hash->{n}) {
3332             $d->{bootindex} = $bootindex_hash->{n};
3333             $bootindex_hash->{n} += 1;
3334          }
3335
3336          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3337          push @$devices, '-netdev', $netdevfull;
3338
3339          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3340          push @$devices, '-device', $netdevicefull;
3341     }
3342
3343     if (!$q35) {
3344         # add pci bridges
3345         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3346            $bridges->{1} = 1;
3347            $bridges->{2} = 1;
3348         }
3349
3350         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3351
3352         while (my ($k, $v) = each %$bridges) {
3353             $pciaddr = print_pci_addr("pci.$k");
3354             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3355         }
3356     }
3357
3358     # add custom args
3359     if ($conf->{args}) {
3360         my $aa = PVE::Tools::split_args($conf->{args});
3361         push @$cmd, @$aa;
3362     }
3363
3364     push @$cmd, @$devices;
3365     push @$cmd, '-rtc', join(',', @$rtcFlags)
3366         if scalar(@$rtcFlags);
3367     push @$cmd, '-machine', join(',', @$machineFlags)
3368         if scalar(@$machineFlags);
3369     push @$cmd, '-global', join(',', @$globalFlags)
3370         if scalar(@$globalFlags);
3371
3372     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3373 }
3374
3375 sub vnc_socket {
3376     my ($vmid) = @_;
3377     return "${var_run_tmpdir}/$vmid.vnc";
3378 }
3379
3380 sub spice_port {
3381     my ($vmid) = @_;
3382
3383     my $res = vm_mon_cmd($vmid, 'query-spice');
3384
3385     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3386 }
3387
3388 sub qmp_socket {
3389     my ($vmid, $qga) = @_;
3390     my $sockettype = $qga ? 'qga' : 'qmp';
3391     return "${var_run_tmpdir}/$vmid.$sockettype";
3392 }
3393
3394 sub pidfile_name {
3395     my ($vmid) = @_;
3396     return "${var_run_tmpdir}/$vmid.pid";
3397 }
3398
3399 sub vm_devices_list {
3400     my ($vmid) = @_;
3401
3402     my $res = vm_mon_cmd($vmid, 'query-pci');
3403     my $devices = {};
3404     foreach my $pcibus (@$res) {
3405         foreach my $device (@{$pcibus->{devices}}) {
3406             next if !$device->{'qdev_id'};
3407             if ($device->{'pci_bridge'}) {
3408                 $devices->{$device->{'qdev_id'}} = 1;
3409                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3410                     next if !$bridge_device->{'qdev_id'};
3411                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3412                     $devices->{$device->{'qdev_id'}}++;
3413                 }
3414             } else {
3415                 $devices->{$device->{'qdev_id'}} = 1;
3416             }
3417         }
3418     }
3419
3420     my $resblock = vm_mon_cmd($vmid, 'query-block');
3421     foreach my $block (@$resblock) {
3422         if($block->{device} =~ m/^drive-(\S+)/){
3423                 $devices->{$1} = 1;
3424         }
3425     }
3426
3427     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3428     foreach my $mice (@$resmice) {
3429         if ($mice->{name} eq 'QEMU HID Tablet') {
3430             $devices->{tablet} = 1;
3431             last;
3432         }
3433     }
3434
3435     return $devices;
3436 }
3437
3438 sub vm_deviceplug {
3439     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3440
3441     my $q35 = machine_type_is_q35($conf);
3442
3443     my $devices_list = vm_devices_list($vmid);
3444     return 1 if defined($devices_list->{$deviceid});
3445
3446     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3447
3448     if ($deviceid eq 'tablet') {
3449
3450         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3451
3452     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3453
3454         qemu_iothread_add($vmid, $deviceid, $device);
3455
3456         qemu_driveadd($storecfg, $vmid, $device);
3457         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3458
3459         qemu_deviceadd($vmid, $devicefull);
3460         eval { qemu_deviceaddverify($vmid, $deviceid); };
3461         if (my $err = $@) {
3462             eval { qemu_drivedel($vmid, $deviceid); };
3463             warn $@ if $@;
3464             die $err;
3465         }
3466
3467     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3468
3469
3470         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3471         my $pciaddr = print_pci_addr($deviceid);
3472         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3473
3474         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3475
3476         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3477             qemu_iothread_add($vmid, $deviceid, $device);
3478             $devicefull .= ",iothread=iothread-$deviceid";
3479         }
3480
3481         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3482             $devicefull .= ",num_queues=$device->{queues}";
3483         }
3484
3485         qemu_deviceadd($vmid, $devicefull);
3486         qemu_deviceaddverify($vmid, $deviceid);
3487
3488     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3489
3490         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3491         qemu_driveadd($storecfg, $vmid, $device);
3492
3493         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3494         eval { qemu_deviceadd($vmid, $devicefull); };
3495         if (my $err = $@) {
3496             eval { qemu_drivedel($vmid, $deviceid); };
3497             warn $@ if $@;
3498             die $err;
3499         }
3500
3501     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3502
3503         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3504
3505         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3506         my $use_old_bios_files = undef;
3507         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3508
3509         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3510         qemu_deviceadd($vmid, $netdevicefull);
3511         eval { qemu_deviceaddverify($vmid, $deviceid); };
3512         if (my $err = $@) {
3513             eval { qemu_netdevdel($vmid, $deviceid); };
3514             warn $@ if $@;
3515             die $err;
3516         }
3517
3518     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3519
3520         my $bridgeid = $2;
3521         my $pciaddr = print_pci_addr($deviceid);
3522         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3523
3524         qemu_deviceadd($vmid, $devicefull);
3525         qemu_deviceaddverify($vmid, $deviceid);
3526
3527     } else {
3528         die "can't hotplug device '$deviceid'\n";
3529     }
3530
3531     return 1;
3532 }
3533
3534 # fixme: this should raise exceptions on error!
3535 sub vm_deviceunplug {
3536     my ($vmid, $conf, $deviceid) = @_;
3537
3538     my $devices_list = vm_devices_list($vmid);
3539     return 1 if !defined($devices_list->{$deviceid});
3540
3541     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3542
3543     if ($deviceid eq 'tablet') {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546
3547     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3548
3549         qemu_devicedel($vmid, $deviceid);
3550         qemu_devicedelverify($vmid, $deviceid);
3551         qemu_drivedel($vmid, $deviceid);
3552         qemu_iothread_del($conf, $vmid, $deviceid);
3553
3554     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3555
3556         qemu_devicedel($vmid, $deviceid);
3557         qemu_devicedelverify($vmid, $deviceid);
3558         qemu_iothread_del($conf, $vmid, $deviceid);
3559
3560     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3561
3562         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3563         my $device = parse_drive($deviceid, $conf->{$deviceid});
3564         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3565
3566         qemu_devicedel($vmid, $deviceid);
3567         qemu_drivedel($vmid, $deviceid);
3568         qemu_deletescsihw($conf, $vmid, $deviceid);
3569
3570     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3571
3572         qemu_devicedel($vmid, $deviceid);
3573         qemu_devicedelverify($vmid, $deviceid);
3574         qemu_netdevdel($vmid, $deviceid);
3575
3576     } else {
3577         die "can't unplug device '$deviceid'\n";
3578     }
3579
3580     return 1;
3581 }
3582
3583 sub qemu_deviceadd {
3584     my ($vmid, $devicefull) = @_;
3585
3586     $devicefull = "driver=".$devicefull;
3587     my %options =  split(/[=,]/, $devicefull);
3588
3589     vm_mon_cmd($vmid, "device_add" , %options);
3590 }
3591
3592 sub qemu_devicedel {
3593     my ($vmid, $deviceid) = @_;
3594
3595     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3596 }
3597
3598 sub qemu_iothread_add {
3599     my($vmid, $deviceid, $device) = @_;
3600
3601     if ($device->{iothread}) {
3602         my $iothreads = vm_iothreads_list($vmid);
3603         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3604     }
3605 }
3606
3607 sub qemu_iothread_del {
3608     my($conf, $vmid, $deviceid) = @_;
3609
3610     my $device = parse_drive($deviceid, $conf->{$deviceid});
3611     if ($device->{iothread}) {
3612         my $iothreads = vm_iothreads_list($vmid);
3613         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3614     }
3615 }
3616
3617 sub qemu_objectadd {
3618     my($vmid, $objectid, $qomtype) = @_;
3619
3620     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3621
3622     return 1;
3623 }
3624
3625 sub qemu_objectdel {
3626     my($vmid, $objectid) = @_;
3627
3628     vm_mon_cmd($vmid, "object-del", id => $objectid);
3629
3630     return 1;
3631 }
3632
3633 sub qemu_driveadd {
3634     my ($storecfg, $vmid, $device) = @_;
3635
3636     my $drive = print_drive_full($storecfg, $vmid, $device);
3637     $drive =~ s/\\/\\\\/g;
3638     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3639
3640     # If the command succeeds qemu prints: "OK"
3641     return 1 if $ret =~ m/OK/s;
3642
3643     die "adding drive failed: $ret\n";
3644 }
3645
3646 sub qemu_drivedel {
3647     my($vmid, $deviceid) = @_;
3648
3649     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3650     $ret =~ s/^\s+//;
3651
3652     return 1 if $ret eq "";
3653
3654     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3655     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3656
3657     die "deleting drive $deviceid failed : $ret\n";
3658 }
3659
3660 sub qemu_deviceaddverify {
3661     my ($vmid, $deviceid) = @_;
3662
3663     for (my $i = 0; $i <= 5; $i++) {
3664          my $devices_list = vm_devices_list($vmid);
3665          return 1 if defined($devices_list->{$deviceid});
3666          sleep 1;
3667     }
3668
3669     die "error on hotplug device '$deviceid'\n";
3670 }
3671
3672
3673 sub qemu_devicedelverify {
3674     my ($vmid, $deviceid) = @_;
3675
3676     # need to verify that the device is correctly removed as device_del
3677     # is async and empty return is not reliable
3678
3679     for (my $i = 0; $i <= 5; $i++) {
3680          my $devices_list = vm_devices_list($vmid);
3681          return 1 if !defined($devices_list->{$deviceid});
3682          sleep 1;
3683     }
3684
3685     die "error on hot-unplugging device '$deviceid'\n";
3686 }
3687
3688 sub qemu_findorcreatescsihw {
3689     my ($storecfg, $conf, $vmid, $device) = @_;
3690
3691     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3692
3693     my $scsihwid="$controller_prefix$controller";
3694     my $devices_list = vm_devices_list($vmid);
3695
3696     if(!defined($devices_list->{$scsihwid})) {
3697         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3698     }
3699
3700     return 1;
3701 }
3702
3703 sub qemu_deletescsihw {
3704     my ($conf, $vmid, $opt) = @_;
3705
3706     my $device = parse_drive($opt, $conf->{$opt});
3707
3708     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3709         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3710         return 1;
3711     }
3712
3713     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3714
3715     my $devices_list = vm_devices_list($vmid);
3716     foreach my $opt (keys %{$devices_list}) {
3717         if (PVE::QemuServer::valid_drivename($opt)) {
3718             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3719             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3720                 return 1;
3721             }
3722         }
3723     }
3724
3725     my $scsihwid="scsihw$controller";
3726
3727     vm_deviceunplug($vmid, $conf, $scsihwid);
3728
3729     return 1;
3730 }
3731
3732 sub qemu_add_pci_bridge {
3733     my ($storecfg, $conf, $vmid, $device) = @_;
3734
3735     my $bridges = {};
3736
3737     my $bridgeid;
3738
3739     print_pci_addr($device, $bridges);
3740
3741     while (my ($k, $v) = each %$bridges) {
3742         $bridgeid = $k;
3743     }
3744     return 1 if !defined($bridgeid) || $bridgeid < 1;
3745
3746     my $bridge = "pci.$bridgeid";
3747     my $devices_list = vm_devices_list($vmid);
3748
3749     if (!defined($devices_list->{$bridge})) {
3750         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3751     }
3752
3753     return 1;
3754 }
3755
3756 sub qemu_set_link_status {
3757     my ($vmid, $device, $up) = @_;
3758
3759     vm_mon_cmd($vmid, "set_link", name => $device,
3760                up => $up ? JSON::true : JSON::false);
3761 }
3762
3763 sub qemu_netdevadd {
3764     my ($vmid, $conf, $device, $deviceid) = @_;
3765
3766     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3767     my %options =  split(/[=,]/, $netdev);
3768
3769     vm_mon_cmd($vmid, "netdev_add",  %options);
3770     return 1;
3771 }
3772
3773 sub qemu_netdevdel {
3774     my ($vmid, $deviceid) = @_;
3775
3776     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3777 }
3778
3779 sub qemu_cpu_hotplug {
3780     my ($vmid, $conf, $vcpus) = @_;
3781
3782     my $sockets = 1;
3783     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3784     $sockets = $conf->{sockets} if  $conf->{sockets};
3785     my $cores = $conf->{cores} || 1;
3786     my $maxcpus = $sockets * $cores;
3787
3788     $vcpus = $maxcpus if !$vcpus;
3789
3790     die "you can't add more vcpus than maxcpus\n"
3791         if $vcpus > $maxcpus;
3792
3793     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3794     die "online cpu unplug is not yet possible\n"
3795         if $vcpus < $currentvcpus;
3796
3797     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3798     die "vcpus in running vm is different than configuration\n"
3799         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3800
3801     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3802         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3803     }
3804 }
3805
3806 sub qemu_memory_hotplug {
3807     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3808
3809     return $value if !check_running($vmid);
3810
3811     my $memory = $conf->{memory} || $defaults->{memory};
3812     $value = $defaults->{memory} if !$value;
3813     return $value if $value == $memory;
3814
3815     my $static_memory = $STATICMEM;
3816     my $dimm_memory = $memory - $static_memory;
3817
3818     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3819     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3820
3821
3822     my $sockets = 1;
3823     $sockets = $conf->{sockets} if $conf->{sockets};
3824
3825     if($value > $memory) {
3826
3827         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3828             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3829
3830                 return if $current_size <= $conf->{memory};
3831
3832                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3833                 if (my $err = $@) {
3834                     eval { qemu_objectdel($vmid, "mem-$name"); };
3835                     die $err;
3836                 }
3837
3838                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3839                 if (my $err = $@) {
3840                     eval { qemu_objectdel($vmid, "mem-$name"); };
3841                     die $err;
3842                 }
3843                 #update conf after each succesful module hotplug
3844                 $conf->{memory} = $current_size;
3845                 update_config_nolock($vmid, $conf, 1);
3846         });
3847
3848     } else {
3849
3850         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3851             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3852
3853                 return if $current_size >= $conf->{memory};
3854                 print "try to unplug memory dimm $name\n";
3855
3856                 my $retry = 0;
3857                 while (1) {
3858                     eval { qemu_devicedel($vmid, $name) };
3859                     sleep 3;
3860                     my $dimm_list = qemu_dimm_list($vmid);
3861                     last if !$dimm_list->{$name};
3862                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3863                     $retry++;
3864                 }
3865
3866                 #update conf after each succesful module unplug
3867                 $conf->{memory} = $current_size;
3868
3869                 eval { qemu_objectdel($vmid, "mem-$name"); };
3870                 update_config_nolock($vmid, $conf, 1);
3871         });
3872     }
3873 }
3874
3875 sub qemu_dimm_list {
3876     my ($vmid) = @_;
3877
3878     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3879     my $dimms = {};
3880
3881     foreach my $dimm (@$dimmarray) {
3882
3883         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3884         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3885         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3886         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3887         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3888     }
3889     return $dimms;
3890 }
3891
3892 sub qemu_block_set_io_throttle {
3893     my ($vmid, $deviceid,
3894         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3895         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3896
3897     return if !check_running($vmid) ;
3898
3899     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3900         bps => int($bps),
3901         bps_rd => int($bps_rd),
3902         bps_wr => int($bps_wr),
3903         iops => int($iops),
3904         iops_rd => int($iops_rd),
3905         iops_wr => int($iops_wr),
3906         bps_max => int($bps_max),
3907         bps_rd_max => int($bps_rd_max),
3908         bps_wr_max => int($bps_wr_max),
3909         iops_max => int($iops_max),
3910         iops_rd_max => int($iops_rd_max),
3911         iops_wr_max => int($iops_wr_max)
3912     );
3913
3914 }
3915
3916 # old code, only used to shutdown old VM after update
3917 sub __read_avail {
3918     my ($fh, $timeout) = @_;
3919
3920     my $sel = new IO::Select;
3921     $sel->add($fh);
3922
3923     my $res = '';
3924     my $buf;
3925
3926     my @ready;
3927     while (scalar (@ready = $sel->can_read($timeout))) {
3928         my $count;
3929         if ($count = $fh->sysread($buf, 8192)) {
3930             if ($buf =~ /^(.*)\(qemu\) $/s) {
3931                 $res .= $1;
3932                 last;
3933             } else {
3934                 $res .= $buf;
3935             }
3936         } else {
3937             if (!defined($count)) {
3938                 die "$!\n";
3939             }
3940             last;
3941         }
3942     }
3943
3944     die "monitor read timeout\n" if !scalar(@ready);
3945
3946     return $res;
3947 }
3948
3949 # old code, only used to shutdown old VM after update
3950 sub vm_monitor_command {
3951     my ($vmid, $cmdstr, $nocheck) = @_;
3952
3953     my $res;
3954
3955     eval {
3956         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3957
3958         my $sname = "${var_run_tmpdir}/$vmid.mon";
3959
3960         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3961             die "unable to connect to VM $vmid socket - $!\n";
3962
3963         my $timeout = 3;
3964
3965         # hack: migrate sometime blocks the monitor (when migrate_downtime
3966         # is set)
3967         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3968             $timeout = 60*60; # 1 hour
3969         }
3970
3971         # read banner;
3972         my $data = __read_avail($sock, $timeout);
3973
3974         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3975             die "got unexpected qemu monitor banner\n";
3976         }
3977
3978         my $sel = new IO::Select;
3979         $sel->add($sock);
3980
3981         if (!scalar(my @ready = $sel->can_write($timeout))) {
3982             die "monitor write error - timeout";
3983         }
3984
3985         my $fullcmd = "$cmdstr\r";
3986
3987         # syslog('info', "VM $vmid monitor command: $cmdstr");
3988
3989         my $b;
3990         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3991             die "monitor write error - $!";
3992         }
3993
3994         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3995
3996         $timeout = 20;
3997
3998         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3999             $timeout = 60*60; # 1 hour
4000         } elsif ($cmdstr =~ m/^(eject|change)/) {
4001             $timeout = 60; # note: cdrom mount command is slow
4002         }
4003         if ($res = __read_avail($sock, $timeout)) {
4004
4005             my @lines = split("\r?\n", $res);
4006
4007             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4008
4009             $res = join("\n", @lines);
4010             $res .= "\n";
4011         }
4012     };
4013
4014     my $err = $@;
4015
4016     if ($err) {
4017         syslog("err", "VM $vmid monitor command failed - $err");
4018         die $err;
4019     }
4020
4021     return $res;
4022 }
4023
4024 sub qemu_block_resize {
4025     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4026
4027     my $running = check_running($vmid);
4028
4029     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4030
4031     return if !$running;
4032
4033     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4034
4035 }
4036
4037 sub qemu_volume_snapshot {
4038     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4039
4040     my $running = check_running($vmid);
4041
4042     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4043         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4044     } else {
4045         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4046     }
4047 }
4048
4049 sub qemu_volume_snapshot_delete {
4050     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4051
4052     my $running = check_running($vmid);
4053
4054     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4055
4056     return if !$running;
4057
4058     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4059 }
4060
4061 sub set_migration_caps {
4062     my ($vmid) = @_;
4063
4064     my $cap_ref = [];
4065
4066     my $enabled_cap = {
4067         "auto-converge" => 1,
4068         "xbzrle" => 1,
4069         "x-rdma-pin-all" => 0,
4070         "zero-blocks" => 0,
4071         "compress" => 0
4072     };
4073
4074     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4075
4076     for my $supported_capability (@$supported_capabilities) {
4077         push @$cap_ref, {
4078             capability => $supported_capability->{capability},
4079             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4080         };
4081     }
4082
4083     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4084 }
4085
4086 my $fast_plug_option = {
4087     'lock' => 1,
4088     'name' => 1,
4089     'onboot' => 1,
4090     'shares' => 1,
4091     'startup' => 1,
4092     'description' => 1,
4093 };
4094
4095 # hotplug changes in [PENDING]
4096 # $selection hash can be used to only apply specified options, for
4097 # example: { cores => 1 } (only apply changed 'cores')
4098 # $errors ref is used to return error messages
4099 sub vmconfig_hotplug_pending {
4100     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4101
4102     my $defaults = load_defaults();
4103
4104     # commit values which do not have any impact on running VM first
4105     # Note: those option cannot raise errors, we we do not care about
4106     # $selection and always apply them.
4107
4108     my $add_error = sub {
4109         my ($opt, $msg) = @_;
4110         $errors->{$opt} = "hotplug problem - $msg";
4111     };
4112
4113     my $changes = 0;
4114     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4115         if ($fast_plug_option->{$opt}) {
4116             $conf->{$opt} = $conf->{pending}->{$opt};
4117             delete $conf->{pending}->{$opt};
4118             $changes = 1;
4119         }
4120     }
4121
4122     if ($changes) {
4123         update_config_nolock($vmid, $conf, 1);
4124         $conf = load_config($vmid); # update/reload
4125     }
4126
4127     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4128
4129     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4130     while (my ($opt, $force) = each %$pending_delete_hash) {
4131         next if $selection && !$selection->{$opt};
4132         eval {
4133             if ($opt eq 'hotplug') {
4134                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4135             } elsif ($opt eq 'tablet') {
4136                 die "skip\n" if !$hotplug_features->{usb};
4137                 if ($defaults->{tablet}) {
4138                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4139                 } else {
4140                     vm_deviceunplug($vmid, $conf, $opt);
4141                 }
4142             } elsif ($opt eq 'vcpus') {
4143                 die "skip\n" if !$hotplug_features->{cpu};
4144                 qemu_cpu_hotplug($vmid, $conf, undef);
4145             } elsif ($opt eq 'balloon') {
4146                 # enable balloon device is not hotpluggable
4147                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4148             } elsif ($fast_plug_option->{$opt}) {
4149                 # do nothing
4150             } elsif ($opt =~ m/^net(\d+)$/) {
4151                 die "skip\n" if !$hotplug_features->{network};
4152                 vm_deviceunplug($vmid, $conf, $opt);
4153             } elsif (valid_drivename($opt)) {
4154                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4155                 vm_deviceunplug($vmid, $conf, $opt);
4156                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4157             } elsif ($opt =~ m/^memory$/) {
4158                 die "skip\n" if !$hotplug_features->{memory};
4159                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4160             } elsif ($opt eq 'cpuunits') {
4161                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4162             } elsif ($opt eq 'cpulimit') {
4163                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4164             } else {
4165                 die "skip\n";
4166             }
4167         };
4168         if (my $err = $@) {
4169             &$add_error($opt, $err) if $err ne "skip\n";
4170         } else {
4171             # save new config if hotplug was successful
4172             delete $conf->{$opt};
4173             vmconfig_undelete_pending_option($conf, $opt);
4174             update_config_nolock($vmid, $conf, 1);
4175             $conf = load_config($vmid); # update/reload
4176         }
4177     }
4178
4179     foreach my $opt (keys %{$conf->{pending}}) {
4180         next if $selection && !$selection->{$opt};
4181         my $value = $conf->{pending}->{$opt};
4182         eval {
4183             if ($opt eq 'hotplug') {
4184                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4185             } elsif ($opt eq 'tablet') {
4186                 die "skip\n" if !$hotplug_features->{usb};
4187                 if ($value == 1) {
4188                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4189                 } elsif ($value == 0) {
4190                     vm_deviceunplug($vmid, $conf, $opt);
4191                 }
4192             } elsif ($opt eq 'vcpus') {
4193                 die "skip\n" if !$hotplug_features->{cpu};
4194                 qemu_cpu_hotplug($vmid, $conf, $value);
4195             } elsif ($opt eq 'balloon') {
4196                 # enable/disable balloning device is not hotpluggable
4197                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4198                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4199                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4200
4201                 # allow manual ballooning if shares is set to zero
4202                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4203                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4204                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4205                 }
4206             } elsif ($opt =~ m/^net(\d+)$/) {
4207                 # some changes can be done without hotplug
4208                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4209                                     $vmid, $opt, $value);
4210             } elsif (valid_drivename($opt)) {
4211                 # some changes can be done without hotplug
4212                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4213                                      $vmid, $opt, $value, 1);
4214             } elsif ($opt =~ m/^memory$/) { #dimms
4215                 die "skip\n" if !$hotplug_features->{memory};
4216                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4217             } elsif ($opt eq 'cpuunits') {
4218                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4219             } elsif ($opt eq 'cpulimit') {
4220                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4221                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4222             } else {
4223                 die "skip\n";  # skip non-hot-pluggable options
4224             }
4225         };
4226         if (my $err = $@) {
4227             &$add_error($opt, $err) if $err ne "skip\n";
4228         } else {
4229             # save new config if hotplug was successful
4230             $conf->{$opt} = $value;
4231             delete $conf->{pending}->{$opt};
4232             update_config_nolock($vmid, $conf, 1);
4233             $conf = load_config($vmid); # update/reload
4234         }
4235     }
4236 }
4237
4238 sub try_deallocate_drive {
4239     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4240
4241     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4242         my $volid = $drive->{file};
4243         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4244             my $sid = PVE::Storage::parse_volume_id($volid);
4245             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4246
4247             # check if the disk is really unused
4248             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4249                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4250             PVE::Storage::vdisk_free($storecfg, $volid);
4251             return 1;
4252         } else {
4253             # If vm is not owner of this disk remove from config
4254             return 1;
4255         }
4256     }
4257
4258     return undef;
4259 }
4260
4261 sub vmconfig_delete_or_detach_drive {
4262     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4263
4264     my $drive = parse_drive($opt, $conf->{$opt});
4265
4266     my $rpcenv = PVE::RPCEnvironment::get();
4267     my $authuser = $rpcenv->get_user();
4268
4269     if ($force) {
4270         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4271         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4272     } else {
4273         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4274     }
4275 }
4276
4277 sub vmconfig_apply_pending {
4278     my ($vmid, $conf, $storecfg) = @_;
4279
4280     # cold plug
4281
4282     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4283     while (my ($opt, $force) = each %$pending_delete_hash) {
4284         die "internal error" if $opt =~ m/^unused/;
4285         $conf = load_config($vmid); # update/reload
4286         if (!defined($conf->{$opt})) {
4287             vmconfig_undelete_pending_option($conf, $opt);
4288             update_config_nolock($vmid, $conf, 1);
4289         } elsif (valid_drivename($opt)) {
4290             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4291             vmconfig_undelete_pending_option($conf, $opt);
4292             delete $conf->{$opt};
4293             update_config_nolock($vmid, $conf, 1);
4294         } else {
4295             vmconfig_undelete_pending_option($conf, $opt);
4296             delete $conf->{$opt};
4297             update_config_nolock($vmid, $conf, 1);
4298         }
4299     }
4300
4301     $conf = load_config($vmid); # update/reload
4302
4303     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4304         $conf = load_config($vmid); # update/reload
4305
4306         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4307             # skip if nothing changed
4308         } elsif (valid_drivename($opt)) {
4309             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4310                 if defined($conf->{$opt});
4311             $conf->{$opt} = $conf->{pending}->{$opt};
4312         } else {
4313             $conf->{$opt} = $conf->{pending}->{$opt};
4314         }
4315
4316         delete $conf->{pending}->{$opt};
4317         update_config_nolock($vmid, $conf, 1);
4318     }
4319 }
4320
4321 my $safe_num_ne = sub {
4322     my ($a, $b) = @_;
4323
4324     return 0 if !defined($a) && !defined($b);
4325     return 1 if !defined($a);
4326     return 1 if !defined($b);
4327
4328     return $a != $b;
4329 };
4330
4331 my $safe_string_ne = sub {
4332     my ($a, $b) = @_;
4333
4334     return 0 if !defined($a) && !defined($b);
4335     return 1 if !defined($a);
4336     return 1 if !defined($b);
4337
4338     return $a ne $b;
4339 };
4340
4341 sub vmconfig_update_net {
4342     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4343
4344     my $newnet = parse_net($value);
4345
4346     if ($conf->{$opt}) {
4347         my $oldnet = parse_net($conf->{$opt});
4348
4349         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4350             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4351             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4352             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4353
4354             # for non online change, we try to hot-unplug
4355             die "skip\n" if !$hotplug;
4356             vm_deviceunplug($vmid, $conf, $opt);
4357         } else {
4358
4359             die "internal error" if $opt !~ m/net(\d+)/;
4360             my $iface = "tap${vmid}i$1";
4361
4362             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4363                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4364             }
4365
4366             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4367                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4368                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4369                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4370                 PVE::Network::tap_unplug($iface);
4371                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks});
4372             }
4373
4374             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4375                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4376             }
4377
4378             return 1;
4379         }
4380     }
4381
4382     if ($hotplug) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4384     } else {
4385         die "skip\n";
4386     }
4387 }
4388
4389 sub vmconfig_update_disk {
4390     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4391
4392     # fixme: do we need force?
4393
4394     my $drive = parse_drive($opt, $value);
4395
4396     if ($conf->{$opt}) {
4397
4398         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4399
4400             my $media = $drive->{media} || 'disk';
4401             my $oldmedia = $old_drive->{media} || 'disk';
4402             die "unable to change media type\n" if $media ne $oldmedia;
4403
4404             if (!drive_is_cdrom($old_drive)) {
4405
4406                 if ($drive->{file} ne $old_drive->{file}) {
4407
4408                     die "skip\n" if !$hotplug;
4409
4410                     # unplug and register as unused
4411                     vm_deviceunplug($vmid, $conf, $opt);
4412                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4413
4414                 } else {
4415                     # update existing disk
4416
4417                     # skip non hotpluggable value
4418                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4419                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4420                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4421                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4422                         die "skip\n";
4423                     }
4424
4425                     # apply throttle
4426                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4427                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4428                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4429                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4430                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4431                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4432                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4433                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4434                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4435                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4436                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4437                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4438
4439                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4440                                                    ($drive->{mbps} || 0)*1024*1024,
4441                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4442                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4443                                                    $drive->{iops} || 0,
4444                                                    $drive->{iops_rd} || 0,
4445                                                    $drive->{iops_wr} || 0,
4446                                                    ($drive->{mbps_max} || 0)*1024*1024,
4447                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4448                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4449                                                    $drive->{iops_max} || 0,
4450                                                    $drive->{iops_rd_max} || 0,
4451                                                    $drive->{iops_wr_max} || 0);
4452
4453                     }
4454
4455                     return 1;
4456                 }
4457
4458             } else { # cdrom
4459
4460                 if ($drive->{file} eq 'none') {
4461                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4462                 } else {
4463                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4464                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4465                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4466                 }
4467
4468                 return 1;
4469             }
4470         }
4471     }
4472
4473     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4474     # hotplug new disks
4475     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4476     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4477 }
4478
4479 sub vm_start {
4480     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4481         $forcemachine, $spice_ticket) = @_;
4482
4483     lock_config($vmid, sub {
4484         my $conf = load_config($vmid, $migratedfrom);
4485
4486         die "you can't start a vm if it's a template\n" if is_template($conf);
4487
4488         check_lock($conf) if !$skiplock;
4489
4490         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4491
4492         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4493             vmconfig_apply_pending($vmid, $conf, $storecfg);
4494             $conf = load_config($vmid); # update/reload
4495         }
4496
4497         my $defaults = load_defaults();
4498
4499         # set environment variable useful inside network script
4500         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4501
4502         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4503
4504         my $migrate_port = 0;
4505         my $migrate_uri;
4506         if ($statefile) {
4507             if ($statefile eq 'tcp') {
4508                 my $localip = "localhost";
4509                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4510                 my $nodename = PVE::INotify::nodename();
4511                 if ($datacenterconf->{migration_unsecure}) {
4512                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4513                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4514                 }
4515                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4516                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4517                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4518                 push @$cmd, '-incoming', $migrate_uri;
4519                 push @$cmd, '-S';
4520             } else {
4521                 push @$cmd, '-loadstate', $statefile;
4522             }
4523         } elsif ($paused) {
4524             push @$cmd, '-S';
4525         }
4526
4527         # host pci devices
4528         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4529           my $d = parse_hostpci($conf->{"hostpci$i"});
4530           next if !$d;
4531           my $pcidevices = $d->{pciid};
4532           foreach my $pcidevice (@$pcidevices) {
4533                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4534
4535                 my $info = pci_device_info("0000:$pciid");
4536                 die "IOMMU not present\n" if !check_iommu_support();
4537                 die "no pci device info for device '$pciid'\n" if !$info;
4538                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4539                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4540           }
4541         }
4542
4543         PVE::Storage::activate_volumes($storecfg, $vollist);
4544
4545         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4546                     umask => 0077); };
4547         my $err = $@;
4548         die "start failed: $err" if $err;
4549
4550         print "migration listens on $migrate_uri\n" if $migrate_uri;
4551
4552         if ($statefile && $statefile ne 'tcp')  {
4553             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4554             warn $@ if $@;
4555         }
4556
4557         if ($migratedfrom) {
4558
4559             eval {
4560                 set_migration_caps($vmid);
4561             };
4562             warn $@ if $@;
4563
4564             if ($spice_port) {
4565                 print "spice listens on port $spice_port\n";
4566                 if ($spice_ticket) {
4567                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4568                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4569                 }
4570             }
4571
4572         } else {
4573
4574             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4575                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4576                     if $conf->{balloon};
4577             }
4578
4579             foreach my $opt (keys %$conf) {
4580                 next if $opt !~  m/^net\d+$/;
4581                 my $nicconf = parse_net($conf->{$opt});
4582                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4583             }
4584         }
4585
4586         vm_mon_cmd_nocheck($vmid, 'qom-set',
4587                     path => "machine/peripheral/balloon0",
4588                     property => "guest-stats-polling-interval",
4589                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4590
4591     });
4592 }
4593
4594 sub vm_mon_cmd {
4595     my ($vmid, $execute, %params) = @_;
4596
4597     my $cmd = { execute => $execute, arguments => \%params };
4598     vm_qmp_command($vmid, $cmd);
4599 }
4600
4601 sub vm_mon_cmd_nocheck {
4602     my ($vmid, $execute, %params) = @_;
4603
4604     my $cmd = { execute => $execute, arguments => \%params };
4605     vm_qmp_command($vmid, $cmd, 1);
4606 }
4607
4608 sub vm_qmp_command {
4609     my ($vmid, $cmd, $nocheck) = @_;
4610
4611     my $res;
4612
4613     my $timeout;
4614     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4615         $timeout = $cmd->{arguments}->{timeout};
4616         delete $cmd->{arguments}->{timeout};
4617     }
4618
4619     eval {
4620         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4621         my $sname = qmp_socket($vmid);
4622         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4623             my $qmpclient = PVE::QMPClient->new();
4624
4625             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4626         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4627             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4628                 if scalar(%{$cmd->{arguments}});
4629             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4630         } else {
4631             die "unable to open monitor socket\n";
4632         }
4633     };
4634     if (my $err = $@) {
4635         syslog("err", "VM $vmid qmp command failed - $err");
4636         die $err;
4637     }
4638
4639     return $res;
4640 }
4641
4642 sub vm_human_monitor_command {
4643     my ($vmid, $cmdline) = @_;
4644
4645     my $res;
4646
4647     my $cmd = {
4648         execute => 'human-monitor-command',
4649         arguments => { 'command-line' => $cmdline},
4650     };
4651
4652     return vm_qmp_command($vmid, $cmd);
4653 }
4654
4655 sub vm_commandline {
4656     my ($storecfg, $vmid) = @_;
4657
4658     my $conf = load_config($vmid);
4659
4660     my $defaults = load_defaults();
4661
4662     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4663
4664     return join(' ', @$cmd);
4665 }
4666
4667 sub vm_reset {
4668     my ($vmid, $skiplock) = @_;
4669
4670     lock_config($vmid, sub {
4671
4672         my $conf = load_config($vmid);
4673
4674         check_lock($conf) if !$skiplock;
4675
4676         vm_mon_cmd($vmid, "system_reset");
4677     });
4678 }
4679
4680 sub get_vm_volumes {
4681     my ($conf) = @_;
4682
4683     my $vollist = [];
4684     foreach_volid($conf, sub {
4685         my ($volid, $is_cdrom) = @_;
4686
4687         return if $volid =~ m|^/|;
4688
4689         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4690         return if !$sid;
4691
4692         push @$vollist, $volid;
4693     });
4694
4695     return $vollist;
4696 }
4697
4698 sub vm_stop_cleanup {
4699     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4700
4701     eval {
4702
4703         if (!$keepActive) {
4704             my $vollist = get_vm_volumes($conf);
4705             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4706         }
4707
4708         foreach my $ext (qw(mon qmp pid vnc qga)) {
4709             unlink "/var/run/qemu-server/${vmid}.$ext";
4710         }
4711
4712         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4713     };
4714     warn $@ if $@; # avoid errors - just warn
4715 }
4716
4717 # Note: use $nockeck to skip tests if VM configuration file exists.
4718 # We need that when migration VMs to other nodes (files already moved)
4719 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4720 sub vm_stop {
4721     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4722
4723     $force = 1 if !defined($force) && !$shutdown;
4724
4725     if ($migratedfrom){
4726         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4727         kill 15, $pid if $pid;
4728         my $conf = load_config($vmid, $migratedfrom);
4729         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4730         return;
4731     }
4732
4733     lock_config($vmid, sub {
4734
4735         my $pid = check_running($vmid, $nocheck);
4736         return if !$pid;
4737
4738         my $conf;
4739         if (!$nocheck) {
4740             $conf = load_config($vmid);
4741             check_lock($conf) if !$skiplock;
4742             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4743                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4744                 $timeout = $opts->{down} if $opts->{down};
4745             }
4746         }
4747
4748         $timeout = 60 if !defined($timeout);
4749
4750         eval {
4751             if ($shutdown) {
4752                 if (defined($conf) && $conf->{agent}) {
4753                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4754                 } else {
4755                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4756                 }
4757             } else {
4758                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4759             }
4760         };
4761         my $err = $@;
4762
4763         if (!$err) {
4764             my $count = 0;
4765             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4766                 $count++;
4767                 sleep 1;
4768             }
4769
4770             if ($count >= $timeout) {
4771                 if ($force) {
4772                     warn "VM still running - terminating now with SIGTERM\n";
4773                     kill 15, $pid;
4774                 } else {
4775                     die "VM quit/powerdown failed - got timeout\n";
4776                 }
4777             } else {
4778                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4779                 return;
4780             }
4781         } else {
4782             if ($force) {
4783                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4784                 kill 15, $pid;
4785             } else {
4786                 die "VM quit/powerdown failed\n";
4787             }
4788         }
4789
4790         # wait again
4791         $timeout = 10;
4792
4793         my $count = 0;
4794         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4795             $count++;
4796             sleep 1;
4797         }
4798
4799         if ($count >= $timeout) {
4800             warn "VM still running - terminating now with SIGKILL\n";
4801             kill 9, $pid;
4802             sleep 1;
4803         }
4804
4805         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4806    });
4807 }
4808
4809 sub vm_suspend {
4810     my ($vmid, $skiplock) = @_;
4811
4812     lock_config($vmid, sub {
4813
4814         my $conf = load_config($vmid);
4815
4816         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4817
4818         vm_mon_cmd($vmid, "stop");
4819     });
4820 }
4821
4822 sub vm_resume {
4823     my ($vmid, $skiplock, $nocheck) = @_;
4824
4825     lock_config($vmid, sub {
4826
4827         if (!$nocheck) {
4828
4829             my $conf = load_config($vmid);
4830
4831             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4832
4833             vm_mon_cmd($vmid, "cont");
4834
4835         } else {
4836             vm_mon_cmd_nocheck($vmid, "cont");
4837         }
4838     });
4839 }
4840
4841 sub vm_sendkey {
4842     my ($vmid, $skiplock, $key) = @_;
4843
4844     lock_config($vmid, sub {
4845
4846         my $conf = load_config($vmid);
4847
4848         # there is no qmp command, so we use the human monitor command
4849         vm_human_monitor_command($vmid, "sendkey $key");
4850     });
4851 }
4852
4853 sub vm_destroy {
4854     my ($storecfg, $vmid, $skiplock) = @_;
4855
4856     lock_config($vmid, sub {
4857
4858         my $conf = load_config($vmid);
4859
4860         check_lock($conf) if !$skiplock;
4861
4862         if (!check_running($vmid)) {
4863             destroy_vm($storecfg, $vmid);
4864         } else {
4865             die "VM $vmid is running - destroy failed\n";
4866         }
4867     });
4868 }
4869
4870 # pci helpers
4871
4872 sub file_write {
4873     my ($filename, $buf) = @_;
4874
4875     my $fh = IO::File->new($filename, "w");
4876     return undef if !$fh;
4877
4878     my $res = print $fh $buf;
4879
4880     $fh->close();
4881
4882     return $res;
4883 }
4884
4885 sub pci_device_info {
4886     my ($name) = @_;
4887
4888     my $res;
4889
4890     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4891     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4892
4893     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4894     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4895
4896     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4897     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4898
4899     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4900     return undef if !defined($product) || $product !~ s/^0x//;
4901
4902     $res = {
4903         name => $name,
4904         vendor => $vendor,
4905         product => $product,
4906         domain => $domain,
4907         bus => $bus,
4908         slot => $slot,
4909         func => $func,
4910         irq => $irq,
4911         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4912     };
4913
4914     return $res;
4915 }
4916
4917 sub pci_dev_reset {
4918     my ($dev) = @_;
4919
4920     my $name = $dev->{name};
4921
4922     my $fn = "$pcisysfs/devices/$name/reset";
4923
4924     return file_write($fn, "1");
4925 }
4926
4927 sub pci_dev_bind_to_vfio {
4928     my ($dev) = @_;
4929
4930     my $name = $dev->{name};
4931
4932     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4933
4934     if (!-d $vfio_basedir) {
4935         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4936     }
4937     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4938
4939     my $testdir = "$vfio_basedir/$name";
4940     return 1 if -d $testdir;
4941
4942     my $data = "$dev->{vendor} $dev->{product}";
4943     return undef if !file_write("$vfio_basedir/new_id", $data);
4944
4945     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4946     if (!file_write($fn, $name)) {
4947         return undef if -f $fn;
4948     }
4949
4950     $fn = "$vfio_basedir/bind";
4951     if (! -d $testdir) {
4952         return undef if !file_write($fn, $name);
4953     }
4954
4955     return -d $testdir;
4956 }
4957
4958 sub pci_dev_group_bind_to_vfio {
4959     my ($pciid) = @_;
4960
4961     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4962
4963     if (!-d $vfio_basedir) {
4964         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4965     }
4966     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4967
4968     # get IOMMU group devices
4969     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4970       my @devs = grep /^0000:/, readdir($D);
4971     closedir($D);
4972
4973     foreach my $pciid (@devs) {
4974         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4975
4976         # pci bridges, switches or root ports are not supported
4977         # they have a pci_bus subdirectory so skip them
4978         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4979
4980         my $info = pci_device_info($1);
4981         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4982     }
4983
4984     return 1;
4985 }
4986
4987 sub print_pci_addr {
4988     my ($id, $bridges) = @_;
4989
4990     my $res = '';
4991     my $devices = {
4992         piix3 => { bus => 0, addr => 1 },
4993         #addr2 : first videocard
4994         balloon0 => { bus => 0, addr => 3 },
4995         watchdog => { bus => 0, addr => 4 },
4996         scsihw0 => { bus => 0, addr => 5 },
4997         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4998         scsihw1 => { bus => 0, addr => 6 },
4999         ahci0 => { bus => 0, addr => 7 },
5000         qga0 => { bus => 0, addr => 8 },
5001         spice => { bus => 0, addr => 9 },
5002         virtio0 => { bus => 0, addr => 10 },
5003         virtio1 => { bus => 0, addr => 11 },
5004         virtio2 => { bus => 0, addr => 12 },
5005         virtio3 => { bus => 0, addr => 13 },
5006         virtio4 => { bus => 0, addr => 14 },
5007         virtio5 => { bus => 0, addr => 15 },
5008         hostpci0 => { bus => 0, addr => 16 },
5009         hostpci1 => { bus => 0, addr => 17 },
5010         net0 => { bus => 0, addr => 18 },
5011         net1 => { bus => 0, addr => 19 },
5012         net2 => { bus => 0, addr => 20 },
5013         net3 => { bus => 0, addr => 21 },
5014         net4 => { bus => 0, addr => 22 },
5015         net5 => { bus => 0, addr => 23 },
5016         vga1 => { bus => 0, addr => 24 },
5017         vga2 => { bus => 0, addr => 25 },
5018         vga3 => { bus => 0, addr => 26 },
5019         hostpci2 => { bus => 0, addr => 27 },
5020         hostpci3 => { bus => 0, addr => 28 },
5021         #addr29 : usb-host (pve-usb.cfg)
5022         'pci.1' => { bus => 0, addr => 30 },
5023         'pci.2' => { bus => 0, addr => 31 },
5024         'net6' => { bus => 1, addr => 1 },
5025         'net7' => { bus => 1, addr => 2 },
5026         'net8' => { bus => 1, addr => 3 },
5027         'net9' => { bus => 1, addr => 4 },
5028         'net10' => { bus => 1, addr => 5 },
5029         'net11' => { bus => 1, addr => 6 },
5030         'net12' => { bus => 1, addr => 7 },
5031         'net13' => { bus => 1, addr => 8 },
5032         'net14' => { bus => 1, addr => 9 },
5033         'net15' => { bus => 1, addr => 10 },
5034         'net16' => { bus => 1, addr => 11 },
5035         'net17' => { bus => 1, addr => 12 },
5036         'net18' => { bus => 1, addr => 13 },
5037         'net19' => { bus => 1, addr => 14 },
5038         'net20' => { bus => 1, addr => 15 },
5039         'net21' => { bus => 1, addr => 16 },
5040         'net22' => { bus => 1, addr => 17 },
5041         'net23' => { bus => 1, addr => 18 },
5042         'net24' => { bus => 1, addr => 19 },
5043         'net25' => { bus => 1, addr => 20 },
5044         'net26' => { bus => 1, addr => 21 },
5045         'net27' => { bus => 1, addr => 22 },
5046         'net28' => { bus => 1, addr => 23 },
5047         'net29' => { bus => 1, addr => 24 },
5048         'net30' => { bus => 1, addr => 25 },
5049         'net31' => { bus => 1, addr => 26 },
5050         'virtio6' => { bus => 2, addr => 1 },
5051         'virtio7' => { bus => 2, addr => 2 },
5052         'virtio8' => { bus => 2, addr => 3 },
5053         'virtio9' => { bus => 2, addr => 4 },
5054         'virtio10' => { bus => 2, addr => 5 },
5055         'virtio11' => { bus => 2, addr => 6 },
5056         'virtio12' => { bus => 2, addr => 7 },
5057         'virtio13' => { bus => 2, addr => 8 },
5058         'virtio14' => { bus => 2, addr => 9 },
5059         'virtio15' => { bus => 2, addr => 10 },
5060         'virtioscsi0' => { bus => 3, addr => 1 },
5061         'virtioscsi1' => { bus => 3, addr => 2 },
5062         'virtioscsi2' => { bus => 3, addr => 3 },
5063         'virtioscsi3' => { bus => 3, addr => 4 },
5064         'virtioscsi4' => { bus => 3, addr => 5 },
5065         'virtioscsi5' => { bus => 3, addr => 6 },
5066         'virtioscsi6' => { bus => 3, addr => 7 },
5067         'virtioscsi7' => { bus => 3, addr => 8 },
5068         'virtioscsi8' => { bus => 3, addr => 9 },
5069         'virtioscsi9' => { bus => 3, addr => 10 },
5070         'virtioscsi10' => { bus => 3, addr => 11 },
5071         'virtioscsi11' => { bus => 3, addr => 12 },
5072         'virtioscsi12' => { bus => 3, addr => 13 },
5073         'virtioscsi13' => { bus => 3, addr => 14 },
5074         'virtioscsi14' => { bus => 3, addr => 15 },
5075         'virtioscsi15' => { bus => 3, addr => 16 },
5076         'virtioscsi16' => { bus => 3, addr => 17 },
5077         'virtioscsi17' => { bus => 3, addr => 18 },
5078         'virtioscsi18' => { bus => 3, addr => 19 },
5079         'virtioscsi19' => { bus => 3, addr => 20 },
5080         'virtioscsi20' => { bus => 3, addr => 21 },
5081         'virtioscsi21' => { bus => 3, addr => 22 },
5082         'virtioscsi22' => { bus => 3, addr => 23 },
5083         'virtioscsi23' => { bus => 3, addr => 24 },
5084         'virtioscsi24' => { bus => 3, addr => 25 },
5085         'virtioscsi25' => { bus => 3, addr => 26 },
5086         'virtioscsi26' => { bus => 3, addr => 27 },
5087         'virtioscsi27' => { bus => 3, addr => 28 },
5088         'virtioscsi28' => { bus => 3, addr => 29 },
5089         'virtioscsi29' => { bus => 3, addr => 30 },
5090         'virtioscsi30' => { bus => 3, addr => 31 },
5091
5092     };
5093
5094     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5095            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5096            my $bus = $devices->{$id}->{bus};
5097            $res = ",bus=pci.$bus,addr=$addr";
5098            $bridges->{$bus} = 1 if $bridges;
5099     }
5100     return $res;
5101
5102 }
5103
5104 sub print_pcie_addr {
5105     my ($id) = @_;
5106
5107     my $res = '';
5108     my $devices = {
5109         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5110         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5111         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5112         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5113     };
5114
5115     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5116            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5117            my $bus = $devices->{$id}->{bus};
5118            $res = ",bus=$bus,addr=$addr";
5119     }
5120     return $res;
5121
5122 }
5123
5124 # vzdump restore implementaion
5125
5126 sub tar_archive_read_firstfile {
5127     my $archive = shift;
5128
5129     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5130
5131     # try to detect archive type first
5132     my $pid = open (TMP, "tar tf '$archive'|") ||
5133         die "unable to open file '$archive'\n";
5134     my $firstfile = <TMP>;
5135     kill 15, $pid;
5136     close TMP;
5137
5138     die "ERROR: archive contaions no data\n" if !$firstfile;
5139     chomp $firstfile;
5140
5141     return $firstfile;
5142 }
5143
5144 sub tar_restore_cleanup {
5145     my ($storecfg, $statfile) = @_;
5146
5147     print STDERR "starting cleanup\n";
5148
5149     if (my $fd = IO::File->new($statfile, "r")) {
5150         while (defined(my $line = <$fd>)) {
5151             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5152                 my $volid = $2;
5153                 eval {
5154                     if ($volid =~ m|^/|) {
5155                         unlink $volid || die 'unlink failed\n';
5156                     } else {
5157                         PVE::Storage::vdisk_free($storecfg, $volid);
5158                     }
5159                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5160                 };
5161                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5162             } else {
5163                 print STDERR "unable to parse line in statfile - $line";
5164             }
5165         }
5166         $fd->close();
5167     }
5168 }
5169
5170 sub restore_archive {
5171     my ($archive, $vmid, $user, $opts) = @_;
5172
5173     my $format = $opts->{format};
5174     my $comp;
5175
5176     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5177         $format = 'tar' if !$format;
5178         $comp = 'gzip';
5179     } elsif ($archive =~ m/\.tar$/) {
5180         $format = 'tar' if !$format;
5181     } elsif ($archive =~ m/.tar.lzo$/) {
5182         $format = 'tar' if !$format;
5183         $comp = 'lzop';
5184     } elsif ($archive =~ m/\.vma$/) {
5185         $format = 'vma' if !$format;
5186     } elsif ($archive =~ m/\.vma\.gz$/) {
5187         $format = 'vma' if !$format;
5188         $comp = 'gzip';
5189     } elsif ($archive =~ m/\.vma\.lzo$/) {
5190         $format = 'vma' if !$format;
5191         $comp = 'lzop';
5192     } else {
5193         $format = 'vma' if !$format; # default
5194     }
5195
5196     # try to detect archive format
5197     if ($format eq 'tar') {
5198         return restore_tar_archive($archive, $vmid, $user, $opts);
5199     } else {
5200         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5201     }
5202 }
5203
5204 sub restore_update_config_line {
5205     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5206
5207     return if $line =~ m/^\#qmdump\#/;
5208     return if $line =~ m/^\#vzdump\#/;
5209     return if $line =~ m/^lock:/;
5210     return if $line =~ m/^unused\d+:/;
5211     return if $line =~ m/^parent:/;
5212     return if $line =~ m/^template:/; # restored VM is never a template
5213
5214     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5215         # try to convert old 1.X settings
5216         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5217         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5218             my ($model, $macaddr) = split(/\=/, $devconfig);
5219             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5220             my $net = {
5221                 model => $model,
5222                 bridge => "vmbr$ind",
5223                 macaddr => $macaddr,
5224             };
5225             my $netstr = print_net($net);
5226
5227             print $outfd "net$cookie->{netcount}: $netstr\n";
5228             $cookie->{netcount}++;
5229         }
5230     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5231         my ($id, $netstr) = ($1, $2);
5232         my $net = parse_net($netstr);
5233         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5234         $netstr = print_net($net);
5235         print $outfd "$id: $netstr\n";
5236     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5237         my $virtdev = $1;
5238         my $value = $3;
5239         if ($line =~ m/backup=no/) {
5240             print $outfd "#$line";
5241         } elsif ($virtdev && $map->{$virtdev}) {
5242             my $di = parse_drive($virtdev, $value);
5243             delete $di->{format}; # format can change on restore
5244             $di->{file} = $map->{$virtdev};
5245             $value = print_drive($vmid, $di);
5246             print $outfd "$virtdev: $value\n";
5247         } else {
5248             print $outfd $line;
5249         }
5250     } else {
5251         print $outfd $line;
5252     }
5253 }
5254
5255 sub scan_volids {
5256     my ($cfg, $vmid) = @_;
5257
5258     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5259
5260     my $volid_hash = {};
5261     foreach my $storeid (keys %$info) {
5262         foreach my $item (@{$info->{$storeid}}) {
5263             next if !($item->{volid} && $item->{size});
5264             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5265             $volid_hash->{$item->{volid}} = $item;
5266         }
5267     }
5268
5269     return $volid_hash;
5270 }
5271
5272 sub is_volume_in_use {
5273     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5274
5275     my $path = PVE::Storage::path($storecfg, $volid);
5276
5277     my $scan_config = sub {
5278         my ($cref, $snapname) = @_;
5279
5280         foreach my $key (keys %$cref) {
5281             my $value = $cref->{$key};
5282             if (valid_drivename($key)) {
5283                 next if $skip_drive && $key eq $skip_drive;
5284                 my $drive = parse_drive($key, $value);
5285                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5286                 return 1 if $volid eq $drive->{file};
5287                 if ($drive->{file} =~ m!^/!) {
5288                     return 1 if $drive->{file} eq $path;
5289                 } else {
5290                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5291                     next if !$storeid;
5292                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5293                     next if !$scfg;
5294                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5295                 }
5296             }
5297         }
5298
5299         return 0;
5300     };
5301
5302     return 1 if &$scan_config($conf);
5303
5304     undef $skip_drive;
5305
5306     foreach my $snapname (keys %{$conf->{snapshots}}) {
5307         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5308     }
5309
5310     return 0;
5311 }
5312
5313 sub update_disksize {
5314     my ($vmid, $conf, $volid_hash) = @_;
5315
5316     my $changes;
5317
5318     my $used = {};
5319
5320     # Note: it is allowed to define multiple storages with same path (alias), so
5321     # we need to check both 'volid' and real 'path' (two different volid can point
5322     # to the same path).
5323
5324     my $usedpath = {};
5325
5326     # update size info
5327     foreach my $opt (keys %$conf) {
5328         if (valid_drivename($opt)) {
5329             my $drive = parse_drive($opt, $conf->{$opt});
5330             my $volid = $drive->{file};
5331             next if !$volid;
5332
5333             $used->{$volid} = 1;
5334             if ($volid_hash->{$volid} &&
5335                 (my $path = $volid_hash->{$volid}->{path})) {
5336                 $usedpath->{$path} = 1;
5337             }
5338
5339             next if drive_is_cdrom($drive);
5340             next if !$volid_hash->{$volid};
5341
5342             $drive->{size} = $volid_hash->{$volid}->{size};
5343             my $new = print_drive($vmid, $drive);
5344             if ($new ne $conf->{$opt}) {
5345                 $changes = 1;
5346                 $conf->{$opt} = $new;
5347             }
5348         }
5349     }
5350
5351     # remove 'unusedX' entry if volume is used
5352     foreach my $opt (keys %$conf) {
5353         next if $opt !~ m/^unused\d+$/;
5354         my $volid = $conf->{$opt};
5355         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5356         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5357             $changes = 1;
5358             delete $conf->{$opt};
5359         }
5360     }
5361
5362     foreach my $volid (sort keys %$volid_hash) {
5363         next if $volid =~ m/vm-$vmid-state-/;
5364         next if $used->{$volid};
5365         my $path = $volid_hash->{$volid}->{path};
5366         next if !$path; # just to be sure
5367         next if $usedpath->{$path};
5368         $changes = 1;
5369         add_unused_volume($conf, $volid);
5370         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5371     }
5372
5373     return $changes;
5374 }
5375
5376 sub rescan {
5377     my ($vmid, $nolock) = @_;
5378
5379     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5380
5381     my $volid_hash = scan_volids($cfg, $vmid);
5382
5383     my $updatefn =  sub {
5384         my ($vmid) = @_;
5385
5386         my $conf = load_config($vmid);
5387
5388         check_lock($conf);
5389
5390         my $vm_volids = {};
5391         foreach my $volid (keys %$volid_hash) {
5392             my $info = $volid_hash->{$volid};
5393             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5394         }
5395
5396         my $changes = update_disksize($vmid, $conf, $vm_volids);
5397
5398         update_config_nolock($vmid, $conf, 1) if $changes;
5399     };
5400
5401     if (defined($vmid)) {
5402         if ($nolock) {
5403             &$updatefn($vmid);
5404         } else {
5405             lock_config($vmid, $updatefn, $vmid);
5406         }
5407     } else {
5408         my $vmlist = config_list();
5409         foreach my $vmid (keys %$vmlist) {
5410             if ($nolock) {
5411                 &$updatefn($vmid);
5412             } else {
5413                 lock_config($vmid, $updatefn, $vmid);
5414             }
5415         }
5416     }
5417 }
5418
5419 sub restore_vma_archive {
5420     my ($archive, $vmid, $user, $opts, $comp) = @_;
5421
5422     my $input = $archive eq '-' ? "<&STDIN" : undef;
5423     my $readfrom = $archive;
5424
5425     my $uncomp = '';
5426     if ($comp) {
5427         $readfrom = '-';
5428         my $qarchive = PVE::Tools::shellquote($archive);
5429         if ($comp eq 'gzip') {
5430             $uncomp = "zcat $qarchive|";
5431         } elsif ($comp eq 'lzop') {
5432             $uncomp = "lzop -d -c $qarchive|";
5433         } else {
5434             die "unknown compression method '$comp'\n";
5435         }
5436
5437     }
5438
5439     my $tmpdir = "/var/tmp/vzdumptmp$$";
5440     rmtree $tmpdir;
5441
5442     # disable interrupts (always do cleanups)
5443     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5444         warn "got interrupt - ignored\n";
5445     };
5446
5447     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5448     POSIX::mkfifo($mapfifo, 0600);
5449     my $fifofh;
5450
5451     my $openfifo = sub {
5452         open($fifofh, '>', $mapfifo) || die $!;
5453     };
5454
5455     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5456
5457     my $oldtimeout;
5458     my $timeout = 5;
5459
5460     my $devinfo = {};
5461
5462     my $rpcenv = PVE::RPCEnvironment::get();
5463
5464     my $conffile = config_file($vmid);
5465     my $tmpfn = "$conffile.$$.tmp";
5466
5467     # Note: $oldconf is undef if VM does not exists
5468     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5469
5470     my $print_devmap = sub {
5471         my $virtdev_hash = {};
5472
5473         my $cfgfn = "$tmpdir/qemu-server.conf";
5474
5475         # we can read the config - that is already extracted
5476         my $fh = IO::File->new($cfgfn, "r") ||
5477             "unable to read qemu-server.conf - $!\n";
5478
5479         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5480         if (-f $fwcfgfn) {
5481             my $pve_firewall_dir = '/etc/pve/firewall';
5482             mkdir $pve_firewall_dir; # make sure the dir exists
5483             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5484         }
5485
5486         while (defined(my $line = <$fh>)) {
5487             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5488                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5489                 die "archive does not contain data for drive '$virtdev'\n"
5490                     if !$devinfo->{$devname};
5491                 if (defined($opts->{storage})) {
5492                     $storeid = $opts->{storage} || 'local';
5493                 } elsif (!$storeid) {
5494                     $storeid = 'local';
5495                 }
5496                 $format = 'raw' if !$format;
5497                 $devinfo->{$devname}->{devname} = $devname;
5498                 $devinfo->{$devname}->{virtdev} = $virtdev;
5499                 $devinfo->{$devname}->{format} = $format;
5500                 $devinfo->{$devname}->{storeid} = $storeid;
5501
5502                 # check permission on storage
5503                 my $pool = $opts->{pool}; # todo: do we need that?
5504                 if ($user ne 'root@pam') {
5505                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5506                 }
5507
5508                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5509             }
5510         }
5511
5512         foreach my $devname (keys %$devinfo) {
5513             die "found no device mapping information for device '$devname'\n"
5514                 if !$devinfo->{$devname}->{virtdev};
5515         }
5516
5517         my $cfg = cfs_read_file('storage.cfg');
5518
5519         # create empty/temp config
5520         if ($oldconf) {
5521             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5522             foreach_drive($oldconf, sub {
5523                 my ($ds, $drive) = @_;
5524
5525                 return if drive_is_cdrom($drive);
5526
5527                 my $volid = $drive->{file};
5528
5529                 return if !$volid || $volid =~ m|^/|;
5530
5531                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5532                 return if !$path || !$owner || ($owner != $vmid);
5533
5534                 # Note: only delete disk we want to restore
5535                 # other volumes will become unused
5536                 if ($virtdev_hash->{$ds}) {
5537                     PVE::Storage::vdisk_free($cfg, $volid);
5538                 }
5539             });
5540         }
5541
5542         my $map = {};
5543         foreach my $virtdev (sort keys %$virtdev_hash) {
5544             my $d = $virtdev_hash->{$virtdev};
5545             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5546             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5547
5548             # test if requested format is supported
5549             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5550             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5551             $d->{format} = $defFormat if !$supported;
5552
5553             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5554                                                   $d->{format}, undef, $alloc_size);
5555             print STDERR "new volume ID is '$volid'\n";
5556             $d->{volid} = $volid;
5557             my $path = PVE::Storage::path($cfg, $volid);
5558
5559             PVE::Storage::activate_volumes($cfg,[$volid]);
5560
5561             my $write_zeros = 1;
5562             # fixme: what other storages types initialize volumes with zero?
5563             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5564                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5565                 $write_zeros = 0;
5566             }
5567
5568             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5569
5570             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5571             $map->{$virtdev} = $volid;
5572         }
5573
5574         $fh->seek(0, 0) || die "seek failed - $!\n";
5575
5576         my $outfd = new IO::File ($tmpfn, "w") ||
5577             die "unable to write config for VM $vmid\n";
5578
5579         my $cookie = { netcount => 0 };
5580         while (defined(my $line = <$fh>)) {
5581             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5582         }
5583
5584         $fh->close();
5585         $outfd->close();
5586     };
5587
5588     eval {
5589         # enable interrupts
5590         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5591             die "interrupted by signal\n";
5592         };
5593         local $SIG{ALRM} = sub { die "got timeout\n"; };
5594
5595         $oldtimeout = alarm($timeout);
5596
5597         my $parser = sub {
5598             my $line = shift;
5599
5600             print "$line\n";
5601
5602             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5603                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5604                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5605             } elsif ($line =~ m/^CTIME: /) {
5606                 # we correctly received the vma config, so we can disable
5607                 # the timeout now for disk allocation (set to 10 minutes, so
5608                 # that we always timeout if something goes wrong)
5609                 alarm(600);
5610                 &$print_devmap();
5611                 print $fifofh "done\n";
5612                 my $tmp = $oldtimeout || 0;
5613                 $oldtimeout = undef;
5614                 alarm($tmp);
5615                 close($fifofh);
5616             }
5617         };
5618
5619         print "restore vma archive: $cmd\n";
5620         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5621     };
5622     my $err = $@;
5623
5624     alarm($oldtimeout) if $oldtimeout;
5625
5626     my $vollist = [];
5627     foreach my $devname (keys %$devinfo) {
5628         my $volid = $devinfo->{$devname}->{volid};
5629         push @$vollist, $volid if $volid;
5630     }
5631
5632     my $cfg = cfs_read_file('storage.cfg');
5633     PVE::Storage::deactivate_volumes($cfg, $vollist);
5634
5635     unlink $mapfifo;
5636
5637     if ($err) {
5638         rmtree $tmpdir;
5639         unlink $tmpfn;
5640
5641         foreach my $devname (keys %$devinfo) {
5642             my $volid = $devinfo->{$devname}->{volid};
5643             next if !$volid;
5644             eval {
5645                 if ($volid =~ m|^/|) {
5646                     unlink $volid || die 'unlink failed\n';
5647                 } else {
5648                     PVE::Storage::vdisk_free($cfg, $volid);
5649                 }
5650                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5651             };
5652             print STDERR "unable to cleanup '$volid' - $@" if $@;
5653         }
5654         die $err;
5655     }
5656
5657     rmtree $tmpdir;
5658
5659     rename($tmpfn, $conffile) ||
5660         die "unable to commit configuration file '$conffile'\n";
5661
5662     PVE::Cluster::cfs_update(); # make sure we read new file
5663
5664     eval { rescan($vmid, 1); };
5665     warn $@ if $@;
5666 }
5667
5668 sub restore_tar_archive {
5669     my ($archive, $vmid, $user, $opts) = @_;
5670
5671     if ($archive ne '-') {
5672         my $firstfile = tar_archive_read_firstfile($archive);
5673         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5674             if $firstfile ne 'qemu-server.conf';
5675     }
5676
5677     my $storecfg = cfs_read_file('storage.cfg');
5678
5679     # destroy existing data - keep empty config
5680     my $vmcfgfn = config_file($vmid);
5681     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5682
5683     my $tocmd = "/usr/lib/qemu-server/qmextract";
5684
5685     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5686     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5687     $tocmd .= ' --prealloc' if $opts->{prealloc};
5688     $tocmd .= ' --info' if $opts->{info};
5689
5690     # tar option "xf" does not autodetect compression when read from STDIN,
5691     # so we pipe to zcat
5692     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5693         PVE::Tools::shellquote("--to-command=$tocmd");
5694
5695     my $tmpdir = "/var/tmp/vzdumptmp$$";
5696     mkpath $tmpdir;
5697
5698     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5699     local $ENV{VZDUMP_VMID} = $vmid;
5700     local $ENV{VZDUMP_USER} = $user;
5701
5702     my $conffile = config_file($vmid);
5703     my $tmpfn = "$conffile.$$.tmp";
5704
5705     # disable interrupts (always do cleanups)
5706     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5707         print STDERR "got interrupt - ignored\n";
5708     };
5709
5710     eval {
5711         # enable interrupts
5712         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5713             die "interrupted by signal\n";
5714         };
5715
5716         if ($archive eq '-') {
5717             print "extracting archive from STDIN\n";
5718             run_command($cmd, input => "<&STDIN");
5719         } else {
5720             print "extracting archive '$archive'\n";
5721             run_command($cmd);
5722         }
5723
5724         return if $opts->{info};
5725
5726         # read new mapping
5727         my $map = {};
5728         my $statfile = "$tmpdir/qmrestore.stat";
5729         if (my $fd = IO::File->new($statfile, "r")) {
5730             while (defined (my $line = <$fd>)) {
5731                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5732                     $map->{$1} = $2 if $1;
5733                 } else {
5734                     print STDERR "unable to parse line in statfile - $line\n";
5735                 }
5736             }
5737             $fd->close();
5738         }
5739
5740         my $confsrc = "$tmpdir/qemu-server.conf";
5741
5742         my $srcfd = new IO::File($confsrc, "r") ||
5743             die "unable to open file '$confsrc'\n";
5744
5745         my $outfd = new IO::File ($tmpfn, "w") ||
5746             die "unable to write config for VM $vmid\n";
5747
5748         my $cookie = { netcount => 0 };
5749         while (defined (my $line = <$srcfd>)) {
5750             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5751         }
5752
5753         $srcfd->close();
5754         $outfd->close();
5755     };
5756     my $err = $@;
5757
5758     if ($err) {
5759
5760         unlink $tmpfn;
5761
5762         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5763
5764         die $err;
5765     }
5766
5767     rmtree $tmpdir;
5768
5769     rename $tmpfn, $conffile ||
5770         die "unable to commit configuration file '$conffile'\n";
5771
5772     PVE::Cluster::cfs_update(); # make sure we read new file
5773
5774     eval { rescan($vmid, 1); };
5775     warn $@ if $@;
5776 };
5777
5778
5779 # Internal snapshots
5780
5781 # NOTE: Snapshot create/delete involves several non-atomic
5782 # action, and can take a long time.
5783 # So we try to avoid locking the file and use 'lock' variable
5784 # inside the config file instead.
5785
5786 my $snapshot_copy_config = sub {
5787     my ($source, $dest) = @_;
5788
5789     foreach my $k (keys %$source) {
5790         next if $k eq 'snapshots';
5791         next if $k eq 'snapstate';
5792         next if $k eq 'snaptime';
5793         next if $k eq 'vmstate';
5794         next if $k eq 'lock';
5795         next if $k eq 'digest';
5796         next if $k eq 'description';
5797         next if $k =~ m/^unused\d+$/;
5798
5799         $dest->{$k} = $source->{$k};
5800     }
5801 };
5802
5803 my $snapshot_apply_config = sub {
5804     my ($conf, $snap) = @_;
5805
5806     # copy snapshot list
5807     my $newconf = {
5808         snapshots => $conf->{snapshots},
5809     };
5810
5811     # keep description and list of unused disks
5812     foreach my $k (keys %$conf) {
5813         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5814         $newconf->{$k} = $conf->{$k};
5815     }
5816
5817     &$snapshot_copy_config($snap, $newconf);
5818
5819     return $newconf;
5820 };
5821
5822 sub foreach_writable_storage {
5823     my ($conf, $func) = @_;
5824
5825     my $sidhash = {};
5826
5827     foreach my $ds (keys %$conf) {
5828         next if !valid_drivename($ds);
5829
5830         my $drive = parse_drive($ds, $conf->{$ds});
5831         next if !$drive;
5832         next if drive_is_cdrom($drive);
5833
5834         my $volid = $drive->{file};
5835
5836         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5837         $sidhash->{$sid} = $sid if $sid;
5838     }
5839
5840     foreach my $sid (sort keys %$sidhash) {
5841         &$func($sid);
5842     }
5843 }
5844
5845 my $alloc_vmstate_volid = sub {
5846     my ($storecfg, $vmid, $conf, $snapname) = @_;
5847
5848     # Note: we try to be smart when selecting a $target storage
5849
5850     my $target;
5851
5852     # search shared storage first
5853     foreach_writable_storage($conf, sub {
5854         my ($sid) = @_;
5855         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5856         return if !$scfg->{shared};
5857
5858         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5859     });
5860
5861     if (!$target) {
5862         # now search local storage
5863         foreach_writable_storage($conf, sub {
5864             my ($sid) = @_;
5865             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5866             return if $scfg->{shared};
5867
5868             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5869         });
5870     }
5871
5872     $target = 'local' if !$target;
5873
5874     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5875     # we abort live save after $conf->{memory}, so we need at max twice that space
5876     my $size = $conf->{memory}*2 + $driver_state_size;
5877
5878     my $name = "vm-$vmid-state-$snapname";
5879     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5880     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5881     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5882
5883     return $volid;
5884 };
5885
5886 my $snapshot_prepare = sub {
5887     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5888
5889     my $snap;
5890
5891     my $updatefn =  sub {
5892
5893         my $conf = load_config($vmid);
5894
5895         die "you can't take a snapshot if it's a template\n"
5896             if is_template($conf);
5897
5898         check_lock($conf);
5899
5900         $conf->{lock} = 'snapshot';
5901
5902         die "snapshot name '$snapname' already used\n"
5903             if defined($conf->{snapshots}->{$snapname});
5904
5905         my $storecfg = PVE::Storage::config();
5906         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5907
5908         $snap = $conf->{snapshots}->{$snapname} = {};
5909
5910         if ($save_vmstate && check_running($vmid)) {
5911             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5912         }
5913
5914         &$snapshot_copy_config($conf, $snap);
5915
5916         $snap->{snapstate} = "prepare";
5917         $snap->{snaptime} = time();
5918         $snap->{description} = $comment if $comment;
5919
5920         # always overwrite machine if we save vmstate. This makes sure we
5921         # can restore it later using correct machine type
5922         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5923
5924         update_config_nolock($vmid, $conf, 1);
5925     };
5926
5927     lock_config($vmid, $updatefn);
5928
5929     return $snap;
5930 };
5931
5932 my $snapshot_commit = sub {
5933     my ($vmid, $snapname) = @_;
5934
5935     my $updatefn = sub {
5936
5937         my $conf = load_config($vmid);
5938
5939         die "missing snapshot lock\n"
5940             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5941
5942         my $has_machine_config = defined($conf->{machine});
5943
5944         my $snap = $conf->{snapshots}->{$snapname};
5945
5946         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5947
5948         die "wrong snapshot state\n"
5949             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5950
5951         delete $snap->{snapstate};
5952         delete $conf->{lock};
5953
5954         my $newconf = &$snapshot_apply_config($conf, $snap);
5955
5956         delete $newconf->{machine} if !$has_machine_config;
5957
5958         $newconf->{parent} = $snapname;
5959
5960         update_config_nolock($vmid, $newconf, 1);
5961     };
5962
5963     lock_config($vmid, $updatefn);
5964 };
5965
5966 sub snapshot_rollback {
5967     my ($vmid, $snapname) = @_;
5968
5969     my $prepare = 1;
5970
5971     my $storecfg = PVE::Storage::config();
5972
5973     my $conf = load_config($vmid);
5974
5975     my $get_snapshot_config = sub {
5976
5977         die "you can't rollback if vm is a template\n" if is_template($conf);
5978
5979         my $res = $conf->{snapshots}->{$snapname};
5980
5981         die "snapshot '$snapname' does not exist\n" if !defined($res);
5982
5983         return $res;
5984     };
5985
5986     my $snap = &$get_snapshot_config();
5987
5988     foreach_drive($snap, sub {
5989         my ($ds, $drive) = @_;
5990
5991         return if drive_is_cdrom($drive);
5992
5993         my $volid = $drive->{file};
5994
5995         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5996     });
5997
5998     my $updatefn = sub {
5999
6000         $conf = load_config($vmid);
6001
6002         $snap = &$get_snapshot_config();
6003
6004         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
6005             if $snap->{snapstate};
6006
6007         if ($prepare) {
6008             check_lock($conf);
6009             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
6010         }
6011
6012         die "unable to rollback vm $vmid: vm is running\n"
6013             if check_running($vmid);
6014
6015         if ($prepare) {
6016             $conf->{lock} = 'rollback';
6017         } else {
6018             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
6019             delete $conf->{lock};
6020         }
6021
6022         my $forcemachine;
6023
6024         if (!$prepare) {
6025             my $has_machine_config = defined($conf->{machine});
6026
6027             # copy snapshot config to current config
6028             $conf = &$snapshot_apply_config($conf, $snap);
6029             $conf->{parent} = $snapname;
6030
6031             # Note: old code did not store 'machine', so we try to be smart
6032             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
6033             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
6034             # we remove the 'machine' configuration if not explicitly specified
6035             # in the original config.
6036             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
6037         }
6038
6039         update_config_nolock($vmid, $conf, 1);
6040
6041         if (!$prepare && $snap->{vmstate}) {
6042             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
6043             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
6044         }
6045     };
6046
6047     lock_config($vmid, $updatefn);
6048
6049     foreach_drive($snap, sub {
6050         my ($ds, $drive) = @_;
6051
6052         return if drive_is_cdrom($drive);
6053
6054         my $volid = $drive->{file};
6055         my $device = "drive-$ds";
6056
6057         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
6058     });
6059
6060     $prepare = 0;
6061     lock_config($vmid, $updatefn);
6062 }
6063
6064 my $savevm_wait = sub {
6065     my ($vmid) = @_;
6066
6067     for(;;) {
6068         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6069         if (!$stat->{status}) {
6070             die "savevm not active\n";
6071         } elsif ($stat->{status} eq 'active') {
6072             sleep(1);
6073             next;
6074         } elsif ($stat->{status} eq 'completed') {
6075             last;
6076         } else {
6077             die "query-savevm returned status '$stat->{status}'\n";
6078         }
6079     }
6080 };
6081
6082 sub do_snapshots_with_qemu {
6083     my ($storecfg, $volid) = @_;
6084
6085     my $storage_name = PVE::Storage::parse_volume_id($volid);
6086
6087     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6088         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6089         return 1;
6090     }
6091
6092     if ($volid =~ m/\.(qcow2|qed)$/){
6093         return 1;
6094     }
6095
6096     return undef;
6097 }
6098
6099 sub snapshot_create {
6100     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
6101
6102     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
6103
6104     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
6105
6106     my $config = load_config($vmid);
6107
6108     my $running = check_running($vmid);
6109
6110     my $freezefs = $running && $config->{agent};
6111     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
6112
6113     my $drivehash = {};
6114
6115     if ($freezefs) {
6116         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6117         warn "guest-fsfreeze-freeze problems - $@" if $@;
6118     }
6119
6120     eval {
6121         # create internal snapshots of all drives
6122
6123         my $storecfg = PVE::Storage::config();
6124
6125         if ($running) {
6126             if ($snap->{vmstate}) {
6127                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6128                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6129                 &$savevm_wait($vmid);
6130             } else {
6131                 vm_mon_cmd($vmid, "savevm-start");
6132             }
6133         };
6134
6135         foreach_drive($snap, sub {
6136             my ($ds, $drive) = @_;
6137
6138             return if drive_is_cdrom($drive);
6139
6140             my $volid = $drive->{file};
6141             my $device = "drive-$ds";
6142
6143             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6144             $drivehash->{$ds} = 1;
6145        });
6146     };
6147     my $err = $@;
6148
6149     if ($running) {
6150         eval { vm_mon_cmd($vmid, "savevm-end")  };
6151         warn $@ if $@;
6152
6153         if ($freezefs) {
6154             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6155             warn "guest-fsfreeze-thaw problems - $@" if $@;
6156         }
6157
6158         # savevm-end is async, we need to wait
6159         for (;;) {
6160             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6161             if (!$stat->{bytes}) {
6162                 last;
6163             } else {
6164                 print "savevm not yet finished\n";
6165                 sleep(1);
6166                 next;
6167             }
6168         }
6169     }
6170
6171     if ($err) {
6172         warn "snapshot create failed: starting cleanup\n";
6173         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6174         warn $@ if $@;
6175         die $err;
6176     }
6177
6178     &$snapshot_commit($vmid, $snapname);
6179 }
6180
6181 # Note: $drivehash is only set when called from snapshot_create.
6182 sub snapshot_delete {
6183     my ($vmid, $snapname, $force, $drivehash) = @_;
6184
6185     my $prepare = 1;
6186
6187     my $snap;
6188     my $unused = [];
6189
6190     my $unlink_parent = sub {
6191         my ($confref, $new_parent) = @_;
6192
6193         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6194             if ($new_parent) {
6195                 $confref->{parent} = $new_parent;
6196             } else {
6197                 delete $confref->{parent};
6198             }
6199         }
6200     };
6201
6202     my $updatefn =  sub {
6203         my ($remove_drive) = @_;
6204
6205         my $conf = load_config($vmid);
6206
6207         if (!$drivehash) {
6208             check_lock($conf);
6209             die "you can't delete a snapshot if vm is a template\n"
6210                 if is_template($conf);
6211         }
6212
6213         $snap = $conf->{snapshots}->{$snapname};
6214
6215         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6216
6217         # remove parent refs
6218         if (!$prepare) {
6219             &$unlink_parent($conf, $snap->{parent});
6220             foreach my $sn (keys %{$conf->{snapshots}}) {
6221                 next if $sn eq $snapname;
6222                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6223             }
6224         }
6225
6226         if ($remove_drive) {
6227             if ($remove_drive eq 'vmstate') {
6228                 delete $snap->{$remove_drive};
6229             } else {
6230                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6231                 my $volid = $drive->{file};
6232                 delete $snap->{$remove_drive};
6233                 add_unused_volume($conf, $volid);
6234             }
6235         }
6236
6237         if ($prepare) {
6238             $snap->{snapstate} = 'delete';
6239         } else {
6240             delete $conf->{snapshots}->{$snapname};
6241             delete $conf->{lock} if $drivehash;
6242             foreach my $volid (@$unused) {
6243                 add_unused_volume($conf, $volid);
6244             }
6245         }
6246
6247         update_config_nolock($vmid, $conf, 1);
6248     };
6249
6250     lock_config($vmid, $updatefn);
6251
6252     # now remove vmstate file
6253
6254     my $storecfg = PVE::Storage::config();
6255
6256     if ($snap->{vmstate}) {
6257         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6258         if (my $err = $@) {
6259             die $err if !$force;
6260             warn $err;
6261         }
6262         # save changes (remove vmstate from snapshot)
6263         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6264     };
6265
6266     # now remove all internal snapshots
6267     foreach_drive($snap, sub {
6268         my ($ds, $drive) = @_;
6269
6270         return if drive_is_cdrom($drive);
6271
6272         my $volid = $drive->{file};
6273         my $device = "drive-$ds";
6274
6275         if (!$drivehash || $drivehash->{$ds}) {
6276             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6277             if (my $err = $@) {
6278                 die $err if !$force;
6279                 warn $err;
6280             }
6281         }
6282
6283         # save changes (remove drive fron snapshot)
6284         lock_config($vmid, $updatefn, $ds) if !$force;
6285         push @$unused, $volid;
6286     });
6287
6288     # now cleanup config
6289     $prepare = 0;
6290     lock_config($vmid, $updatefn);
6291 }
6292
6293 sub has_feature {
6294     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6295
6296     my $err;
6297     foreach_drive($conf, sub {
6298         my ($ds, $drive) = @_;
6299
6300         return if drive_is_cdrom($drive);
6301         my $volid = $drive->{file};
6302         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6303     });
6304
6305     return $err ? 0 : 1;
6306 }
6307
6308 sub template_create {
6309     my ($vmid, $conf, $disk) = @_;
6310
6311     my $storecfg = PVE::Storage::config();
6312
6313     foreach_drive($conf, sub {
6314         my ($ds, $drive) = @_;
6315
6316         return if drive_is_cdrom($drive);
6317         return if $disk && $ds ne $disk;
6318
6319         my $volid = $drive->{file};
6320         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6321
6322         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6323         $drive->{file} = $voliddst;
6324         $conf->{$ds} = print_drive($vmid, $drive);
6325         update_config_nolock($vmid, $conf, 1);
6326     });
6327 }
6328
6329 sub is_template {
6330     my ($conf) = @_;
6331
6332     return 1 if defined $conf->{template} && $conf->{template} == 1;
6333 }
6334
6335 sub qemu_img_convert {
6336     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6337
6338     my $storecfg = PVE::Storage::config();
6339     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6340     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6341
6342     if ($src_storeid && $dst_storeid) {
6343
6344         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6345
6346         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6347         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6348
6349         my $src_format = qemu_img_format($src_scfg, $src_volname);
6350         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6351
6352         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6353         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6354
6355         my $cmd = [];
6356         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6357         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6358         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6359
6360         my $parser = sub {
6361             my $line = shift;
6362             if($line =~ m/\((\S+)\/100\%\)/){
6363                 my $percent = $1;
6364                 my $transferred = int($size * $percent / 100);
6365                 my $remaining = $size - $transferred;
6366
6367                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6368             }
6369
6370         };
6371
6372         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6373         my $err = $@;
6374         die "copy failed: $err" if $err;
6375     }
6376 }
6377
6378 sub qemu_img_format {
6379     my ($scfg, $volname) = @_;
6380
6381     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6382         return $1;
6383     } else {
6384         return "raw";
6385     }
6386 }
6387
6388 sub qemu_drive_mirror {
6389     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6390
6391     my $storecfg = PVE::Storage::config();
6392     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6393
6394     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6395
6396     my $format = qemu_img_format($dst_scfg, $dst_volname);
6397
6398     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6399
6400     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6401     $opts->{format} = $format if $format;
6402
6403     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6404
6405     eval {
6406     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6407         while (1) {
6408             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6409             my $stat = @$stats[0];
6410             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6411             die "error job is not mirroring" if $stat->{type} ne "mirror";
6412
6413             my $busy = $stat->{busy};
6414             my $ready = $stat->{ready};
6415
6416             if (my $total = $stat->{len}) {
6417                 my $transferred = $stat->{offset} || 0;
6418                 my $remaining = $total - $transferred;
6419                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6420
6421                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6422             }
6423
6424
6425             if ($stat->{ready} eq 'true') {
6426
6427                 last if $vmiddst != $vmid;
6428
6429                 # try to switch the disk if source and destination are on the same guest
6430                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6431                 last if !$@;
6432                 die $@ if $@ !~ m/cannot be completed/;
6433             }
6434             sleep 1;
6435         }
6436
6437
6438     };
6439     my $err = $@;
6440
6441     my $cancel_job = sub {
6442         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6443         while (1) {
6444             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6445             my $stat = @$stats[0];
6446             last if !$stat;
6447             sleep 1;
6448         }
6449     };
6450
6451     if ($err) {
6452         eval { &$cancel_job(); };
6453         die "mirroring error: $err";
6454     }
6455
6456     if ($vmiddst != $vmid) {
6457         # if we clone a disk for a new target vm, we don't switch the disk
6458         &$cancel_job(); # so we call block-job-cancel
6459     }
6460 }
6461
6462 sub clone_disk {
6463     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6464         $newvmid, $storage, $format, $full, $newvollist) = @_;
6465
6466     my $newvolid;
6467
6468     if (!$full) {
6469         print "create linked clone of drive $drivename ($drive->{file})\n";
6470         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6471         push @$newvollist, $newvolid;
6472     } else {
6473         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6474         $storeid = $storage if $storage;
6475
6476         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6477         if (!$format) {
6478             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6479             $format = qemu_img_format($scfg, $volname);
6480         }
6481
6482         # test if requested format is supported - else use default
6483         my $supported = grep { $_ eq $format } @$validFormats;
6484         $format = $defFormat if !$supported;
6485
6486         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6487
6488         print "create full clone of drive $drivename ($drive->{file})\n";
6489         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6490         push @$newvollist, $newvolid;
6491
6492         PVE::Storage::activate_volumes($storecfg, $newvollist);
6493
6494         if (!$running || $snapname) {
6495             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6496         } else {
6497             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6498         }
6499     }
6500
6501     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6502
6503     my $disk = $drive;
6504     $disk->{format} = undef;
6505     $disk->{file} = $newvolid;
6506     $disk->{size} = $size;
6507
6508     return $disk;
6509 }
6510
6511 # this only works if VM is running
6512 sub get_current_qemu_machine {
6513     my ($vmid) = @_;
6514
6515     my $cmd = { execute => 'query-machines', arguments => {} };
6516     my $res = vm_qmp_command($vmid, $cmd);
6517
6518     my ($current, $default);
6519     foreach my $e (@$res) {
6520         $default = $e->{name} if $e->{'is-default'};
6521         $current = $e->{name} if $e->{'is-current'};
6522     }
6523
6524     # fallback to the default machine if current is not supported by qemu
6525     return $current || $default || 'pc';
6526 }
6527
6528 sub qemu_machine_feature_enabled {
6529     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6530
6531     my $current_major;
6532     my $current_minor;
6533
6534     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6535
6536         $current_major = $3;
6537         $current_minor = $4;
6538
6539     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6540
6541         $current_major = $1;
6542         $current_minor = $2;
6543     }
6544
6545     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6546
6547
6548 }
6549
6550 sub qemu_machine_pxe {
6551     my ($vmid, $conf, $machine) = @_;
6552
6553     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6554
6555     foreach my $opt (keys %$conf) {
6556         next if $opt !~ m/^net(\d+)$/;
6557         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6558         next if !$net;
6559         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6560         return $machine.".pxe" if $romfile =~ m/pxe/;
6561         last;
6562     }
6563
6564 }
6565
6566 sub qemu_use_old_bios_files {
6567     my ($machine_type) = @_;
6568
6569     return if !$machine_type;
6570
6571     my $use_old_bios_files = undef;
6572
6573     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6574         $machine_type = $1;
6575         $use_old_bios_files = 1;
6576     } else {
6577         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6578         # load new efi bios files on migration. So this hack is required to allow
6579         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6580         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6581         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6582     }
6583
6584     return ($use_old_bios_files, $machine_type);
6585 }
6586
6587 sub lspci {
6588
6589     my $devices = {};
6590
6591     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6592             my (undef, $id, $function) = @_;
6593             my $res = { id => $id, function => $function};
6594             push @{$devices->{$id}}, $res;
6595     });
6596
6597     return $devices;
6598 }
6599
6600 sub vm_iothreads_list {
6601     my ($vmid) = @_;
6602
6603     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6604
6605     my $iothreads = {};
6606     foreach my $iothread (@$res) {
6607         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6608     }
6609
6610     return $iothreads;
6611 }
6612
6613 sub scsihw_infos {
6614     my ($conf, $drive) = @_;
6615
6616     my $maxdev = 0;
6617
6618     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6619         $maxdev = 7;
6620     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6621         $maxdev = 1;
6622     } else {
6623         $maxdev = 256;
6624     }
6625
6626     my $controller = int($drive->{index} / $maxdev);
6627     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6628
6629     return ($maxdev, $controller, $controller_prefix);
6630 }
6631
6632 # bash completion helper
6633
6634 sub complete_backup_archives {
6635     my ($cmdname, $pname, $cvalue) = @_;
6636
6637     my $cfg = PVE::Storage::config();
6638
6639     my $storeid;
6640
6641     if ($cvalue =~ m/^([^:]+):/) {
6642         $storeid = $1;
6643     }
6644
6645     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6646
6647     my $res = [];
6648     foreach my $id (keys %$data) {
6649         foreach my $item (@{$data->{$id}}) {
6650             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6651             push @$res, $item->{volid} if defined($item->{volid});
6652         }
6653     }
6654
6655     return $res;
6656 }
6657
6658 my $complete_vmid_full = sub {
6659     my ($running) = @_;
6660
6661     my $idlist = vmstatus();
6662
6663     my $res = [];
6664
6665     foreach my $id (keys %$idlist) {
6666         my $d = $idlist->{$id};
6667         if (defined($running)) {
6668             next if $d->{template};
6669             next if $running && $d->{status} ne 'running';
6670             next if !$running && $d->{status} eq 'running';
6671         }
6672         push @$res, $id;
6673
6674     }
6675     return $res;
6676 };
6677
6678 sub complete_vmid {
6679     return &$complete_vmid_full();
6680 }
6681
6682 sub complete_vmid_stopped {
6683     return &$complete_vmid_full(0);
6684 }
6685
6686 sub complete_vmid_running {
6687     return &$complete_vmid_full(1);
6688 }
6689
6690 sub complete_storage {
6691
6692     my $cfg = PVE::Storage::config();
6693     my $ids = $cfg->{ids};
6694
6695     my $res = [];
6696     foreach my $sid (keys %$ids) {
6697         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6698         next if !$ids->{$sid}->{content}->{images};
6699         push @$res, $sid;
6700     }
6701
6702     return $res;
6703 }
6704
6705 1;