PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393     bios => {
 394         optional => 1,
 395         type => 'string',
 396         enum => [ qw(seabios ovmf) ],
 397         description => "Select BIOS implementation.",
 398         default => 'seabios',
 399     },
 400 };
 401
 402 # what about other qemu settings ?
 403 #cpu => 'string',
 404 #machine => 'string',
 405 #fda => 'file',
 406 #fdb => 'file',
 407 #mtdblock => 'file',
 408 #sd => 'file',
 409 #pflash => 'file',
 410 #snapshot => 'bool',
 411 #bootp => 'file',
 412 ##tftp => 'dir',
 413 ##smb => 'dir',
 414 #kernel => 'file',
 415 #append => 'string',
 416 #initrd => 'file',
 417 ##soundhw => 'string',
 418
 419 while (my ($k, $v) = each %$confdesc) {
 420     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 421 }
 422
 423 my $MAX_IDE_DISKS = 4;
 424 my $MAX_SCSI_DISKS = 14;
 425 my $MAX_VIRTIO_DISKS = 16;
 426 my $MAX_SATA_DISKS = 6;
 427 my $MAX_USB_DEVICES = 5;
 428 my $MAX_NETS = 32;
 429 my $MAX_UNUSED_DISKS = 8;
 430 my $MAX_HOSTPCI_DEVICES = 4;
 431 my $MAX_SERIAL_PORTS = 4;
 432 my $MAX_PARALLEL_PORTS = 3;
 433 my $MAX_NUMA = 8;
 434 my $MAX_MEM = 4194304;
 435 my $STATICMEM = 1024;
 436
 437 my $numadesc = {
 438     optional => 1,
 439     type => 'string', format => 'pve-qm-numanode',
 440     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 441     description => "numa topology",
 442 };
 443 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 444
 445 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 446     $confdesc->{"numa$i"} = $numadesc;
 447 }
 448
 449 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 450                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 451                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 452 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 453
 454 my $netdesc = {
 455     optional => 1,
 456     type => 'string', format => 'pve-qm-net',
 457     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 458     description => <<EODESCR,
 459 Specify network devices.
 460
 461 MODEL is one of: $nic_model_list_txt
 462
 463 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 464 automatically generated if not specified.
 465
 466 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 467
 468 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 469
 470 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 471
 472 10.0.2.2   Gateway
 473 10.0.2.3   DNS Server
 474 10.0.2.4   SMB Server
 475
 476 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 477
 478 EODESCR
 479 };
 480 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 481
 482 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 483     $confdesc->{"net$i"} = $netdesc;
 484 }
 485
 486 my $drivename_hash;
 487
 488 my $idedesc = {
 489     optional => 1,
 490     type => 'string', format => 'pve-qm-drive',
 491     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial][,model=model]',
 492     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 493 };
 494 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 495
 496 my $scsidesc = {
 497     optional => 1,
 498     type => 'string', format => 'pve-qm-drive',
 499     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 500     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 501 };
 502 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 503
 504 my $satadesc = {
 505     optional => 1,
 506     type => 'string', format => 'pve-qm-drive',
 507     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial]',
 508     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 509 };
 510 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 511
 512 my $virtiodesc = {
 513     optional => 1,
 514     type => 'string', format => 'pve-qm-drive',
 515     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,serial=serial]',
 516     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 517 };
 518 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 519
 520 my $usbdesc = {
 521     optional => 1,
 522     type => 'string', format => 'pve-qm-usb-device',
 523     typetext => 'host=HOSTUSBDEVICE|spice',
 524     description => <<EODESCR,
 525 Configure an USB device (n is 0 to 4). This can be used to
 526 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 527
 528 'bus-port(.port)*' (decimal numbers) or
 529 'vendor_id:product_id' (hexadeciaml numbers)
 530
 531 You can use the 'lsusb -t' command to list existing usb devices.
 532
 533 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 534
 535 The value 'spice' can be used to add a usb redirection devices for spice.
 536
 537 EODESCR
 538 };
 539 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 540
 541 my $hostpcidesc = {
 542         optional => 1,
 543         type => 'string', format => 'pve-qm-hostpci',
 544         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 545         description => <<EODESCR,
 546 Map host pci devices. HOSTPCIDEVICE syntax is:
 547
 548 'bus:dev.func' (hexadecimal numbers)
 549
 550 You can us the 'lspci' command to list existing pci devices.
 551
 552 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 553
 554 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 555
 556 Experimental: user reported problems with this option.
 557 EODESCR
 558 };
 559 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 560
 561 my $serialdesc = {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '(/dev/.+|socket)',
 565         description =>  <<EODESCR,
 566 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 567
 568 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 my $paralleldesc= {
 575         optional => 1,
 576         type => 'string',
 577         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 578         description =>  <<EODESCR,
 579 Map host parallel devices (n is 0 to 2).
 580
 581 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 582
 583 Experimental: user reported problems with this option.
 584 EODESCR
 585 };
 586
 587 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 588     $confdesc->{"parallel$i"} = $paralleldesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 592     $confdesc->{"serial$i"} = $serialdesc;
 593 }
 594
 595 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 596     $confdesc->{"hostpci$i"} = $hostpcidesc;
 597 }
 598
 599 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 600     $drivename_hash->{"ide$i"} = 1;
 601     $confdesc->{"ide$i"} = $idedesc;
 602 }
 603
 604 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 605     $drivename_hash->{"sata$i"} = 1;
 606     $confdesc->{"sata$i"} = $satadesc;
 607 }
 608
 609 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 610     $drivename_hash->{"scsi$i"} = 1;
 611     $confdesc->{"scsi$i"} = $scsidesc ;
 612 }
 613
 614 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 615     $drivename_hash->{"virtio$i"} = 1;
 616     $confdesc->{"virtio$i"} = $virtiodesc;
 617 }
 618
 619 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 620     $confdesc->{"usb$i"} = $usbdesc;
 621 }
 622
 623 my $unuseddesc = {
 624     optional => 1,
 625     type => 'string', format => 'pve-volume-id',
 626     description => "Reference to unused volumes.",
 627 };
 628
 629 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 630     $confdesc->{"unused$i"} = $unuseddesc;
 631 }
 632
 633 my $kvm_api_version = 0;
 634
 635 sub kvm_version {
 636
 637     return $kvm_api_version if $kvm_api_version;
 638
 639     my $fh = IO::File->new("</dev/kvm") ||
 640         return 0;
 641
 642     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 643         $kvm_api_version = $v;
 644     }
 645
 646     $fh->close();
 647
 648     return  $kvm_api_version;
 649 }
 650
 651 my $kvm_user_version;
 652
 653 sub kvm_user_version {
 654
 655     return $kvm_user_version if $kvm_user_version;
 656
 657     $kvm_user_version = 'unknown';
 658
 659     my $tmp = `kvm -help 2>/dev/null`;
 660
 661     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 662         $kvm_user_version = $2;
 663     }
 664
 665     return $kvm_user_version;
 666
 667 }
 668
 669 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 670
 671 sub disknames {
 672     # order is important - used to autoselect boot disk
 673     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 674             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 675             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 676             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 677 }
 678
 679 sub valid_drivename {
 680     my $dev = shift;
 681
 682     return defined($drivename_hash->{$dev});
 683 }
 684
 685 sub option_exists {
 686     my $key = shift;
 687     return defined($confdesc->{$key});
 688 }
 689
 690 sub nic_models {
 691     return $nic_model_list;
 692 }
 693
 694 sub os_list_description {
 695
 696     return {
 697         other => 'Other',
 698         wxp => 'Windows XP',
 699         w2k => 'Windows 2000',
 700         w2k3 =>, 'Windows 2003',
 701         w2k8 => 'Windows 2008',
 702         wvista => 'Windows Vista',
 703         win7 => 'Windows 7',
 704         win8 => 'Windows 8/2012',
 705         l24 => 'Linux 2.4',
 706         l26 => 'Linux 2.6',
 707     };
 708 }
 709
 710 my $cdrom_path;
 711
 712 sub get_cdrom_path {
 713
 714     return  $cdrom_path if $cdrom_path;
 715
 716     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 717     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 718     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 719 }
 720
 721 sub get_iso_path {
 722     my ($storecfg, $vmid, $cdrom) = @_;
 723
 724     if ($cdrom eq 'cdrom') {
 725         return get_cdrom_path();
 726     } elsif ($cdrom eq 'none') {
 727         return '';
 728     } elsif ($cdrom =~ m|^/|) {
 729         return $cdrom;
 730     } else {
 731         return PVE::Storage::path($storecfg, $cdrom);
 732     }
 733 }
 734
 735 # try to convert old style file names to volume IDs
 736 sub filename_to_volume_id {
 737     my ($vmid, $file, $media) = @_;
 738
 739     if (!($file eq 'none' || $file eq 'cdrom' ||
 740           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 741
 742         return undef if $file =~ m|/|;
 743
 744         if ($media && $media eq 'cdrom') {
 745             $file = "local:iso/$file";
 746         } else {
 747             $file = "local:$vmid/$file";
 748         }
 749     }
 750
 751     return $file;
 752 }
 753
 754 sub verify_media_type {
 755     my ($opt, $vtype, $media) = @_;
 756
 757     return if !$media;
 758
 759     my $etype;
 760     if ($media eq 'disk') {
 761         $etype = 'images';
 762     } elsif ($media eq 'cdrom') {
 763         $etype = 'iso';
 764     } else {
 765         die "internal error";
 766     }
 767
 768     return if ($vtype eq $etype);
 769
 770     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 771 }
 772
 773 sub cleanup_drive_path {
 774     my ($opt, $storecfg, $drive) = @_;
 775
 776     # try to convert filesystem paths to volume IDs
 777
 778     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 779         ($drive->{file} !~ m|^/dev/.+|) &&
 780         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 781         ($drive->{file} !~ m/^\d+$/)) {
 782         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 783         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 784         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 785         verify_media_type($opt, $vtype, $drive->{media});
 786         $drive->{file} = $volid;
 787     }
 788
 789     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 790 }
 791
 792 sub create_conf_nolock {
 793     my ($vmid, $settings) = @_;
 794
 795     my $filename = config_file($vmid);
 796
 797     die "configuration file '$filename' already exists\n" if -f $filename;
 798
 799     my $defaults = load_defaults();
 800
 801     $settings->{name} = "vm$vmid" if !$settings->{name};
 802     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 803
 804     my $data = '';
 805     foreach my $opt (keys %$settings) {
 806         next if !$confdesc->{$opt};
 807
 808         my $value = $settings->{$opt};
 809         next if !$value;
 810
 811         $data .= "$opt: $value\n";
 812     }
 813
 814     PVE::Tools::file_set_contents($filename, $data);
 815 }
 816
 817 sub parse_hotplug_features {
 818     my ($data) = @_;
 819
 820     my $res = {};
 821
 822     return $res if $data eq '0';
 823
 824     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 825
 826     foreach my $feature (PVE::Tools::split_list($data)) {
 827         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 828             $res->{$1} = 1;
 829         } else {
 830             warn "ignoring unknown hotplug feature '$feature'\n";
 831         }
 832     }
 833     return $res;
 834 }
 835
 836 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 837 sub pve_verify_hotplug_features {
 838     my ($value, $noerr) = @_;
 839
 840     return $value if parse_hotplug_features($value);
 841
 842     return undef if $noerr;
 843
 844     die "unable to parse hotplug option\n";
 845 }
 846
 847 my $parse_size = sub {
 848     my ($value) = @_;
 849
 850     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 851     my ($size, $unit) = ($1, $3);
 852     if ($unit) {
 853         if ($unit eq 'K') {
 854             $size = $size * 1024;
 855         } elsif ($unit eq 'M') {
 856             $size = $size * 1024 * 1024;
 857         } elsif ($unit eq 'G') {
 858             $size = $size * 1024 * 1024 * 1024;
 859         }
 860     }
 861     return int($size);
 862 };
 863
 864 my $format_size = sub {
 865     my ($size) = @_;
 866
 867     $size = int($size);
 868
 869     my $kb = int($size/1024);
 870     return $size if $kb*1024 != $size;
 871
 872     my $mb = int($kb/1024);
 873     return "${kb}K" if $mb*1024 != $kb;
 874
 875     my $gb = int($mb/1024);
 876     return "${mb}M" if $gb*1024 != $mb;
 877
 878     return "${gb}G";
 879 };
 880
 881 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 882 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 883 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 884 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
 885 #        [,iothread=on][,serial=serial][,model=model]
 886
 887 sub parse_drive {
 888     my ($key, $data) = @_;
 889
 890     my $res = {};
 891
 892     # $key may be undefined - used to verify JSON parameters
 893     if (!defined($key)) {
 894         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 895         $res->{index} = 0;
 896     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 897         $res->{interface} = $1;
 898         $res->{index} = $2;
 899     } else {
 900         return undef;
 901     }
 902
 903     foreach my $p (split (/,/, $data)) {
 904         next if $p =~ m/^\s*$/;
 905
 906         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|detect_zeroes|iothread|queues|serial|model)=(.+)$/) {
 907             my ($k, $v) = ($1, $2);
 908
 909             $k = 'file' if $k eq 'volume';
 910
 911             return undef if defined $res->{$k};
 912
 913             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 914                 return undef if !$v || $v !~ m/^\d+/;
 915                 $k = "m$k";
 916                 $v = sprintf("%.3f", $v / (1024*1024));
 917             }
 918             $res->{$k} = $v;
 919         } else {
 920             if (!$res->{file} && $p !~ m/=/) {
 921                 $res->{file} = $p;
 922             } else {
 923                 return undef;
 924             }
 925         }
 926     }
 927
 928     return undef if !$res->{file};
 929
 930     return undef if $res->{cache} &&
 931         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 932     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 933     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 934     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 935     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 936     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 937     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 938     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 939     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 940     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 941     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 942     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 943     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 944     return undef if $res->{detect_zeroes} && $res->{detect_zeroes} !~ m/^(on|off)$/;
 945     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 946     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 947
 948     return undef if $res->{mbps_rd} && $res->{mbps};
 949     return undef if $res->{mbps_wr} && $res->{mbps};
 950
 951     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 952     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 957
 958     return undef if $res->{iops_rd} && $res->{iops};
 959     return undef if $res->{iops_wr} && $res->{iops};
 960
 961
 962     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 963     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 964     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 966     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 968
 969     if ($res->{size}) {
 970         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 971     }
 972
 973     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 974         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 975         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 976         return undef if $res->{interface} eq 'virtio';
 977     }
 978
 979     # rerror does not work with scsi drives
 980     if ($res->{rerror}) {
 981         return undef if $res->{interface} eq 'scsi';
 982     }
 983
 984     return $res;
 985 }
 986
 987 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 988
 989 sub print_drive {
 990     my ($vmid, $drive) = @_;
 991
 992     my $opts = '';
 993     foreach my $o (@qemu_drive_options, qw(mbps mbps_rd mbps_wr mbps_max mbps_rd_max mbps_wr_max backup iothread queues detect_zeroes)) {
 994         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 995     }
 996
 997     if ($drive->{size}) {
 998         $opts .= ",size=" . &$format_size($drive->{size});
 999     }
1000
1001     if (my $model = $drive->{model}) {
1002         $opts .= ",model=$model";
1003     }
1004
1005     return "$drive->{file}$opts";
1006 }
1007
1008 sub scsi_inquiry {
1009     my($fh, $noerr) = @_;
1010
1011     my $SG_IO = 0x2285;
1012     my $SG_GET_VERSION_NUM = 0x2282;
1013
1014     my $versionbuf = "\x00" x 8;
1015     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1016     if (!$ret) {
1017         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1018         return undef;
1019     }
1020     my $version = unpack("I", $versionbuf);
1021     if ($version < 30000) {
1022         die "scsi generic interface too old\n"  if !$noerr;
1023         return undef;
1024     }
1025
1026     my $buf = "\x00" x 36;
1027     my $sensebuf = "\x00" x 8;
1028     my $cmd = pack("C x3 C x1", 0x12, 36);
1029
1030     # see /usr/include/scsi/sg.h
1031     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1032
1033     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1034                       length($sensebuf), 0, length($buf), $buf,
1035                       $cmd, $sensebuf, 6000);
1036
1037     $ret = ioctl($fh, $SG_IO, $packet);
1038     if (!$ret) {
1039         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1040         return undef;
1041     }
1042
1043     my @res = unpack($sg_io_hdr_t, $packet);
1044     if ($res[17] || $res[18]) {
1045         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1046         return undef;
1047     }
1048
1049     my $res = {};
1050     (my $byte0, my $byte1, $res->{vendor},
1051      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1052
1053     $res->{removable} = $byte1 & 128 ? 1 : 0;
1054     $res->{type} = $byte0 & 31;
1055
1056     return $res;
1057 }
1058
1059 sub path_is_scsi {
1060     my ($path) = @_;
1061
1062     my $fh = IO::File->new("+<$path") || return undef;
1063     my $res = scsi_inquiry($fh, 1);
1064     close($fh);
1065
1066     return $res;
1067 }
1068
1069 sub machine_type_is_q35 {
1070     my ($conf) = @_;
1071
1072     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1073 }
1074
1075 sub print_tabletdevice_full {
1076     my ($conf) = @_;
1077
1078     my $q35 = machine_type_is_q35($conf);
1079
1080     # we use uhci for old VMs because tablet driver was buggy in older qemu
1081     my $usbbus = $q35 ? "ehci" : "uhci";
1082
1083     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1084 }
1085
1086 sub print_drivedevice_full {
1087     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1088
1089     my $device = '';
1090     my $maxdev = 0;
1091
1092     if ($drive->{interface} eq 'virtio') {
1093         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1094         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1095         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1096     } elsif ($drive->{interface} eq 'scsi') {
1097
1098         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1099         my $unit = $drive->{index} % $maxdev;
1100         my $devicetype = 'hd';
1101         my $path = '';
1102         if (drive_is_cdrom($drive)) {
1103               $devicetype = 'cd';
1104         } else {
1105               if ($drive->{file} =~ m|^/|) {
1106                   $path = $drive->{file};
1107               } else {
1108                   $path = PVE::Storage::path($storecfg, $drive->{file});
1109               }
1110
1111               if($path =~ m/^iscsi\:\/\//){
1112                   $devicetype = 'generic';
1113               } else {
1114                   if (my $info = path_is_scsi($path)) {
1115                       if ($info->{type} == 0) {
1116                           $devicetype = 'block';
1117                       } elsif ($info->{type} == 1) { # tape
1118                           $devicetype = 'generic';
1119                       }
1120                   }
1121               }
1122          }
1123
1124         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1125             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1126         } else {
1127             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1128         }
1129
1130     } elsif ($drive->{interface} eq 'ide'){
1131         $maxdev = 2;
1132         my $controller = int($drive->{index} / $maxdev);
1133         my $unit = $drive->{index} % $maxdev;
1134         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1135
1136         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1137         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1138             $device .= ",model=$model";
1139         }
1140     } elsif ($drive->{interface} eq 'sata'){
1141         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1142         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1143         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1144     } elsif ($drive->{interface} eq 'usb') {
1145         die "implement me";
1146         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1147     } else {
1148         die "unsupported interface type";
1149     }
1150
1151     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1152
1153     return $device;
1154 }
1155
1156 sub get_initiator_name {
1157     my $initiator;
1158
1159     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1160     while (defined(my $line = <$fh>)) {
1161         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1162         $initiator = $1;
1163         last;
1164     }
1165     $fh->close();
1166
1167     return $initiator;
1168 }
1169
1170 sub print_drive_full {
1171     my ($storecfg, $vmid, $drive) = @_;
1172
1173     my $path;
1174     my $volid = $drive->{file};
1175     my $format;
1176
1177     if (drive_is_cdrom($drive)) {
1178         $path = get_iso_path($storecfg, $vmid, $volid);
1179     } else {
1180         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1181         if ($storeid) {
1182             $path = PVE::Storage::path($storecfg, $volid);
1183             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1184             $format = qemu_img_format($scfg, $volname);
1185         } else {
1186             $path = $volid;
1187         }
1188    }
1189
1190     my $opts = '';
1191     foreach my $o (@qemu_drive_options) {
1192         next if $o eq 'bootindex';
1193         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1194     }
1195
1196     $opts .= ",format=$format" if $format && !$drive->{format};
1197
1198     foreach my $o (qw(bps bps_rd bps_wr)) {
1199         my $v = $drive->{"m$o"};
1200         $opts .= ",$o=" . int($v*1024*1024) if $v;
1201     }
1202
1203     my $cache_direct = 0;
1204
1205     if (my $cache = $drive->{cache}) {
1206         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1207     } elsif (!drive_is_cdrom($drive)) {
1208         $opts .= ",cache=none";
1209         $cache_direct = 1;
1210     }
1211
1212     # aio native works only with O_DIRECT
1213     if (!$drive->{aio}) {
1214         if($cache_direct) {
1215             $opts .= ",aio=native";
1216         } else {
1217             $opts .= ",aio=threads";
1218         }
1219     }
1220
1221     if (!drive_is_cdrom($drive)) {
1222         my $detectzeroes;
1223         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1224             $detectzeroes = 'off';
1225         } elsif ($drive->{discard}) {
1226             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1227         } else {
1228             # This used to be our default with discard not being specified:
1229             $detectzeroes = 'on';
1230         }
1231         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1232     }
1233
1234     my $pathinfo = $path ? "file=$path," : '';
1235
1236     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1237 }
1238
1239 sub print_netdevice_full {
1240     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1241
1242     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1243
1244     my $device = $net->{model};
1245     if ($net->{model} eq 'virtio') {
1246          $device = 'virtio-net-pci';
1247      };
1248
1249     my $pciaddr = print_pci_addr("$netid", $bridges);
1250     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1251     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1252         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1253         my $vectors = $net->{queues} * 2 + 2;
1254         $tmpstr .= ",vectors=$vectors,mq=on";
1255     }
1256     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1257
1258     if ($use_old_bios_files) {
1259         my $romfile;
1260         if ($device eq 'virtio-net-pci') {
1261             $romfile = 'pxe-virtio.rom';
1262         } elsif ($device eq 'e1000') {
1263             $romfile = 'pxe-e1000.rom';
1264         } elsif ($device eq 'ne2k') {
1265             $romfile = 'pxe-ne2k_pci.rom';
1266         } elsif ($device eq 'pcnet') {
1267             $romfile = 'pxe-pcnet.rom';
1268         } elsif ($device eq 'rtl8139') {
1269             $romfile = 'pxe-rtl8139.rom';
1270         }
1271         $tmpstr .= ",romfile=$romfile" if $romfile;
1272     }
1273
1274     return $tmpstr;
1275 }
1276
1277 sub print_netdev_full {
1278     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1279
1280     my $i = '';
1281     if ($netid =~ m/^net(\d+)$/) {
1282         $i = int($1);
1283     }
1284
1285     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1286
1287     my $ifname = "tap${vmid}i$i";
1288
1289     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1290     die "interface name '$ifname' is too long (max 15 character)\n"
1291         if length($ifname) >= 16;
1292
1293     my $vhostparam = '';
1294     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1295
1296     my $vmname = $conf->{name} || "vm$vmid";
1297
1298     my $netdev = "";
1299     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1300
1301     if ($net->{bridge}) {
1302         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1303     } else {
1304         $netdev = "type=user,id=$netid,hostname=$vmname";
1305     }
1306
1307     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1308
1309     return $netdev;
1310 }
1311
1312 sub drive_is_cdrom {
1313     my ($drive) = @_;
1314
1315     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1316
1317 }
1318
1319 sub parse_numa {
1320     my ($data) = @_;
1321
1322     my $res = {};
1323
1324     foreach my $kvp (split(/,/, $data)) {
1325
1326         if ($kvp =~ m/^memory=(\S+)$/) {
1327             $res->{memory} = $1;
1328         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1329             $res->{policy} = $1;
1330         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1331             $res->{cpus}->{start} = $1;
1332             $res->{cpus}->{end} = $3;
1333         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1334             $res->{hostnodes}->{start} = $1;
1335             $res->{hostnodes}->{end} = $3;
1336         } else {
1337             return undef;
1338         }
1339     }
1340
1341     return $res;
1342 }
1343
1344 sub parse_hostpci {
1345     my ($value) = @_;
1346
1347     return undef if !$value;
1348
1349
1350     my @list = split(/,/, $value);
1351     my $found;
1352
1353     my $res = {};
1354     foreach my $kv (@list) {
1355
1356         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1357             $found = 1;
1358             if(defined($4)){
1359                 push @{$res->{pciid}}, { id => $2 , function => $4};
1360
1361             }else{
1362                 my $pcidevices = lspci($2);
1363                 $res->{pciid} = $pcidevices->{$2};
1364             }
1365         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1366             $res->{rombar} = $1;
1367         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1368             $res->{'x-vga'} = $1;
1369         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1370             $res->{pcie} = 1 if $1 == 1;
1371         } else {
1372             warn "unknown hostpci setting '$kv'\n";
1373         }
1374     }
1375
1376     return undef if !$found;
1377
1378     return $res;
1379 }
1380
1381 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1382 sub parse_net {
1383     my ($data) = @_;
1384
1385     my $res = {};
1386
1387     foreach my $kvp (split(/,/, $data)) {
1388
1389         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1390             my $model = lc($1);
1391             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1392             $res->{model} = $model;
1393             $res->{macaddr} = $mac;
1394         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1395             $res->{bridge} = $1;
1396         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1397             $res->{queues} = $1;
1398         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1399             $res->{rate} = $1;
1400         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1401             $res->{tag} = $1;
1402         } elsif ($kvp =~ m/^firewall=([01])$/) {
1403             $res->{firewall} = $1;
1404         } elsif ($kvp =~ m/^link_down=([01])$/) {
1405             $res->{link_down} = $1;
1406         } else {
1407             return undef;
1408         }
1409
1410     }
1411
1412     return undef if !$res->{model};
1413
1414     return $res;
1415 }
1416
1417 sub print_net {
1418     my $net = shift;
1419
1420     my $res = "$net->{model}";
1421     $res .= "=$net->{macaddr}" if $net->{macaddr};
1422     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1423     $res .= ",rate=$net->{rate}" if $net->{rate};
1424     $res .= ",tag=$net->{tag}" if $net->{tag};
1425     $res .= ",firewall=1" if $net->{firewall};
1426     $res .= ",link_down=1" if $net->{link_down};
1427     $res .= ",queues=$net->{queues}" if $net->{queues};
1428
1429     return $res;
1430 }
1431
1432 sub add_random_macs {
1433     my ($settings) = @_;
1434
1435     foreach my $opt (keys %$settings) {
1436         next if $opt !~ m/^net(\d+)$/;
1437         my $net = parse_net($settings->{$opt});
1438         next if !$net;
1439         $settings->{$opt} = print_net($net);
1440     }
1441 }
1442
1443 sub add_unused_volume {
1444     my ($config, $volid) = @_;
1445
1446     my $key;
1447     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1448         my $test = "unused$ind";
1449         if (my $vid = $config->{$test}) {
1450             return if $vid eq $volid; # do not add duplicates
1451         } else {
1452             $key = $test;
1453         }
1454     }
1455
1456     die "To many unused volume - please delete them first.\n" if !$key;
1457
1458     $config->{$key} = $volid;
1459
1460     return $key;
1461 }
1462
1463 sub vm_is_volid_owner {
1464     my ($storecfg, $vmid, $volid) = @_;
1465
1466     if ($volid !~  m|^/|) {
1467         my ($path, $owner);
1468         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1469         if ($owner && ($owner == $vmid)) {
1470             return 1;
1471         }
1472     }
1473
1474     return undef;
1475 }
1476
1477 sub split_flagged_list {
1478     my $text = shift || '';
1479     $text =~ s/[,;]/ /g;
1480     $text =~ s/^\s+//;
1481     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1482 }
1483
1484 sub join_flagged_list {
1485     my ($how, $lst) = @_;
1486     join $how, map { $lst->{$_} . $_ } keys %$lst;
1487 }
1488
1489 sub vmconfig_delete_pending_option {
1490     my ($conf, $key, $force) = @_;
1491
1492     delete $conf->{pending}->{$key};
1493     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1494     $pending_delete_hash->{$key} = $force ? '!' : '';
1495     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1496 }
1497
1498 sub vmconfig_undelete_pending_option {
1499     my ($conf, $key) = @_;
1500
1501     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1502     delete $pending_delete_hash->{$key};
1503
1504     if (%$pending_delete_hash) {
1505         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1506     } else {
1507         delete $conf->{pending}->{delete};
1508     }
1509 }
1510
1511 sub vmconfig_register_unused_drive {
1512     my ($storecfg, $vmid, $conf, $drive) = @_;
1513
1514     if (!drive_is_cdrom($drive)) {
1515         my $volid = $drive->{file};
1516         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1517             add_unused_volume($conf, $volid, $vmid);
1518         }
1519     }
1520 }
1521
1522 sub vmconfig_cleanup_pending {
1523     my ($conf) = @_;
1524
1525     # remove pending changes when nothing changed
1526     my $changes;
1527     foreach my $opt (keys %{$conf->{pending}}) {
1528         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1529             $changes = 1;
1530             delete $conf->{pending}->{$opt};
1531         }
1532     }
1533
1534     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1535     my $pending_delete_hash = {};
1536     while (my ($opt, $force) = each %$current_delete_hash) {
1537         if (defined($conf->{$opt})) {
1538             $pending_delete_hash->{$opt} = $force;
1539         } else {
1540             $changes = 1;
1541         }
1542     }
1543
1544     if (%$pending_delete_hash) {
1545         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1546     } else {
1547         delete $conf->{pending}->{delete};
1548     }
1549
1550     return $changes;
1551 }
1552
1553 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1554 my $smbios1_desc = {
1555     uuid => {
1556         type => 'string',
1557         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1558         format_description => 'UUID',
1559         optional => 1,
1560     },
1561     version => {
1562         type => 'string',
1563         pattern => '\S+',
1564         format_description => 'str',
1565         optional => 1,
1566     },
1567     serial => {
1568         type => 'string',
1569         pattern => '\S+',
1570         format_description => 'str',
1571         optional => 1,
1572     },
1573     manufacturer => {
1574         type => 'string',
1575         pattern => '\S+',
1576         format_description => 'name',
1577         optional => 1,
1578     },
1579     product => {
1580         type => 'string',
1581         pattern => '\S+',
1582         format_description => 'name',
1583         optional => 1,
1584     },
1585     sku => {
1586         type => 'string',
1587         pattern => '\S+',
1588         format_description => 'str',
1589         optional => 1,
1590     },
1591     family => {
1592         type => 'string',
1593         pattern => '\S+',
1594         format_description => 'str',
1595         optional => 1,
1596     },
1597 };
1598
1599 sub parse_smbios1 {
1600     my ($data) = @_;
1601
1602     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1603     warn $@ if $@;
1604     return $res;
1605 }
1606
1607 sub print_smbios1 {
1608     my ($smbios1) = @_;
1609     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1610 }
1611
1612 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1613
1614 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1615 sub verify_bootdisk {
1616     my ($value, $noerr) = @_;
1617
1618     return $value if valid_drivename($value);
1619
1620     return undef if $noerr;
1621
1622     die "invalid boot disk '$value'\n";
1623 }
1624
1625 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1626 sub verify_numa {
1627     my ($value, $noerr) = @_;
1628
1629     return $value if parse_numa($value);
1630
1631     return undef if $noerr;
1632
1633     die "unable to parse numa options\n";
1634 }
1635
1636 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1637 sub verify_net {
1638     my ($value, $noerr) = @_;
1639
1640     return $value if parse_net($value);
1641
1642     return undef if $noerr;
1643
1644     die "unable to parse network options\n";
1645 }
1646
1647 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1648 sub verify_drive {
1649     my ($value, $noerr) = @_;
1650
1651     return $value if parse_drive(undef, $value);
1652
1653     return undef if $noerr;
1654
1655     die "unable to parse drive options\n";
1656 }
1657
1658 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1659 sub verify_hostpci {
1660     my ($value, $noerr) = @_;
1661
1662     return $value if parse_hostpci($value);
1663
1664     return undef if $noerr;
1665
1666     die "unable to parse pci id\n";
1667 }
1668
1669 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1670 sub verify_watchdog {
1671     my ($value, $noerr) = @_;
1672
1673     return $value if parse_watchdog($value);
1674
1675     return undef if $noerr;
1676
1677     die "unable to parse watchdog options\n";
1678 }
1679
1680 sub parse_watchdog {
1681     my ($value) = @_;
1682
1683     return undef if !$value;
1684
1685     my $res = {};
1686
1687     foreach my $p (split(/,/, $value)) {
1688         next if $p =~ m/^\s*$/;
1689
1690         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1691             $res->{model} = $2;
1692         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1693             $res->{action} = $2;
1694         } else {
1695             return undef;
1696         }
1697     }
1698
1699     return $res;
1700 }
1701
1702 sub parse_usb_device {
1703     my ($value) = @_;
1704
1705     return undef if !$value;
1706
1707     my @dl = split(/,/, $value);
1708     my $found;
1709
1710     my $res = {};
1711     foreach my $v (@dl) {
1712         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1713             $found = 1;
1714             $res->{vendorid} = $2;
1715             $res->{productid} = $4;
1716         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1717             $found = 1;
1718             $res->{hostbus} = $1;
1719             $res->{hostport} = $2;
1720         } elsif ($v =~ m/^spice$/) {
1721             $found = 1;
1722             $res->{spice} = 1;
1723         } else {
1724             return undef;
1725         }
1726     }
1727     return undef if !$found;
1728
1729     return $res;
1730 }
1731
1732 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1733 sub verify_usb_device {
1734     my ($value, $noerr) = @_;
1735
1736     return $value if parse_usb_device($value);
1737
1738     return undef if $noerr;
1739
1740     die "unable to parse usb device\n";
1741 }
1742
1743 # add JSON properties for create and set function
1744 sub json_config_properties {
1745     my $prop = shift;
1746
1747     foreach my $opt (keys %$confdesc) {
1748         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1749         $prop->{$opt} = $confdesc->{$opt};
1750     }
1751
1752     return $prop;
1753 }
1754
1755 sub check_type {
1756     my ($key, $value) = @_;
1757
1758     die "unknown setting '$key'\n" if !$confdesc->{$key};
1759
1760     my $type = $confdesc->{$key}->{type};
1761
1762     if (!defined($value)) {
1763         die "got undefined value\n";
1764     }
1765
1766     if ($value =~ m/[\n\r]/) {
1767         die "property contains a line feed\n";
1768     }
1769
1770     if ($type eq 'boolean') {
1771         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1772         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1773         die "type check ('boolean') failed - got '$value'\n";
1774     } elsif ($type eq 'integer') {
1775         return int($1) if $value =~ m/^(\d+)$/;
1776         die "type check ('integer') failed - got '$value'\n";
1777     } elsif ($type eq 'number') {
1778         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1779         die "type check ('number') failed - got '$value'\n";
1780     } elsif ($type eq 'string') {
1781         if (my $fmt = $confdesc->{$key}->{format}) {
1782             if ($fmt eq 'pve-qm-drive') {
1783                 # special case - we need to pass $key to parse_drive()
1784                 my $drive = parse_drive($key, $value);
1785                 return $value if $drive;
1786                 die "unable to parse drive options\n";
1787             }
1788             PVE::JSONSchema::check_format($fmt, $value);
1789             return $value;
1790         }
1791         $value =~ s/^\"(.*)\"$/$1/;
1792         return $value;
1793     } else {
1794         die "internal error"
1795     }
1796 }
1797
1798 sub lock_config_full {
1799     my ($vmid, $timeout, $code, @param) = @_;
1800
1801     my $filename = config_file_lock($vmid);
1802
1803     my $res = lock_file($filename, $timeout, $code, @param);
1804
1805     die $@ if $@;
1806
1807     return $res;
1808 }
1809
1810 sub lock_config_mode {
1811     my ($vmid, $timeout, $shared, $code, @param) = @_;
1812
1813     my $filename = config_file_lock($vmid);
1814
1815     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1816
1817     die $@ if $@;
1818
1819     return $res;
1820 }
1821
1822 sub lock_config {
1823     my ($vmid, $code, @param) = @_;
1824
1825     return lock_config_full($vmid, 10, $code, @param);
1826 }
1827
1828 sub cfs_config_path {
1829     my ($vmid, $node) = @_;
1830
1831     $node = $nodename if !$node;
1832     return "nodes/$node/qemu-server/$vmid.conf";
1833 }
1834
1835 sub check_iommu_support{
1836     #fixme : need to check IOMMU support
1837     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1838
1839     my $iommu=1;
1840     return $iommu;
1841
1842 }
1843
1844 sub config_file {
1845     my ($vmid, $node) = @_;
1846
1847     my $cfspath = cfs_config_path($vmid, $node);
1848     return "/etc/pve/$cfspath";
1849 }
1850
1851 sub config_file_lock {
1852     my ($vmid) = @_;
1853
1854     return "$lock_dir/lock-$vmid.conf";
1855 }
1856
1857 sub touch_config {
1858     my ($vmid) = @_;
1859
1860     my $conf = config_file($vmid);
1861     utime undef, undef, $conf;
1862 }
1863
1864 sub destroy_vm {
1865     my ($storecfg, $vmid, $keep_empty_config) = @_;
1866
1867     my $conffile = config_file($vmid);
1868
1869     my $conf = load_config($vmid);
1870
1871     check_lock($conf);
1872
1873     # only remove disks owned by this VM
1874     foreach_drive($conf, sub {
1875         my ($ds, $drive) = @_;
1876
1877         return if drive_is_cdrom($drive);
1878
1879         my $volid = $drive->{file};
1880
1881         return if !$volid || $volid =~ m|^/|;
1882
1883         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1884         return if !$path || !$owner || ($owner != $vmid);
1885
1886         PVE::Storage::vdisk_free($storecfg, $volid);
1887     });
1888
1889     if ($keep_empty_config) {
1890         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1891     } else {
1892         unlink $conffile;
1893     }
1894
1895     # also remove unused disk
1896     eval {
1897         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1898
1899         eval {
1900             PVE::Storage::foreach_volid($dl, sub {
1901                 my ($volid, $sid, $volname, $d) = @_;
1902                 PVE::Storage::vdisk_free($storecfg, $volid);
1903             });
1904         };
1905         warn $@ if $@;
1906
1907     };
1908     warn $@ if $@;
1909 }
1910
1911 sub load_config {
1912     my ($vmid, $node) = @_;
1913
1914     my $cfspath = cfs_config_path($vmid, $node);
1915
1916     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1917
1918     die "no such VM ('$vmid')\n" if !defined($conf);
1919
1920     return $conf;
1921 }
1922
1923 sub parse_vm_config {
1924     my ($filename, $raw) = @_;
1925
1926     return undef if !defined($raw);
1927
1928     my $res = {
1929         digest => Digest::SHA::sha1_hex($raw),
1930         snapshots => {},
1931         pending => {},
1932     };
1933
1934     $filename =~ m|/qemu-server/(\d+)\.conf$|
1935         || die "got strange filename '$filename'";
1936
1937     my $vmid = $1;
1938
1939     my $conf = $res;
1940     my $descr;
1941     my $section = '';
1942
1943     my @lines = split(/\n/, $raw);
1944     foreach my $line (@lines) {
1945         next if $line =~ m/^\s*$/;
1946
1947         if ($line =~ m/^\[PENDING\]\s*$/i) {
1948             $section = 'pending';
1949             if (defined($descr)) {
1950                 $descr =~ s/\s+$//;
1951                 $conf->{description} = $descr;
1952             }
1953             $descr = undef;
1954             $conf = $res->{$section} = {};
1955             next;
1956
1957         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1958             $section = $1;
1959             if (defined($descr)) {
1960                 $descr =~ s/\s+$//;
1961                 $conf->{description} = $descr;
1962             }
1963             $descr = undef;
1964             $conf = $res->{snapshots}->{$section} = {};
1965             next;
1966         }
1967
1968         if ($line =~ m/^\#(.*)\s*$/) {
1969             $descr = '' if !defined($descr);
1970             $descr .= PVE::Tools::decode_text($1) . "\n";
1971             next;
1972         }
1973
1974         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1975             $descr = '' if !defined($descr);
1976             $descr .= PVE::Tools::decode_text($2);
1977         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1978             $conf->{snapstate} = $1;
1979         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1980             my $key = $1;
1981             my $value = $2;
1982             $conf->{$key} = $value;
1983         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1984             my $value = $1;
1985             if ($section eq 'pending') {
1986                 $conf->{delete} = $value; # we parse this later
1987             } else {
1988                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1989             }
1990         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1991             my $key = $1;
1992             my $value = $2;
1993             eval { $value = check_type($key, $value); };
1994             if ($@) {
1995                 warn "vm $vmid - unable to parse value of '$key' - $@";
1996             } else {
1997                 my $fmt = $confdesc->{$key}->{format};
1998                 if ($fmt && $fmt eq 'pve-qm-drive') {
1999                     my $v = parse_drive($key, $value);
2000                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2001                         $v->{file} = $volid;
2002                         $value = print_drive($vmid, $v);
2003                     } else {
2004                         warn "vm $vmid - unable to parse value of '$key'\n";
2005                         next;
2006                     }
2007                 }
2008
2009                 if ($key eq 'cdrom') {
2010                     $conf->{ide2} = $value;
2011                 } else {
2012                     $conf->{$key} = $value;
2013                 }
2014             }
2015         }
2016     }
2017
2018     if (defined($descr)) {
2019         $descr =~ s/\s+$//;
2020         $conf->{description} = $descr;
2021     }
2022     delete $res->{snapstate}; # just to be sure
2023
2024     return $res;
2025 }
2026
2027 sub write_vm_config {
2028     my ($filename, $conf) = @_;
2029
2030     delete $conf->{snapstate}; # just to be sure
2031
2032     if ($conf->{cdrom}) {
2033         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2034         $conf->{ide2} = $conf->{cdrom};
2035         delete $conf->{cdrom};
2036     }
2037
2038     # we do not use 'smp' any longer
2039     if ($conf->{sockets}) {
2040         delete $conf->{smp};
2041     } elsif ($conf->{smp}) {
2042         $conf->{sockets} = $conf->{smp};
2043         delete $conf->{cores};
2044         delete $conf->{smp};
2045     }
2046
2047     my $used_volids = {};
2048
2049     my $cleanup_config = sub {
2050         my ($cref, $pending, $snapname) = @_;
2051
2052         foreach my $key (keys %$cref) {
2053             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2054                 $key eq 'snapstate' || $key eq 'pending';
2055             my $value = $cref->{$key};
2056             if ($key eq 'delete') {
2057                 die "propertry 'delete' is only allowed in [PENDING]\n"
2058                     if !$pending;
2059                 # fixme: check syntax?
2060                 next;
2061             }
2062             eval { $value = check_type($key, $value); };
2063             die "unable to parse value of '$key' - $@" if $@;
2064
2065             $cref->{$key} = $value;
2066
2067             if (!$snapname && valid_drivename($key)) {
2068                 my $drive = parse_drive($key, $value);
2069                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2070             }
2071         }
2072     };
2073
2074     &$cleanup_config($conf);
2075
2076     &$cleanup_config($conf->{pending}, 1);
2077
2078     foreach my $snapname (keys %{$conf->{snapshots}}) {
2079         die "internal error" if $snapname eq 'pending';
2080         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2081     }
2082
2083     # remove 'unusedX' settings if we re-add a volume
2084     foreach my $key (keys %$conf) {
2085         my $value = $conf->{$key};
2086         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2087             delete $conf->{$key};
2088         }
2089     }
2090
2091     my $generate_raw_config = sub {
2092         my ($conf, $pending) = @_;
2093
2094         my $raw = '';
2095
2096         # add description as comment to top of file
2097         if (defined(my $descr = $conf->{description})) {
2098             if ($descr) {
2099                 foreach my $cl (split(/\n/, $descr)) {
2100                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2101                 }
2102             } else {
2103                 $raw .= "#\n" if $pending;
2104             }
2105         }
2106
2107         foreach my $key (sort keys %$conf) {
2108             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2109             $raw .= "$key: $conf->{$key}\n";
2110         }
2111         return $raw;
2112     };
2113
2114     my $raw = &$generate_raw_config($conf);
2115
2116     if (scalar(keys %{$conf->{pending}})){
2117         $raw .= "\n[PENDING]\n";
2118         $raw .= &$generate_raw_config($conf->{pending}, 1);
2119     }
2120
2121     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2122         $raw .= "\n[$snapname]\n";
2123         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2124     }
2125
2126     return $raw;
2127 }
2128
2129 sub update_config_nolock {
2130     my ($vmid, $conf, $skiplock) = @_;
2131
2132     check_lock($conf) if !$skiplock;
2133
2134     my $cfspath = cfs_config_path($vmid);
2135
2136     PVE::Cluster::cfs_write_file($cfspath, $conf);
2137 }
2138
2139 sub update_config {
2140     my ($vmid, $conf, $skiplock) = @_;
2141
2142     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2143 }
2144
2145 sub load_defaults {
2146
2147     my $res = {};
2148
2149     # we use static defaults from our JSON schema configuration
2150     foreach my $key (keys %$confdesc) {
2151         if (defined(my $default = $confdesc->{$key}->{default})) {
2152             $res->{$key} = $default;
2153         }
2154     }
2155
2156     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2157     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2158
2159     return $res;
2160 }
2161
2162 sub config_list {
2163     my $vmlist = PVE::Cluster::get_vmlist();
2164     my $res = {};
2165     return $res if !$vmlist || !$vmlist->{ids};
2166     my $ids = $vmlist->{ids};
2167
2168     foreach my $vmid (keys %$ids) {
2169         my $d = $ids->{$vmid};
2170         next if !$d->{node} || $d->{node} ne $nodename;
2171         next if !$d->{type} || $d->{type} ne 'qemu';
2172         $res->{$vmid}->{exists} = 1;
2173     }
2174     return $res;
2175 }
2176
2177 # test if VM uses local resources (to prevent migration)
2178 sub check_local_resources {
2179     my ($conf, $noerr) = @_;
2180
2181     my $loc_res = 0;
2182
2183     $loc_res = 1 if $conf->{hostusb}; # old syntax
2184     $loc_res = 1 if $conf->{hostpci}; # old syntax
2185
2186     foreach my $k (keys %$conf) {
2187         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2188         # sockets are safe: they will recreated be on the target side post-migrate
2189         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2190         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2191     }
2192
2193     die "VM uses local resources\n" if $loc_res && !$noerr;
2194
2195     return $loc_res;
2196 }
2197
2198 # check if used storages are available on all nodes (use by migrate)
2199 sub check_storage_availability {
2200     my ($storecfg, $conf, $node) = @_;
2201
2202     foreach_drive($conf, sub {
2203         my ($ds, $drive) = @_;
2204
2205         my $volid = $drive->{file};
2206         return if !$volid;
2207
2208         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2209         return if !$sid;
2210
2211         # check if storage is available on both nodes
2212         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2213         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2214    });
2215 }
2216
2217 # list nodes where all VM images are available (used by has_feature API)
2218 sub shared_nodes {
2219     my ($conf, $storecfg) = @_;
2220
2221     my $nodelist = PVE::Cluster::get_nodelist();
2222     my $nodehash = { map { $_ => 1 } @$nodelist };
2223     my $nodename = PVE::INotify::nodename();
2224
2225     foreach_drive($conf, sub {
2226         my ($ds, $drive) = @_;
2227
2228         my $volid = $drive->{file};
2229         return if !$volid;
2230
2231         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2232         if ($storeid) {
2233             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2234             if ($scfg->{disable}) {
2235                 $nodehash = {};
2236             } elsif (my $avail = $scfg->{nodes}) {
2237                 foreach my $node (keys %$nodehash) {
2238                     delete $nodehash->{$node} if !$avail->{$node};
2239                 }
2240             } elsif (!$scfg->{shared}) {
2241                 foreach my $node (keys %$nodehash) {
2242                     delete $nodehash->{$node} if $node ne $nodename
2243                 }
2244             }
2245         }
2246     });
2247
2248     return $nodehash
2249 }
2250
2251 sub check_lock {
2252     my ($conf) = @_;
2253
2254     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2255 }
2256
2257 sub check_cmdline {
2258     my ($pidfile, $pid) = @_;
2259
2260     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2261     if (defined($fh)) {
2262         my $line = <$fh>;
2263         $fh->close;
2264         return undef if !$line;
2265         my @param = split(/\0/, $line);
2266
2267         my $cmd = $param[0];
2268         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2269
2270         for (my $i = 0; $i < scalar (@param); $i++) {
2271             my $p = $param[$i];
2272             next if !$p;
2273             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2274                 my $p = $param[$i+1];
2275                 return 1 if $p && ($p eq $pidfile);
2276                 return undef;
2277             }
2278         }
2279     }
2280     return undef;
2281 }
2282
2283 sub check_running {
2284     my ($vmid, $nocheck, $node) = @_;
2285
2286     my $filename = config_file($vmid, $node);
2287
2288     die "unable to find configuration file for VM $vmid - no such machine\n"
2289         if !$nocheck && ! -f $filename;
2290
2291     my $pidfile = pidfile_name($vmid);
2292
2293     if (my $fd = IO::File->new("<$pidfile")) {
2294         my $st = stat($fd);
2295         my $line = <$fd>;
2296         close($fd);
2297
2298         my $mtime = $st->mtime;
2299         if ($mtime > time()) {
2300             warn "file '$filename' modified in future\n";
2301         }
2302
2303         if ($line =~ m/^(\d+)$/) {
2304             my $pid = $1;
2305             if (check_cmdline($pidfile, $pid)) {
2306                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2307                     return $pid;
2308                 }
2309             }
2310         }
2311     }
2312
2313     return undef;
2314 }
2315
2316 sub vzlist {
2317
2318     my $vzlist = config_list();
2319
2320     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2321
2322     while (defined(my $de = $fd->read)) {
2323         next if $de !~ m/^(\d+)\.pid$/;
2324         my $vmid = $1;
2325         next if !defined($vzlist->{$vmid});
2326         if (my $pid = check_running($vmid)) {
2327             $vzlist->{$vmid}->{pid} = $pid;
2328         }
2329     }
2330
2331     return $vzlist;
2332 }
2333
2334 sub disksize {
2335     my ($storecfg, $conf) = @_;
2336
2337     my $bootdisk = $conf->{bootdisk};
2338     return undef if !$bootdisk;
2339     return undef if !valid_drivename($bootdisk);
2340
2341     return undef if !$conf->{$bootdisk};
2342
2343     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2344     return undef if !defined($drive);
2345
2346     return undef if drive_is_cdrom($drive);
2347
2348     my $volid = $drive->{file};
2349     return undef if !$volid;
2350
2351     return $drive->{size};
2352 }
2353
2354 my $last_proc_pid_stat;
2355
2356 # get VM status information
2357 # This must be fast and should not block ($full == false)
2358 # We only query KVM using QMP if $full == true (this can be slow)
2359 sub vmstatus {
2360     my ($opt_vmid, $full) = @_;
2361
2362     my $res = {};
2363
2364     my $storecfg = PVE::Storage::config();
2365
2366     my $list = vzlist();
2367     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2368
2369     my $cpucount = $cpuinfo->{cpus} || 1;
2370
2371     foreach my $vmid (keys %$list) {
2372         next if $opt_vmid && ($vmid ne $opt_vmid);
2373
2374         my $cfspath = cfs_config_path($vmid);
2375         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2376
2377         my $d = {};
2378         $d->{pid} = $list->{$vmid}->{pid};
2379
2380         # fixme: better status?
2381         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2382
2383         my $size = disksize($storecfg, $conf);
2384         if (defined($size)) {
2385             $d->{disk} = 0; # no info available
2386             $d->{maxdisk} = $size;
2387         } else {
2388             $d->{disk} = 0;
2389             $d->{maxdisk} = 0;
2390         }
2391
2392         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2393         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2394         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2395
2396         $d->{name} = $conf->{name} || "VM $vmid";
2397         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2398
2399         if ($conf->{balloon}) {
2400             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2401             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2402         }
2403
2404         $d->{uptime} = 0;
2405         $d->{cpu} = 0;
2406         $d->{mem} = 0;
2407
2408         $d->{netout} = 0;
2409         $d->{netin} = 0;
2410
2411         $d->{diskread} = 0;
2412         $d->{diskwrite} = 0;
2413
2414         $d->{template} = is_template($conf);
2415
2416         $res->{$vmid} = $d;
2417     }
2418
2419     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2420     foreach my $dev (keys %$netdev) {
2421         next if $dev !~ m/^tap([1-9]\d*)i/;
2422         my $vmid = $1;
2423         my $d = $res->{$vmid};
2424         next if !$d;
2425
2426         $d->{netout} += $netdev->{$dev}->{receive};
2427         $d->{netin} += $netdev->{$dev}->{transmit};
2428
2429         if ($full) {
2430             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2431             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2432         }
2433
2434     }
2435
2436     my $ctime = gettimeofday;
2437
2438     foreach my $vmid (keys %$list) {
2439
2440         my $d = $res->{$vmid};
2441         my $pid = $d->{pid};
2442         next if !$pid;
2443
2444         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2445         next if !$pstat; # not running
2446
2447         my $used = $pstat->{utime} + $pstat->{stime};
2448
2449         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2450
2451         if ($pstat->{vsize}) {
2452             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2453         }
2454
2455         my $old = $last_proc_pid_stat->{$pid};
2456         if (!$old) {
2457             $last_proc_pid_stat->{$pid} = {
2458                 time => $ctime,
2459                 used => $used,
2460                 cpu => 0,
2461             };
2462             next;
2463         }
2464
2465         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2466
2467         if ($dtime > 1000) {
2468             my $dutime = $used -  $old->{used};
2469
2470             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2471             $last_proc_pid_stat->{$pid} = {
2472                 time => $ctime,
2473                 used => $used,
2474                 cpu => $d->{cpu},
2475             };
2476         } else {
2477             $d->{cpu} = $old->{cpu};
2478         }
2479     }
2480
2481     return $res if !$full;
2482
2483     my $qmpclient = PVE::QMPClient->new();
2484
2485     my $ballooncb = sub {
2486         my ($vmid, $resp) = @_;
2487
2488         my $info = $resp->{'return'};
2489         return if !$info->{max_mem};
2490
2491         my $d = $res->{$vmid};
2492
2493         # use memory assigned to VM
2494         $d->{maxmem} = $info->{max_mem};
2495         $d->{balloon} = $info->{actual};
2496
2497         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2498             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2499             $d->{freemem} = $info->{free_mem};
2500         }
2501
2502         $d->{ballooninfo} = $info;
2503     };
2504
2505     my $blockstatscb = sub {
2506         my ($vmid, $resp) = @_;
2507         my $data = $resp->{'return'} || [];
2508         my $totalrdbytes = 0;
2509         my $totalwrbytes = 0;
2510
2511         for my $blockstat (@$data) {
2512             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2513             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2514
2515             $blockstat->{device} =~ s/drive-//;
2516             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2517         }
2518         $res->{$vmid}->{diskread} = $totalrdbytes;
2519         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2520     };
2521
2522     my $statuscb = sub {
2523         my ($vmid, $resp) = @_;
2524
2525         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2526         # this fails if ballon driver is not loaded, so this must be
2527         # the last commnand (following command are aborted if this fails).
2528         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2529
2530         my $status = 'unknown';
2531         if (!defined($status = $resp->{'return'}->{status})) {
2532             warn "unable to get VM status\n";
2533             return;
2534         }
2535
2536         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2537     };
2538
2539     foreach my $vmid (keys %$list) {
2540         next if $opt_vmid && ($vmid ne $opt_vmid);
2541         next if !$res->{$vmid}->{pid}; # not running
2542         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2543     }
2544
2545     $qmpclient->queue_execute(undef, 1);
2546
2547     foreach my $vmid (keys %$list) {
2548         next if $opt_vmid && ($vmid ne $opt_vmid);
2549         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2550     }
2551
2552     return $res;
2553 }
2554
2555 sub foreach_dimm {
2556     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2557
2558     my $dimm_id = 0;
2559     my $current_size = 1024;
2560     my $dimm_size = 512;
2561     return if $current_size == $memory;
2562
2563     for (my $j = 0; $j < 8; $j++) {
2564         for (my $i = 0; $i < 32; $i++) {
2565             my $name = "dimm${dimm_id}";
2566             $dimm_id++;
2567             my $numanode = $i % $sockets;
2568             $current_size += $dimm_size;
2569             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2570             return  $current_size if $current_size >= $memory;
2571         }
2572         $dimm_size *= 2;
2573     }
2574 }
2575
2576 sub foreach_reverse_dimm {
2577     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2578
2579     my $dimm_id = 253;
2580     my $current_size = 4177920;
2581     my $dimm_size = 65536;
2582     return if $current_size == $memory;
2583
2584     for (my $j = 0; $j < 8; $j++) {
2585         for (my $i = 0; $i < 32; $i++) {
2586             my $name = "dimm${dimm_id}";
2587             $dimm_id--;
2588             my $numanode = $i % $sockets;
2589             $current_size -= $dimm_size;
2590             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2591             return  $current_size if $current_size <= $memory;
2592         }
2593         $dimm_size /= 2;
2594     }
2595 }
2596
2597 sub foreach_drive {
2598     my ($conf, $func) = @_;
2599
2600     foreach my $ds (keys %$conf) {
2601         next if !valid_drivename($ds);
2602
2603         my $drive = parse_drive($ds, $conf->{$ds});
2604         next if !$drive;
2605
2606         &$func($ds, $drive);
2607     }
2608 }
2609
2610 sub foreach_volid {
2611     my ($conf, $func) = @_;
2612
2613     my $volhash = {};
2614
2615     my $test_volid = sub {
2616         my ($volid, $is_cdrom) = @_;
2617
2618         return if !$volid;
2619
2620         $volhash->{$volid} = $is_cdrom || 0;
2621     };
2622
2623     foreach_drive($conf, sub {
2624         my ($ds, $drive) = @_;
2625         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2626     });
2627
2628     foreach my $snapname (keys %{$conf->{snapshots}}) {
2629         my $snap = $conf->{snapshots}->{$snapname};
2630         &$test_volid($snap->{vmstate}, 0);
2631         foreach_drive($snap, sub {
2632             my ($ds, $drive) = @_;
2633             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2634         });
2635     }
2636
2637     foreach my $volid (keys %$volhash) {
2638         &$func($volid, $volhash->{$volid});
2639     }
2640 }
2641
2642 sub vga_conf_has_spice {
2643     my ($vga) = @_;
2644
2645     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2646
2647     return $1 || 1;
2648 }
2649
2650 sub config_to_command {
2651     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2652
2653     my $cmd = [];
2654     my $globalFlags = [];
2655     my $machineFlags = [];
2656     my $rtcFlags = [];
2657     my $cpuFlags = [];
2658     my $devices = [];
2659     my $pciaddr = '';
2660     my $bridges = {};
2661     my $kvmver = kvm_user_version();
2662     my $vernum = 0; # unknown
2663     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2664         $vernum = $1*1000000+$2*1000;
2665     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2666         $vernum = $1*1000000+$2*1000+$3;
2667     }
2668
2669     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2670
2671     my $have_ovz = -f '/proc/vz/vestat';
2672
2673     my $q35 = machine_type_is_q35($conf);
2674     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2675     my $machine_type = $forcemachine || $conf->{machine};
2676     my $use_old_bios_files = undef;
2677     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2678
2679     my $cpuunits = defined($conf->{cpuunits}) ?
2680             $conf->{cpuunits} : $defaults->{cpuunits};
2681
2682     push @$cmd, '/usr/bin/systemd-run';
2683     push @$cmd, '--scope';
2684     push @$cmd, '--slice', "qemu";
2685     push @$cmd, '--unit', $vmid;
2686     # set KillMode=none, so that systemd don't kill those scopes
2687     # at shutdown (pve-manager service should stop the VMs instead)
2688     push @$cmd, '-p', "KillMode=none";
2689     push @$cmd, '-p', "CPUShares=$cpuunits";
2690     if ($conf->{cpulimit}) {
2691         my $cpulimit = int($conf->{cpulimit} * 100);
2692         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2693     }
2694
2695     push @$cmd, '/usr/bin/kvm';
2696
2697     push @$cmd, '-id', $vmid;
2698
2699     my $use_virtio = 0;
2700
2701     my $qmpsocket = qmp_socket($vmid);
2702     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2703     push @$cmd, '-mon', "chardev=qmp,mode=control";
2704
2705     my $socket = vnc_socket($vmid);
2706     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2707
2708     push @$cmd, '-pidfile' , pidfile_name($vmid);
2709
2710     push @$cmd, '-daemonize';
2711
2712     if ($conf->{smbios1}) {
2713         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2714     }
2715
2716     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2717         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2718         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2719         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2720         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2721         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2722         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2723     }
2724
2725     if ($q35) {
2726         # the q35 chipset support native usb2, so we enable usb controller
2727         # by default for this machine type
2728         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2729     } else {
2730         $pciaddr = print_pci_addr("piix3", $bridges);
2731         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2732
2733         my $use_usb2 = 0;
2734         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2735             next if !$conf->{"usb$i"};
2736             $use_usb2 = 1;
2737         }
2738         # include usb device config
2739         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2740     }
2741
2742     my $vga = $conf->{vga};
2743
2744     my $qxlnum = vga_conf_has_spice($vga);
2745     $vga = 'qxl' if $qxlnum;
2746
2747     if (!$vga) {
2748         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2749                                 $conf->{ostype} eq 'win7' ||
2750                                 $conf->{ostype} eq 'w2k8')) {
2751             $vga = 'std';
2752         } else {
2753             $vga = 'cirrus';
2754         }
2755     }
2756
2757     # enable absolute mouse coordinates (needed by vnc)
2758     my $tablet;
2759     if (defined($conf->{tablet})) {
2760         $tablet = $conf->{tablet};
2761     } else {
2762         $tablet = $defaults->{tablet};
2763         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2764         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2765     }
2766
2767     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2768
2769     my $nohyperv;
2770     # host pci devices
2771     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2772         my $d = parse_hostpci($conf->{"hostpci$i"});
2773         next if !$d;
2774
2775         my $pcie = $d->{pcie};
2776         if($pcie){
2777             die "q35 machine model is not enabled" if !$q35;
2778             $pciaddr = print_pcie_addr("hostpci$i");
2779         }else{
2780             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2781         }
2782
2783         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2784         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2785         if ($xvga && $xvga ne '') {
2786             push @$cpuFlags, 'kvm=off';
2787             $vga = 'none';
2788             $nohyperv = 1;
2789             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2790                 $xvga = "";
2791             }
2792         }
2793         my $pcidevices = $d->{pciid};
2794         my $multifunction = 1 if @$pcidevices > 1;
2795
2796         my $j=0;
2797         foreach my $pcidevice (@$pcidevices) {
2798
2799             my $id = "hostpci$i";
2800             $id .= ".$j" if $multifunction;
2801             my $addr = $pciaddr;
2802             $addr .= ".$j" if $multifunction;
2803             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2804
2805             if($j == 0){
2806                 $devicestr .= "$rombar$xvga";
2807                 $devicestr .= ",multifunction=on" if $multifunction;
2808             }
2809
2810             push @$devices, '-device', $devicestr;
2811             $j++;
2812         }
2813     }
2814
2815     # usb devices
2816     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2817         my $d = parse_usb_device($conf->{"usb$i"});
2818         next if !$d;
2819         if ($d->{vendorid} && $d->{productid}) {
2820             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2821         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2822             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2823         } elsif ($d->{spice}) {
2824             # usb redir support for spice
2825             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2826             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2827         }
2828     }
2829
2830     # serial devices
2831     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2832         if (my $path = $conf->{"serial$i"}) {
2833             if ($path eq 'socket') {
2834                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2835                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2836                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2837             } else {
2838                 die "no such serial device\n" if ! -c $path;
2839                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2840                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2841             }
2842         }
2843     }
2844
2845     # parallel devices
2846     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2847         if (my $path = $conf->{"parallel$i"}) {
2848             die "no such parallel device\n" if ! -c $path;
2849             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2850             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2851             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2852         }
2853     }
2854
2855     my $vmname = $conf->{name} || "vm$vmid";
2856
2857     push @$cmd, '-name', $vmname;
2858
2859     my $sockets = 1;
2860     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2861     $sockets = $conf->{sockets} if  $conf->{sockets};
2862
2863     my $cores = $conf->{cores} || 1;
2864
2865     my $maxcpus = $sockets * $cores;
2866
2867     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2868
2869     my $allowed_vcpus = $cpuinfo->{cpus};
2870
2871     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2872         if ($allowed_vcpus < $maxcpus);
2873
2874     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2875
2876     push @$cmd, '-nodefaults';
2877
2878     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2879
2880     my $bootindex_hash = {};
2881     my $i = 1;
2882     foreach my $o (split(//, $bootorder)) {
2883         $bootindex_hash->{$o} = $i*100;
2884         $i++;
2885     }
2886
2887     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2888
2889     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2890
2891     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2892
2893     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2894
2895     # time drift fix
2896     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2897
2898     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2899     my $useLocaltime = $conf->{localtime};
2900
2901     if (my $ost = $conf->{ostype}) {
2902         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2903
2904         if ($ost =~ m/^w/) { # windows
2905             $useLocaltime = 1 if !defined($conf->{localtime});
2906
2907             # use time drift fix when acpi is enabled
2908             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2909                 $tdf = 1 if !defined($conf->{tdf});
2910             }
2911         }
2912
2913         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2914             $ost eq 'wvista') {
2915             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2916             push @$cmd, '-no-hpet';
2917             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2918                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
2919                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
2920                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
2921
2922             } else {
2923                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
2924             }
2925         }
2926
2927         if ($ost eq 'win7' || $ost eq 'win8') {
2928             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
2929         }
2930     }
2931
2932     push @$rtcFlags, 'driftfix=slew' if $tdf;
2933
2934     if ($nokvm) {
2935         push @$machineFlags, 'accel=tcg';
2936     } else {
2937         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2938     }
2939
2940     if ($machine_type) {
2941         push @$machineFlags, "type=${machine_type}";
2942     }
2943
2944     if ($conf->{startdate}) {
2945         push @$rtcFlags, "base=$conf->{startdate}";
2946     } elsif ($useLocaltime) {
2947         push @$rtcFlags, 'base=localtime';
2948     }
2949
2950     my $cpu = $nokvm ? "qemu64" : "kvm64";
2951     $cpu = $conf->{cpu} if $conf->{cpu};
2952
2953     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2954
2955     push @$cpuFlags , '-x2apic'
2956         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2957
2958     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2959
2960     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2961
2962     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2963
2964         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2965         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2966     }
2967
2968     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2969
2970     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2971
2972     push @$cmd, '-cpu', $cpu;
2973
2974     my $memory = $conf->{memory} || $defaults->{memory};
2975     my $static_memory = 0;
2976     my $dimm_memory = 0;
2977
2978     if ($hotplug_features->{memory}) {
2979         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2980         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2981         $static_memory = $STATICMEM;
2982         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2983         $dimm_memory = $memory - $static_memory;
2984         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2985
2986     } else {
2987
2988         $static_memory = $memory;
2989         push @$cmd, '-m', $static_memory;
2990     }
2991
2992     if ($conf->{numa}) {
2993
2994         my $numa_totalmemory = undef;
2995         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2996             next if !$conf->{"numa$i"};
2997             my $numa = parse_numa($conf->{"numa$i"});
2998             next if !$numa;
2999             # memory
3000             die "missing numa node$i memory value\n" if !$numa->{memory};
3001             my $numa_memory = $numa->{memory};
3002             $numa_totalmemory += $numa_memory;
3003             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3004
3005             # cpus
3006             my $cpus_start = $numa->{cpus}->{start};
3007             die "missing numa node$i cpus\n" if !defined($cpus_start);
3008             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3009             my $cpus = $cpus_start;
3010             if (defined($cpus_end)) {
3011                 $cpus .= "-$cpus_end";
3012                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3013             }
3014
3015             # hostnodes
3016             my $hostnodes_start = $numa->{hostnodes}->{start};
3017             if (defined($hostnodes_start)) {
3018                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3019                 my $hostnodes = $hostnodes_start;
3020                 if (defined($hostnodes_end)) {
3021                     $hostnodes .= "-$hostnodes_end";
3022                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3023                 }
3024
3025                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3026                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3027                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3028                 }
3029
3030                 # policy
3031                 my $policy = $numa->{policy};
3032                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3033                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3034             }
3035
3036             push @$cmd, '-object', $numa_object;
3037             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3038         }
3039
3040         die "total memory for NUMA nodes must be equal to vm static memory\n"
3041             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3042
3043         #if no custom tology, we split memory and cores across numa nodes
3044         if(!$numa_totalmemory) {
3045
3046             my $numa_memory = ($static_memory / $sockets) . "M";
3047
3048             for (my $i = 0; $i < $sockets; $i++)  {
3049
3050                 my $cpustart = ($cores * $i);
3051                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3052                 my $cpus = $cpustart;
3053                 $cpus .= "-$cpuend" if $cpuend;
3054
3055                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3056                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3057             }
3058         }
3059     }
3060
3061     if ($hotplug_features->{memory}) {
3062         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3063             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3064             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3065             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3066
3067             #if dimm_memory is not aligned to dimm map
3068             if($current_size > $memory) {
3069                  $conf->{memory} = $current_size;
3070                  update_config_nolock($vmid, $conf, 1);
3071             }
3072         });
3073     }
3074
3075     push @$cmd, '-S' if $conf->{freeze};
3076
3077     # set keyboard layout
3078     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3079     push @$cmd, '-k', $kb if $kb;
3080
3081     # enable sound
3082     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3083     #push @$cmd, '-soundhw', 'es1370';
3084     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3085
3086     if($conf->{agent}) {
3087         my $qgasocket = qmp_socket($vmid, 1);
3088         my $pciaddr = print_pci_addr("qga0", $bridges);
3089         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3090         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3091         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3092     }
3093
3094     my $spice_port;
3095
3096     if ($qxlnum) {
3097         if ($qxlnum > 1) {
3098             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3099                 for(my $i = 1; $i < $qxlnum; $i++){
3100                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3101                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3102                 }
3103             } else {
3104                 # assume other OS works like Linux
3105                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3106                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3107             }
3108         }
3109
3110         my $pciaddr = print_pci_addr("spice", $bridges);
3111
3112         my $nodename = PVE::INotify::nodename();
3113         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3114         $spice_port = PVE::Tools::next_spice_port($pfamily);
3115
3116         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3117
3118         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3119         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3120         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3121     }
3122
3123     # enable balloon by default, unless explicitly disabled
3124     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3125         $pciaddr = print_pci_addr("balloon0", $bridges);
3126         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3127     }
3128
3129     if ($conf->{watchdog}) {
3130         my $wdopts = parse_watchdog($conf->{watchdog});
3131         $pciaddr = print_pci_addr("watchdog", $bridges);
3132         my $watchdog = $wdopts->{model} || 'i6300esb';
3133         push @$devices, '-device', "$watchdog$pciaddr";
3134         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3135     }
3136
3137     my $vollist = [];
3138     my $scsicontroller = {};
3139     my $ahcicontroller = {};
3140     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3141
3142     # Add iscsi initiator name if available
3143     if (my $initiator = get_initiator_name()) {
3144         push @$devices, '-iscsi', "initiator-name=$initiator";
3145     }
3146
3147     foreach_drive($conf, sub {
3148         my ($ds, $drive) = @_;
3149
3150         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3151             push @$vollist, $drive->{file};
3152         }
3153
3154         $use_virtio = 1 if $ds =~ m/^virtio/;
3155
3156         if (drive_is_cdrom ($drive)) {
3157             if ($bootindex_hash->{d}) {
3158                 $drive->{bootindex} = $bootindex_hash->{d};
3159                 $bootindex_hash->{d} += 1;
3160             }
3161         } else {
3162             if ($bootindex_hash->{c}) {
3163                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3164                 $bootindex_hash->{c} += 1;
3165             }
3166         }
3167
3168         if($drive->{interface} eq 'virtio'){
3169            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3170         }
3171
3172         if ($drive->{interface} eq 'scsi') {
3173
3174             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3175
3176             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3177             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3178
3179             my $iothread = '';
3180             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3181                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3182                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3183             }
3184
3185             my $queues = '';
3186             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3187                 $queues = ",num_queues=$drive->{queues}";
3188             }
3189
3190             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3191             $scsicontroller->{$controller}=1;
3192         }
3193
3194         if ($drive->{interface} eq 'sata') {
3195            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3196            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3197            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3198            $ahcicontroller->{$controller}=1;
3199         }
3200
3201         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3202         push @$devices, '-drive',$drive_cmd;
3203         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3204     });
3205
3206     for (my $i = 0; $i < $MAX_NETS; $i++) {
3207          next if !$conf->{"net$i"};
3208          my $d = parse_net($conf->{"net$i"});
3209          next if !$d;
3210
3211          $use_virtio = 1 if $d->{model} eq 'virtio';
3212
3213          if ($bootindex_hash->{n}) {
3214             $d->{bootindex} = $bootindex_hash->{n};
3215             $bootindex_hash->{n} += 1;
3216          }
3217
3218          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3219          push @$devices, '-netdev', $netdevfull;
3220
3221          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3222          push @$devices, '-device', $netdevicefull;
3223     }
3224
3225     if (!$q35) {
3226         # add pci bridges
3227         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3228            $bridges->{1} = 1;
3229            $bridges->{2} = 1;
3230         }
3231
3232         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3233
3234         while (my ($k, $v) = each %$bridges) {
3235             $pciaddr = print_pci_addr("pci.$k");
3236             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3237         }
3238     }
3239
3240     # add custom args
3241     if ($conf->{args}) {
3242         my $aa = PVE::Tools::split_args($conf->{args});
3243         push @$cmd, @$aa;
3244     }
3245
3246     push @$cmd, @$devices;
3247     push @$cmd, '-rtc', join(',', @$rtcFlags)
3248         if scalar(@$rtcFlags);
3249     push @$cmd, '-machine', join(',', @$machineFlags)
3250         if scalar(@$machineFlags);
3251     push @$cmd, '-global', join(',', @$globalFlags)
3252         if scalar(@$globalFlags);
3253
3254     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3255 }
3256
3257 sub vnc_socket {
3258     my ($vmid) = @_;
3259     return "${var_run_tmpdir}/$vmid.vnc";
3260 }
3261
3262 sub spice_port {
3263     my ($vmid) = @_;
3264
3265     my $res = vm_mon_cmd($vmid, 'query-spice');
3266
3267     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3268 }
3269
3270 sub qmp_socket {
3271     my ($vmid, $qga) = @_;
3272     my $sockettype = $qga ? 'qga' : 'qmp';
3273     return "${var_run_tmpdir}/$vmid.$sockettype";
3274 }
3275
3276 sub pidfile_name {
3277     my ($vmid) = @_;
3278     return "${var_run_tmpdir}/$vmid.pid";
3279 }
3280
3281 sub vm_devices_list {
3282     my ($vmid) = @_;
3283
3284     my $res = vm_mon_cmd($vmid, 'query-pci');
3285     my $devices = {};
3286     foreach my $pcibus (@$res) {
3287         foreach my $device (@{$pcibus->{devices}}) {
3288             next if !$device->{'qdev_id'};
3289             if ($device->{'pci_bridge'}) {
3290                 $devices->{$device->{'qdev_id'}} = 1;
3291                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3292                     next if !$bridge_device->{'qdev_id'};
3293                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3294                     $devices->{$device->{'qdev_id'}}++;
3295                 }
3296             } else {
3297                 $devices->{$device->{'qdev_id'}} = 1;
3298             }
3299         }
3300     }
3301
3302     my $resblock = vm_mon_cmd($vmid, 'query-block');
3303     foreach my $block (@$resblock) {
3304         if($block->{device} =~ m/^drive-(\S+)/){
3305                 $devices->{$1} = 1;
3306         }
3307     }
3308
3309     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3310     foreach my $mice (@$resmice) {
3311         if ($mice->{name} eq 'QEMU HID Tablet') {
3312             $devices->{tablet} = 1;
3313             last;
3314         }
3315     }
3316
3317     return $devices;
3318 }
3319
3320 sub vm_deviceplug {
3321     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3322
3323     my $q35 = machine_type_is_q35($conf);
3324
3325     my $devices_list = vm_devices_list($vmid);
3326     return 1 if defined($devices_list->{$deviceid});
3327
3328     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3329
3330     if ($deviceid eq 'tablet') {
3331
3332         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3333
3334     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3335
3336         qemu_iothread_add($vmid, $deviceid, $device);
3337
3338         qemu_driveadd($storecfg, $vmid, $device);
3339         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3340
3341         qemu_deviceadd($vmid, $devicefull);
3342         eval { qemu_deviceaddverify($vmid, $deviceid); };
3343         if (my $err = $@) {
3344             eval { qemu_drivedel($vmid, $deviceid); };
3345             warn $@ if $@;
3346             die $err;
3347         }
3348
3349     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3350
3351
3352         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3353         my $pciaddr = print_pci_addr($deviceid);
3354         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3355
3356         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3357
3358         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3359             qemu_iothread_add($vmid, $deviceid, $device);
3360             $devicefull .= ",iothread=iothread-$deviceid";
3361         }
3362
3363         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3364             $devicefull .= ",num_queues=$device->{queues}";
3365         }
3366
3367         qemu_deviceadd($vmid, $devicefull);
3368         qemu_deviceaddverify($vmid, $deviceid);
3369
3370     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3371
3372         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3373         qemu_driveadd($storecfg, $vmid, $device);
3374
3375         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3376         eval { qemu_deviceadd($vmid, $devicefull); };
3377         if (my $err = $@) {
3378             eval { qemu_drivedel($vmid, $deviceid); };
3379             warn $@ if $@;
3380             die $err;
3381         }
3382
3383     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3384
3385         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3386
3387         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3388         my $use_old_bios_files = undef;
3389         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3390
3391         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3392         qemu_deviceadd($vmid, $netdevicefull);
3393         eval { qemu_deviceaddverify($vmid, $deviceid); };
3394         if (my $err = $@) {
3395             eval { qemu_netdevdel($vmid, $deviceid); };
3396             warn $@ if $@;
3397             die $err;
3398         }
3399
3400     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3401
3402         my $bridgeid = $2;
3403         my $pciaddr = print_pci_addr($deviceid);
3404         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3405
3406         qemu_deviceadd($vmid, $devicefull);
3407         qemu_deviceaddverify($vmid, $deviceid);
3408
3409     } else {
3410         die "can't hotplug device '$deviceid'\n";
3411     }
3412
3413     return 1;
3414 }
3415
3416 # fixme: this should raise exceptions on error!
3417 sub vm_deviceunplug {
3418     my ($vmid, $conf, $deviceid) = @_;
3419
3420     my $devices_list = vm_devices_list($vmid);
3421     return 1 if !defined($devices_list->{$deviceid});
3422
3423     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3424
3425     if ($deviceid eq 'tablet') {
3426
3427         qemu_devicedel($vmid, $deviceid);
3428
3429     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3430
3431         qemu_devicedel($vmid, $deviceid);
3432         qemu_devicedelverify($vmid, $deviceid);
3433         qemu_drivedel($vmid, $deviceid);
3434         qemu_iothread_del($conf, $vmid, $deviceid);
3435
3436     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3437
3438         qemu_devicedel($vmid, $deviceid);
3439         qemu_devicedelverify($vmid, $deviceid);
3440         qemu_iothread_del($conf, $vmid, $deviceid);
3441
3442     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3443
3444         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3445         my $device = parse_drive($deviceid, $conf->{$deviceid});
3446         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3447
3448         qemu_devicedel($vmid, $deviceid);
3449         qemu_drivedel($vmid, $deviceid);
3450         qemu_deletescsihw($conf, $vmid, $deviceid);
3451
3452     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3453
3454         qemu_devicedel($vmid, $deviceid);
3455         qemu_devicedelverify($vmid, $deviceid);
3456         qemu_netdevdel($vmid, $deviceid);
3457
3458     } else {
3459         die "can't unplug device '$deviceid'\n";
3460     }
3461
3462     return 1;
3463 }
3464
3465 sub qemu_deviceadd {
3466     my ($vmid, $devicefull) = @_;
3467
3468     $devicefull = "driver=".$devicefull;
3469     my %options =  split(/[=,]/, $devicefull);
3470
3471     vm_mon_cmd($vmid, "device_add" , %options);
3472 }
3473
3474 sub qemu_devicedel {
3475     my ($vmid, $deviceid) = @_;
3476
3477     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3478 }
3479
3480 sub qemu_iothread_add {
3481     my($vmid, $deviceid, $device) = @_;
3482
3483     if ($device->{iothread}) {
3484         my $iothreads = vm_iothreads_list($vmid);
3485         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3486     }
3487 }
3488
3489 sub qemu_iothread_del {
3490     my($conf, $vmid, $deviceid) = @_;
3491
3492     my $device = parse_drive($deviceid, $conf->{$deviceid});
3493     if ($device->{iothread}) {
3494         my $iothreads = vm_iothreads_list($vmid);
3495         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3496     }
3497 }
3498
3499 sub qemu_objectadd {
3500     my($vmid, $objectid, $qomtype) = @_;
3501
3502     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3503
3504     return 1;
3505 }
3506
3507 sub qemu_objectdel {
3508     my($vmid, $objectid) = @_;
3509
3510     vm_mon_cmd($vmid, "object-del", id => $objectid);
3511
3512     return 1;
3513 }
3514
3515 sub qemu_driveadd {
3516     my ($storecfg, $vmid, $device) = @_;
3517
3518     my $drive = print_drive_full($storecfg, $vmid, $device);
3519     $drive =~ s/\\/\\\\/g;
3520     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3521
3522     # If the command succeeds qemu prints: "OK"
3523     return 1 if $ret =~ m/OK/s;
3524
3525     die "adding drive failed: $ret\n";
3526 }
3527
3528 sub qemu_drivedel {
3529     my($vmid, $deviceid) = @_;
3530
3531     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3532     $ret =~ s/^\s+//;
3533
3534     return 1 if $ret eq "";
3535
3536     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3537     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3538
3539     die "deleting drive $deviceid failed : $ret\n";
3540 }
3541
3542 sub qemu_deviceaddverify {
3543     my ($vmid, $deviceid) = @_;
3544
3545     for (my $i = 0; $i <= 5; $i++) {
3546          my $devices_list = vm_devices_list($vmid);
3547          return 1 if defined($devices_list->{$deviceid});
3548          sleep 1;
3549     }
3550
3551     die "error on hotplug device '$deviceid'\n";
3552 }
3553
3554
3555 sub qemu_devicedelverify {
3556     my ($vmid, $deviceid) = @_;
3557
3558     # need to verify that the device is correctly removed as device_del
3559     # is async and empty return is not reliable
3560
3561     for (my $i = 0; $i <= 5; $i++) {
3562          my $devices_list = vm_devices_list($vmid);
3563          return 1 if !defined($devices_list->{$deviceid});
3564          sleep 1;
3565     }
3566
3567     die "error on hot-unplugging device '$deviceid'\n";
3568 }
3569
3570 sub qemu_findorcreatescsihw {
3571     my ($storecfg, $conf, $vmid, $device) = @_;
3572
3573     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3574
3575     my $scsihwid="$controller_prefix$controller";
3576     my $devices_list = vm_devices_list($vmid);
3577
3578     if(!defined($devices_list->{$scsihwid})) {
3579         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3580     }
3581
3582     return 1;
3583 }
3584
3585 sub qemu_deletescsihw {
3586     my ($conf, $vmid, $opt) = @_;
3587
3588     my $device = parse_drive($opt, $conf->{$opt});
3589
3590     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3591         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3592         return 1;
3593     }
3594
3595     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3596
3597     my $devices_list = vm_devices_list($vmid);
3598     foreach my $opt (keys %{$devices_list}) {
3599         if (PVE::QemuServer::valid_drivename($opt)) {
3600             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3601             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3602                 return 1;
3603             }
3604         }
3605     }
3606
3607     my $scsihwid="scsihw$controller";
3608
3609     vm_deviceunplug($vmid, $conf, $scsihwid);
3610
3611     return 1;
3612 }
3613
3614 sub qemu_add_pci_bridge {
3615     my ($storecfg, $conf, $vmid, $device) = @_;
3616
3617     my $bridges = {};
3618
3619     my $bridgeid;
3620
3621     print_pci_addr($device, $bridges);
3622
3623     while (my ($k, $v) = each %$bridges) {
3624         $bridgeid = $k;
3625     }
3626     return 1 if !defined($bridgeid) || $bridgeid < 1;
3627
3628     my $bridge = "pci.$bridgeid";
3629     my $devices_list = vm_devices_list($vmid);
3630
3631     if (!defined($devices_list->{$bridge})) {
3632         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3633     }
3634
3635     return 1;
3636 }
3637
3638 sub qemu_set_link_status {
3639     my ($vmid, $device, $up) = @_;
3640
3641     vm_mon_cmd($vmid, "set_link", name => $device,
3642                up => $up ? JSON::true : JSON::false);
3643 }
3644
3645 sub qemu_netdevadd {
3646     my ($vmid, $conf, $device, $deviceid) = @_;
3647
3648     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3649     my %options =  split(/[=,]/, $netdev);
3650
3651     vm_mon_cmd($vmid, "netdev_add",  %options);
3652     return 1;
3653 }
3654
3655 sub qemu_netdevdel {
3656     my ($vmid, $deviceid) = @_;
3657
3658     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3659 }
3660
3661 sub qemu_cpu_hotplug {
3662     my ($vmid, $conf, $vcpus) = @_;
3663
3664     my $sockets = 1;
3665     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3666     $sockets = $conf->{sockets} if  $conf->{sockets};
3667     my $cores = $conf->{cores} || 1;
3668     my $maxcpus = $sockets * $cores;
3669
3670     $vcpus = $maxcpus if !$vcpus;
3671
3672     die "you can't add more vcpus than maxcpus\n"
3673         if $vcpus > $maxcpus;
3674
3675     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3676     die "online cpu unplug is not yet possible\n"
3677         if $vcpus < $currentvcpus;
3678
3679     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3680     die "vcpus in running vm is different than configuration\n"
3681         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3682
3683     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3684         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3685     }
3686 }
3687
3688 sub qemu_memory_hotplug {
3689     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3690
3691     return $value if !check_running($vmid);
3692
3693     my $memory = $conf->{memory} || $defaults->{memory};
3694     $value = $defaults->{memory} if !$value;
3695     return $value if $value == $memory;
3696
3697     my $static_memory = $STATICMEM;
3698     my $dimm_memory = $memory - $static_memory;
3699
3700     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3701     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3702
3703
3704     my $sockets = 1;
3705     $sockets = $conf->{sockets} if $conf->{sockets};
3706
3707     if($value > $memory) {
3708
3709         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3710             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3711
3712                 return if $current_size <= $conf->{memory};
3713
3714                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3715                 if (my $err = $@) {
3716                     eval { qemu_objectdel($vmid, "mem-$name"); };
3717                     die $err;
3718                 }
3719
3720                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3721                 if (my $err = $@) {
3722                     eval { qemu_objectdel($vmid, "mem-$name"); };
3723                     die $err;
3724                 }
3725                 #update conf after each succesful module hotplug
3726                 $conf->{memory} = $current_size;
3727                 update_config_nolock($vmid, $conf, 1);
3728         });
3729
3730     } else {
3731
3732         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3733             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3734
3735                 return if $current_size >= $conf->{memory};
3736                 print "try to unplug memory dimm $name\n";
3737
3738                 my $retry = 0;
3739                 while (1) {
3740                     eval { qemu_devicedel($vmid, $name) };
3741                     sleep 3;
3742                     my $dimm_list = qemu_dimm_list($vmid);
3743                     last if !$dimm_list->{$name};
3744                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3745                     $retry++;
3746                 }
3747
3748                 #update conf after each succesful module unplug
3749                 $conf->{memory} = $current_size;
3750
3751                 eval { qemu_objectdel($vmid, "mem-$name"); };
3752                 update_config_nolock($vmid, $conf, 1);
3753         });
3754     }
3755 }
3756
3757 sub qemu_dimm_list {
3758     my ($vmid) = @_;
3759
3760     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3761     my $dimms = {};
3762
3763     foreach my $dimm (@$dimmarray) {
3764
3765         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3766         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3767         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3768         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3769         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3770     }
3771     return $dimms;
3772 }
3773
3774 sub qemu_block_set_io_throttle {
3775     my ($vmid, $deviceid,
3776         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3777         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3778
3779     return if !check_running($vmid) ;
3780
3781     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3782         bps => int($bps),
3783         bps_rd => int($bps_rd),
3784         bps_wr => int($bps_wr),
3785         iops => int($iops),
3786         iops_rd => int($iops_rd),
3787         iops_wr => int($iops_wr),
3788         bps_max => int($bps_max),
3789         bps_rd_max => int($bps_rd_max),
3790         bps_wr_max => int($bps_wr_max),
3791         iops_max => int($iops_max),
3792         iops_rd_max => int($iops_rd_max),
3793         iops_wr_max => int($iops_wr_max)
3794     );
3795
3796 }
3797
3798 # old code, only used to shutdown old VM after update
3799 sub __read_avail {
3800     my ($fh, $timeout) = @_;
3801
3802     my $sel = new IO::Select;
3803     $sel->add($fh);
3804
3805     my $res = '';
3806     my $buf;
3807
3808     my @ready;
3809     while (scalar (@ready = $sel->can_read($timeout))) {
3810         my $count;
3811         if ($count = $fh->sysread($buf, 8192)) {
3812             if ($buf =~ /^(.*)\(qemu\) $/s) {
3813                 $res .= $1;
3814                 last;
3815             } else {
3816                 $res .= $buf;
3817             }
3818         } else {
3819             if (!defined($count)) {
3820                 die "$!\n";
3821             }
3822             last;
3823         }
3824     }
3825
3826     die "monitor read timeout\n" if !scalar(@ready);
3827
3828     return $res;
3829 }
3830
3831 # old code, only used to shutdown old VM after update
3832 sub vm_monitor_command {
3833     my ($vmid, $cmdstr, $nocheck) = @_;
3834
3835     my $res;
3836
3837     eval {
3838         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3839
3840         my $sname = "${var_run_tmpdir}/$vmid.mon";
3841
3842         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3843             die "unable to connect to VM $vmid socket - $!\n";
3844
3845         my $timeout = 3;
3846
3847         # hack: migrate sometime blocks the monitor (when migrate_downtime
3848         # is set)
3849         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3850             $timeout = 60*60; # 1 hour
3851         }
3852
3853         # read banner;
3854         my $data = __read_avail($sock, $timeout);
3855
3856         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3857             die "got unexpected qemu monitor banner\n";
3858         }
3859
3860         my $sel = new IO::Select;
3861         $sel->add($sock);
3862
3863         if (!scalar(my @ready = $sel->can_write($timeout))) {
3864             die "monitor write error - timeout";
3865         }
3866
3867         my $fullcmd = "$cmdstr\r";
3868
3869         # syslog('info', "VM $vmid monitor command: $cmdstr");
3870
3871         my $b;
3872         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3873             die "monitor write error - $!";
3874         }
3875
3876         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3877
3878         $timeout = 20;
3879
3880         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3881             $timeout = 60*60; # 1 hour
3882         } elsif ($cmdstr =~ m/^(eject|change)/) {
3883             $timeout = 60; # note: cdrom mount command is slow
3884         }
3885         if ($res = __read_avail($sock, $timeout)) {
3886
3887             my @lines = split("\r?\n", $res);
3888
3889             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3890
3891             $res = join("\n", @lines);
3892             $res .= "\n";
3893         }
3894     };
3895
3896     my $err = $@;
3897
3898     if ($err) {
3899         syslog("err", "VM $vmid monitor command failed - $err");
3900         die $err;
3901     }
3902
3903     return $res;
3904 }
3905
3906 sub qemu_block_resize {
3907     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3908
3909     my $running = check_running($vmid);
3910
3911     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3912
3913     return if !$running;
3914
3915     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3916
3917 }
3918
3919 sub qemu_volume_snapshot {
3920     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3921
3922     my $running = check_running($vmid);
3923
3924     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3925         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3926     } else {
3927         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3928     }
3929 }
3930
3931 sub qemu_volume_snapshot_delete {
3932     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3933
3934     my $running = check_running($vmid);
3935
3936     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3937
3938     return if !$running;
3939
3940     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3941 }
3942
3943 sub set_migration_caps {
3944     my ($vmid) = @_;
3945
3946     my $cap_ref = [];
3947
3948     my $enabled_cap = {
3949         "auto-converge" => 1,
3950         "xbzrle" => 1,
3951         "x-rdma-pin-all" => 0,
3952         "zero-blocks" => 0,
3953         "compress" => 0
3954     };
3955
3956     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3957
3958     for my $supported_capability (@$supported_capabilities) {
3959         push @$cap_ref, {
3960             capability => $supported_capability->{capability},
3961             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3962         };
3963     }
3964
3965     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3966 }
3967
3968 my $fast_plug_option = {
3969     'lock' => 1,
3970     'name' => 1,
3971     'onboot' => 1,
3972     'shares' => 1,
3973     'startup' => 1,
3974     'description' => 1,
3975 };
3976
3977 # hotplug changes in [PENDING]
3978 # $selection hash can be used to only apply specified options, for
3979 # example: { cores => 1 } (only apply changed 'cores')
3980 # $errors ref is used to return error messages
3981 sub vmconfig_hotplug_pending {
3982     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3983
3984     my $defaults = load_defaults();
3985
3986     # commit values which do not have any impact on running VM first
3987     # Note: those option cannot raise errors, we we do not care about
3988     # $selection and always apply them.
3989
3990     my $add_error = sub {
3991         my ($opt, $msg) = @_;
3992         $errors->{$opt} = "hotplug problem - $msg";
3993     };
3994
3995     my $changes = 0;
3996     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3997         if ($fast_plug_option->{$opt}) {
3998             $conf->{$opt} = $conf->{pending}->{$opt};
3999             delete $conf->{pending}->{$opt};
4000             $changes = 1;
4001         }
4002     }
4003
4004     if ($changes) {
4005         update_config_nolock($vmid, $conf, 1);
4006         $conf = load_config($vmid); # update/reload
4007     }
4008
4009     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4010
4011     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4012     while (my ($opt, $force) = each %$pending_delete_hash) {
4013         next if $selection && !$selection->{$opt};
4014         eval {
4015             if ($opt eq 'hotplug') {
4016                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4017             } elsif ($opt eq 'tablet') {
4018                 die "skip\n" if !$hotplug_features->{usb};
4019                 if ($defaults->{tablet}) {
4020                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4021                 } else {
4022                     vm_deviceunplug($vmid, $conf, $opt);
4023                 }
4024             } elsif ($opt eq 'vcpus') {
4025                 die "skip\n" if !$hotplug_features->{cpu};
4026                 qemu_cpu_hotplug($vmid, $conf, undef);
4027             } elsif ($opt eq 'balloon') {
4028                 # enable balloon device is not hotpluggable
4029                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4030             } elsif ($fast_plug_option->{$opt}) {
4031                 # do nothing
4032             } elsif ($opt =~ m/^net(\d+)$/) {
4033                 die "skip\n" if !$hotplug_features->{network};
4034                 vm_deviceunplug($vmid, $conf, $opt);
4035             } elsif (valid_drivename($opt)) {
4036                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4037                 vm_deviceunplug($vmid, $conf, $opt);
4038                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4039             } elsif ($opt =~ m/^memory$/) {
4040                 die "skip\n" if !$hotplug_features->{memory};
4041                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4042             } elsif ($opt eq 'cpuunits') {
4043                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4044             } elsif ($opt eq 'cpulimit') {
4045                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4046             } else {
4047                 die "skip\n";
4048             }
4049         };
4050         if (my $err = $@) {
4051             &$add_error($opt, $err) if $err ne "skip\n";
4052         } else {
4053             # save new config if hotplug was successful
4054             delete $conf->{$opt};
4055             vmconfig_undelete_pending_option($conf, $opt);
4056             update_config_nolock($vmid, $conf, 1);
4057             $conf = load_config($vmid); # update/reload
4058         }
4059     }
4060
4061     foreach my $opt (keys %{$conf->{pending}}) {
4062         next if $selection && !$selection->{$opt};
4063         my $value = $conf->{pending}->{$opt};
4064         eval {
4065             if ($opt eq 'hotplug') {
4066                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4067             } elsif ($opt eq 'tablet') {
4068                 die "skip\n" if !$hotplug_features->{usb};
4069                 if ($value == 1) {
4070                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4071                 } elsif ($value == 0) {
4072                     vm_deviceunplug($vmid, $conf, $opt);
4073                 }
4074             } elsif ($opt eq 'vcpus') {
4075                 die "skip\n" if !$hotplug_features->{cpu};
4076                 qemu_cpu_hotplug($vmid, $conf, $value);
4077             } elsif ($opt eq 'balloon') {
4078                 # enable/disable balloning device is not hotpluggable
4079                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4080                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4081                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4082
4083                 # allow manual ballooning if shares is set to zero
4084                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4085                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4086                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4087                 }
4088             } elsif ($opt =~ m/^net(\d+)$/) {
4089                 # some changes can be done without hotplug
4090                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4091                                     $vmid, $opt, $value);
4092             } elsif (valid_drivename($opt)) {
4093                 # some changes can be done without hotplug
4094                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4095                                      $vmid, $opt, $value, 1);
4096             } elsif ($opt =~ m/^memory$/) { #dimms
4097                 die "skip\n" if !$hotplug_features->{memory};
4098                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4099             } elsif ($opt eq 'cpuunits') {
4100                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4101             } elsif ($opt eq 'cpulimit') {
4102                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4103                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4104             } else {
4105                 die "skip\n";  # skip non-hot-pluggable options
4106             }
4107         };
4108         if (my $err = $@) {
4109             &$add_error($opt, $err) if $err ne "skip\n";
4110         } else {
4111             # save new config if hotplug was successful
4112             $conf->{$opt} = $value;
4113             delete $conf->{pending}->{$opt};
4114             update_config_nolock($vmid, $conf, 1);
4115             $conf = load_config($vmid); # update/reload
4116         }
4117     }
4118 }
4119
4120 sub try_deallocate_drive {
4121     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4122
4123     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4124         my $volid = $drive->{file};
4125         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4126             my $sid = PVE::Storage::parse_volume_id($volid);
4127             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4128
4129             # check if the disk is really unused
4130             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4131                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4132             PVE::Storage::vdisk_free($storecfg, $volid);
4133             return 1;
4134         } else {
4135             # If vm is not owner of this disk remove from config
4136             return 1;
4137         }
4138     }
4139
4140     return undef;
4141 }
4142
4143 sub vmconfig_delete_or_detach_drive {
4144     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4145
4146     my $drive = parse_drive($opt, $conf->{$opt});
4147
4148     my $rpcenv = PVE::RPCEnvironment::get();
4149     my $authuser = $rpcenv->get_user();
4150
4151     if ($force) {
4152         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4153         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4154     } else {
4155         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4156     }
4157 }
4158
4159 sub vmconfig_apply_pending {
4160     my ($vmid, $conf, $storecfg) = @_;
4161
4162     # cold plug
4163
4164     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4165     while (my ($opt, $force) = each %$pending_delete_hash) {
4166         die "internal error" if $opt =~ m/^unused/;
4167         $conf = load_config($vmid); # update/reload
4168         if (!defined($conf->{$opt})) {
4169             vmconfig_undelete_pending_option($conf, $opt);
4170             update_config_nolock($vmid, $conf, 1);
4171         } elsif (valid_drivename($opt)) {
4172             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4173             vmconfig_undelete_pending_option($conf, $opt);
4174             delete $conf->{$opt};
4175             update_config_nolock($vmid, $conf, 1);
4176         } else {
4177             vmconfig_undelete_pending_option($conf, $opt);
4178             delete $conf->{$opt};
4179             update_config_nolock($vmid, $conf, 1);
4180         }
4181     }
4182
4183     $conf = load_config($vmid); # update/reload
4184
4185     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4186         $conf = load_config($vmid); # update/reload
4187
4188         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4189             # skip if nothing changed
4190         } elsif (valid_drivename($opt)) {
4191             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4192                 if defined($conf->{$opt});
4193             $conf->{$opt} = $conf->{pending}->{$opt};
4194         } else {
4195             $conf->{$opt} = $conf->{pending}->{$opt};
4196         }
4197
4198         delete $conf->{pending}->{$opt};
4199         update_config_nolock($vmid, $conf, 1);
4200     }
4201 }
4202
4203 my $safe_num_ne = sub {
4204     my ($a, $b) = @_;
4205
4206     return 0 if !defined($a) && !defined($b);
4207     return 1 if !defined($a);
4208     return 1 if !defined($b);
4209
4210     return $a != $b;
4211 };
4212
4213 my $safe_string_ne = sub {
4214     my ($a, $b) = @_;
4215
4216     return 0 if !defined($a) && !defined($b);
4217     return 1 if !defined($a);
4218     return 1 if !defined($b);
4219
4220     return $a ne $b;
4221 };
4222
4223 sub vmconfig_update_net {
4224     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4225
4226     my $newnet = parse_net($value);
4227
4228     if ($conf->{$opt}) {
4229         my $oldnet = parse_net($conf->{$opt});
4230
4231         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4232             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4233             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4234             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4235
4236             # for non online change, we try to hot-unplug
4237             die "skip\n" if !$hotplug;
4238             vm_deviceunplug($vmid, $conf, $opt);
4239         } else {
4240
4241             die "internal error" if $opt !~ m/net(\d+)/;
4242             my $iface = "tap${vmid}i$1";
4243
4244             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4245                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4246             }
4247
4248             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4249                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4250                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4251                 PVE::Network::tap_unplug($iface);
4252                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4253             }
4254
4255             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4256                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4257             }
4258
4259             return 1;
4260         }
4261     }
4262
4263     if ($hotplug) {
4264         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4265     } else {
4266         die "skip\n";
4267     }
4268 }
4269
4270 sub vmconfig_update_disk {
4271     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4272
4273     # fixme: do we need force?
4274
4275     my $drive = parse_drive($opt, $value);
4276
4277     if ($conf->{$opt}) {
4278
4279         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4280
4281             my $media = $drive->{media} || 'disk';
4282             my $oldmedia = $old_drive->{media} || 'disk';
4283             die "unable to change media type\n" if $media ne $oldmedia;
4284
4285             if (!drive_is_cdrom($old_drive)) {
4286
4287                 if ($drive->{file} ne $old_drive->{file}) {
4288
4289                     die "skip\n" if !$hotplug;
4290
4291                     # unplug and register as unused
4292                     vm_deviceunplug($vmid, $conf, $opt);
4293                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4294
4295                 } else {
4296                     # update existing disk
4297
4298                     # skip non hotpluggable value
4299                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4300                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4301                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4302                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4303                         die "skip\n";
4304                     }
4305
4306                     # apply throttle
4307                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4308                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4309                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4310                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4311                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4312                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4313                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4314                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4315                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4316                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4317                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4318                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4319
4320                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4321                                                    ($drive->{mbps} || 0)*1024*1024,
4322                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4323                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4324                                                    $drive->{iops} || 0,
4325                                                    $drive->{iops_rd} || 0,
4326                                                    $drive->{iops_wr} || 0,
4327                                                    ($drive->{mbps_max} || 0)*1024*1024,
4328                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4329                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4330                                                    $drive->{iops_max} || 0,
4331                                                    $drive->{iops_rd_max} || 0,
4332                                                    $drive->{iops_wr_max} || 0);
4333
4334                     }
4335
4336                     return 1;
4337                 }
4338
4339             } else { # cdrom
4340
4341                 if ($drive->{file} eq 'none') {
4342                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4343                 } else {
4344                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4345                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4346                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4347                 }
4348
4349                 return 1;
4350             }
4351         }
4352     }
4353
4354     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4355     # hotplug new disks
4356     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4357     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4358 }
4359
4360 sub vm_start {
4361     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4362         $forcemachine, $spice_ticket) = @_;
4363
4364     lock_config($vmid, sub {
4365         my $conf = load_config($vmid, $migratedfrom);
4366
4367         die "you can't start a vm if it's a template\n" if is_template($conf);
4368
4369         check_lock($conf) if !$skiplock;
4370
4371         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4372
4373         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4374             vmconfig_apply_pending($vmid, $conf, $storecfg);
4375             $conf = load_config($vmid); # update/reload
4376         }
4377
4378         my $defaults = load_defaults();
4379
4380         # set environment variable useful inside network script
4381         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4382
4383         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4384
4385         my $migrate_port = 0;
4386         my $migrate_uri;
4387         if ($statefile) {
4388             if ($statefile eq 'tcp') {
4389                 my $localip = "localhost";
4390                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4391                 my $nodename = PVE::INotify::nodename();
4392                 if ($datacenterconf->{migration_unsecure}) {
4393                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4394                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4395                 }
4396                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4397                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4398                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4399                 push @$cmd, '-incoming', $migrate_uri;
4400                 push @$cmd, '-S';
4401             } else {
4402                 push @$cmd, '-loadstate', $statefile;
4403             }
4404         } elsif ($paused) {
4405             push @$cmd, '-S';
4406         }
4407
4408         # host pci devices
4409         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4410           my $d = parse_hostpci($conf->{"hostpci$i"});
4411           next if !$d;
4412           my $pcidevices = $d->{pciid};
4413           foreach my $pcidevice (@$pcidevices) {
4414                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4415
4416                 my $info = pci_device_info("0000:$pciid");
4417                 die "IOMMU not present\n" if !check_iommu_support();
4418                 die "no pci device info for device '$pciid'\n" if !$info;
4419                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4420                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4421           }
4422         }
4423
4424         PVE::Storage::activate_volumes($storecfg, $vollist);
4425
4426         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4427                     umask => 0077); };
4428         my $err = $@;
4429         die "start failed: $err" if $err;
4430
4431         print "migration listens on $migrate_uri\n" if $migrate_uri;
4432
4433         if ($statefile && $statefile ne 'tcp')  {
4434             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4435             warn $@ if $@;
4436         }
4437
4438         if ($migratedfrom) {
4439
4440             eval {
4441                 set_migration_caps($vmid);
4442             };
4443             warn $@ if $@;
4444
4445             if ($spice_port) {
4446                 print "spice listens on port $spice_port\n";
4447                 if ($spice_ticket) {
4448                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4449                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4450                 }
4451             }
4452
4453         } else {
4454
4455             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4456                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4457                     if $conf->{balloon};
4458             }
4459
4460             foreach my $opt (keys %$conf) {
4461                 next if $opt !~  m/^net\d+$/;
4462                 my $nicconf = parse_net($conf->{$opt});
4463                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4464             }
4465         }
4466
4467         vm_mon_cmd_nocheck($vmid, 'qom-set',
4468                     path => "machine/peripheral/balloon0",
4469                     property => "guest-stats-polling-interval",
4470                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4471
4472     });
4473 }
4474
4475 sub vm_mon_cmd {
4476     my ($vmid, $execute, %params) = @_;
4477
4478     my $cmd = { execute => $execute, arguments => \%params };
4479     vm_qmp_command($vmid, $cmd);
4480 }
4481
4482 sub vm_mon_cmd_nocheck {
4483     my ($vmid, $execute, %params) = @_;
4484
4485     my $cmd = { execute => $execute, arguments => \%params };
4486     vm_qmp_command($vmid, $cmd, 1);
4487 }
4488
4489 sub vm_qmp_command {
4490     my ($vmid, $cmd, $nocheck) = @_;
4491
4492     my $res;
4493
4494     my $timeout;
4495     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4496         $timeout = $cmd->{arguments}->{timeout};
4497         delete $cmd->{arguments}->{timeout};
4498     }
4499
4500     eval {
4501         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4502         my $sname = qmp_socket($vmid);
4503         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4504             my $qmpclient = PVE::QMPClient->new();
4505
4506             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4507         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4508             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4509                 if scalar(%{$cmd->{arguments}});
4510             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4511         } else {
4512             die "unable to open monitor socket\n";
4513         }
4514     };
4515     if (my $err = $@) {
4516         syslog("err", "VM $vmid qmp command failed - $err");
4517         die $err;
4518     }
4519
4520     return $res;
4521 }
4522
4523 sub vm_human_monitor_command {
4524     my ($vmid, $cmdline) = @_;
4525
4526     my $res;
4527
4528     my $cmd = {
4529         execute => 'human-monitor-command',
4530         arguments => { 'command-line' => $cmdline},
4531     };
4532
4533     return vm_qmp_command($vmid, $cmd);
4534 }
4535
4536 sub vm_commandline {
4537     my ($storecfg, $vmid) = @_;
4538
4539     my $conf = load_config($vmid);
4540
4541     my $defaults = load_defaults();
4542
4543     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4544
4545     return join(' ', @$cmd);
4546 }
4547
4548 sub vm_reset {
4549     my ($vmid, $skiplock) = @_;
4550
4551     lock_config($vmid, sub {
4552
4553         my $conf = load_config($vmid);
4554
4555         check_lock($conf) if !$skiplock;
4556
4557         vm_mon_cmd($vmid, "system_reset");
4558     });
4559 }
4560
4561 sub get_vm_volumes {
4562     my ($conf) = @_;
4563
4564     my $vollist = [];
4565     foreach_volid($conf, sub {
4566         my ($volid, $is_cdrom) = @_;
4567
4568         return if $volid =~ m|^/|;
4569
4570         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4571         return if !$sid;
4572
4573         push @$vollist, $volid;
4574     });
4575
4576     return $vollist;
4577 }
4578
4579 sub vm_stop_cleanup {
4580     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4581
4582     eval {
4583
4584         if (!$keepActive) {
4585             my $vollist = get_vm_volumes($conf);
4586             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4587         }
4588
4589         foreach my $ext (qw(mon qmp pid vnc qga)) {
4590             unlink "/var/run/qemu-server/${vmid}.$ext";
4591         }
4592
4593         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4594     };
4595     warn $@ if $@; # avoid errors - just warn
4596 }
4597
4598 # Note: use $nockeck to skip tests if VM configuration file exists.
4599 # We need that when migration VMs to other nodes (files already moved)
4600 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4601 sub vm_stop {
4602     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4603
4604     $force = 1 if !defined($force) && !$shutdown;
4605
4606     if ($migratedfrom){
4607         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4608         kill 15, $pid if $pid;
4609         my $conf = load_config($vmid, $migratedfrom);
4610         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4611         return;
4612     }
4613
4614     lock_config($vmid, sub {
4615
4616         my $pid = check_running($vmid, $nocheck);
4617         return if !$pid;
4618
4619         my $conf;
4620         if (!$nocheck) {
4621             $conf = load_config($vmid);
4622             check_lock($conf) if !$skiplock;
4623             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4624                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4625                 $timeout = $opts->{down} if $opts->{down};
4626             }
4627         }
4628
4629         $timeout = 60 if !defined($timeout);
4630
4631         eval {
4632             if ($shutdown) {
4633                 if (defined($conf) && $conf->{agent}) {
4634                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4635                 } else {
4636                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4637                 }
4638             } else {
4639                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4640             }
4641         };
4642         my $err = $@;
4643
4644         if (!$err) {
4645             my $count = 0;
4646             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4647                 $count++;
4648                 sleep 1;
4649             }
4650
4651             if ($count >= $timeout) {
4652                 if ($force) {
4653                     warn "VM still running - terminating now with SIGTERM\n";
4654                     kill 15, $pid;
4655                 } else {
4656                     die "VM quit/powerdown failed - got timeout\n";
4657                 }
4658             } else {
4659                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4660                 return;
4661             }
4662         } else {
4663             if ($force) {
4664                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4665                 kill 15, $pid;
4666             } else {
4667                 die "VM quit/powerdown failed\n";
4668             }
4669         }
4670
4671         # wait again
4672         $timeout = 10;
4673
4674         my $count = 0;
4675         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4676             $count++;
4677             sleep 1;
4678         }
4679
4680         if ($count >= $timeout) {
4681             warn "VM still running - terminating now with SIGKILL\n";
4682             kill 9, $pid;
4683             sleep 1;
4684         }
4685
4686         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4687    });
4688 }
4689
4690 sub vm_suspend {
4691     my ($vmid, $skiplock) = @_;
4692
4693     lock_config($vmid, sub {
4694
4695         my $conf = load_config($vmid);
4696
4697         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4698
4699         vm_mon_cmd($vmid, "stop");
4700     });
4701 }
4702
4703 sub vm_resume {
4704     my ($vmid, $skiplock, $nocheck) = @_;
4705
4706     lock_config($vmid, sub {
4707
4708         if (!$nocheck) {
4709
4710             my $conf = load_config($vmid);
4711
4712             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4713
4714             vm_mon_cmd($vmid, "cont");
4715
4716         } else {
4717             vm_mon_cmd_nocheck($vmid, "cont");
4718         }
4719     });
4720 }
4721
4722 sub vm_sendkey {
4723     my ($vmid, $skiplock, $key) = @_;
4724
4725     lock_config($vmid, sub {
4726
4727         my $conf = load_config($vmid);
4728
4729         # there is no qmp command, so we use the human monitor command
4730         vm_human_monitor_command($vmid, "sendkey $key");
4731     });
4732 }
4733
4734 sub vm_destroy {
4735     my ($storecfg, $vmid, $skiplock) = @_;
4736
4737     lock_config($vmid, sub {
4738
4739         my $conf = load_config($vmid);
4740
4741         check_lock($conf) if !$skiplock;
4742
4743         if (!check_running($vmid)) {
4744             destroy_vm($storecfg, $vmid);
4745         } else {
4746             die "VM $vmid is running - destroy failed\n";
4747         }
4748     });
4749 }
4750
4751 # pci helpers
4752
4753 sub file_write {
4754     my ($filename, $buf) = @_;
4755
4756     my $fh = IO::File->new($filename, "w");
4757     return undef if !$fh;
4758
4759     my $res = print $fh $buf;
4760
4761     $fh->close();
4762
4763     return $res;
4764 }
4765
4766 sub pci_device_info {
4767     my ($name) = @_;
4768
4769     my $res;
4770
4771     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4772     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4773
4774     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4775     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4776
4777     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4778     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4779
4780     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4781     return undef if !defined($product) || $product !~ s/^0x//;
4782
4783     $res = {
4784         name => $name,
4785         vendor => $vendor,
4786         product => $product,
4787         domain => $domain,
4788         bus => $bus,
4789         slot => $slot,
4790         func => $func,
4791         irq => $irq,
4792         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4793     };
4794
4795     return $res;
4796 }
4797
4798 sub pci_dev_reset {
4799     my ($dev) = @_;
4800
4801     my $name = $dev->{name};
4802
4803     my $fn = "$pcisysfs/devices/$name/reset";
4804
4805     return file_write($fn, "1");
4806 }
4807
4808 sub pci_dev_bind_to_vfio {
4809     my ($dev) = @_;
4810
4811     my $name = $dev->{name};
4812
4813     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4814
4815     if (!-d $vfio_basedir) {
4816         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4817     }
4818     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4819
4820     my $testdir = "$vfio_basedir/$name";
4821     return 1 if -d $testdir;
4822
4823     my $data = "$dev->{vendor} $dev->{product}";
4824     return undef if !file_write("$vfio_basedir/new_id", $data);
4825
4826     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4827     if (!file_write($fn, $name)) {
4828         return undef if -f $fn;
4829     }
4830
4831     $fn = "$vfio_basedir/bind";
4832     if (! -d $testdir) {
4833         return undef if !file_write($fn, $name);
4834     }
4835
4836     return -d $testdir;
4837 }
4838
4839 sub pci_dev_group_bind_to_vfio {
4840     my ($pciid) = @_;
4841
4842     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4843
4844     if (!-d $vfio_basedir) {
4845         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4846     }
4847     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4848
4849     # get IOMMU group devices
4850     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4851       my @devs = grep /^0000:/, readdir($D);
4852     closedir($D);
4853
4854     foreach my $pciid (@devs) {
4855         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4856
4857         # pci bridges, switches or root ports are not supported
4858         # they have a pci_bus subdirectory so skip them
4859         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4860
4861         my $info = pci_device_info($1);
4862         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4863     }
4864
4865     return 1;
4866 }
4867
4868 sub print_pci_addr {
4869     my ($id, $bridges) = @_;
4870
4871     my $res = '';
4872     my $devices = {
4873         piix3 => { bus => 0, addr => 1 },
4874         #addr2 : first videocard
4875         balloon0 => { bus => 0, addr => 3 },
4876         watchdog => { bus => 0, addr => 4 },
4877         scsihw0 => { bus => 0, addr => 5 },
4878         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4879         scsihw1 => { bus => 0, addr => 6 },
4880         ahci0 => { bus => 0, addr => 7 },
4881         qga0 => { bus => 0, addr => 8 },
4882         spice => { bus => 0, addr => 9 },
4883         virtio0 => { bus => 0, addr => 10 },
4884         virtio1 => { bus => 0, addr => 11 },
4885         virtio2 => { bus => 0, addr => 12 },
4886         virtio3 => { bus => 0, addr => 13 },
4887         virtio4 => { bus => 0, addr => 14 },
4888         virtio5 => { bus => 0, addr => 15 },
4889         hostpci0 => { bus => 0, addr => 16 },
4890         hostpci1 => { bus => 0, addr => 17 },
4891         net0 => { bus => 0, addr => 18 },
4892         net1 => { bus => 0, addr => 19 },
4893         net2 => { bus => 0, addr => 20 },
4894         net3 => { bus => 0, addr => 21 },
4895         net4 => { bus => 0, addr => 22 },
4896         net5 => { bus => 0, addr => 23 },
4897         vga1 => { bus => 0, addr => 24 },
4898         vga2 => { bus => 0, addr => 25 },
4899         vga3 => { bus => 0, addr => 26 },
4900         hostpci2 => { bus => 0, addr => 27 },
4901         hostpci3 => { bus => 0, addr => 28 },
4902         #addr29 : usb-host (pve-usb.cfg)
4903         'pci.1' => { bus => 0, addr => 30 },
4904         'pci.2' => { bus => 0, addr => 31 },
4905         'net6' => { bus => 1, addr => 1 },
4906         'net7' => { bus => 1, addr => 2 },
4907         'net8' => { bus => 1, addr => 3 },
4908         'net9' => { bus => 1, addr => 4 },
4909         'net10' => { bus => 1, addr => 5 },
4910         'net11' => { bus => 1, addr => 6 },
4911         'net12' => { bus => 1, addr => 7 },
4912         'net13' => { bus => 1, addr => 8 },
4913         'net14' => { bus => 1, addr => 9 },
4914         'net15' => { bus => 1, addr => 10 },
4915         'net16' => { bus => 1, addr => 11 },
4916         'net17' => { bus => 1, addr => 12 },
4917         'net18' => { bus => 1, addr => 13 },
4918         'net19' => { bus => 1, addr => 14 },
4919         'net20' => { bus => 1, addr => 15 },
4920         'net21' => { bus => 1, addr => 16 },
4921         'net22' => { bus => 1, addr => 17 },
4922         'net23' => { bus => 1, addr => 18 },
4923         'net24' => { bus => 1, addr => 19 },
4924         'net25' => { bus => 1, addr => 20 },
4925         'net26' => { bus => 1, addr => 21 },
4926         'net27' => { bus => 1, addr => 22 },
4927         'net28' => { bus => 1, addr => 23 },
4928         'net29' => { bus => 1, addr => 24 },
4929         'net30' => { bus => 1, addr => 25 },
4930         'net31' => { bus => 1, addr => 26 },
4931         'virtio6' => { bus => 2, addr => 1 },
4932         'virtio7' => { bus => 2, addr => 2 },
4933         'virtio8' => { bus => 2, addr => 3 },
4934         'virtio9' => { bus => 2, addr => 4 },
4935         'virtio10' => { bus => 2, addr => 5 },
4936         'virtio11' => { bus => 2, addr => 6 },
4937         'virtio12' => { bus => 2, addr => 7 },
4938         'virtio13' => { bus => 2, addr => 8 },
4939         'virtio14' => { bus => 2, addr => 9 },
4940         'virtio15' => { bus => 2, addr => 10 },
4941         'virtioscsi0' => { bus => 3, addr => 1 },
4942         'virtioscsi1' => { bus => 3, addr => 2 },
4943         'virtioscsi2' => { bus => 3, addr => 3 },
4944         'virtioscsi3' => { bus => 3, addr => 4 },
4945         'virtioscsi4' => { bus => 3, addr => 5 },
4946         'virtioscsi5' => { bus => 3, addr => 6 },
4947         'virtioscsi6' => { bus => 3, addr => 7 },
4948         'virtioscsi7' => { bus => 3, addr => 8 },
4949         'virtioscsi8' => { bus => 3, addr => 9 },
4950         'virtioscsi9' => { bus => 3, addr => 10 },
4951         'virtioscsi10' => { bus => 3, addr => 11 },
4952         'virtioscsi11' => { bus => 3, addr => 12 },
4953         'virtioscsi12' => { bus => 3, addr => 13 },
4954         'virtioscsi13' => { bus => 3, addr => 14 },
4955         'virtioscsi14' => { bus => 3, addr => 15 },
4956         'virtioscsi15' => { bus => 3, addr => 16 },
4957         'virtioscsi16' => { bus => 3, addr => 17 },
4958         'virtioscsi17' => { bus => 3, addr => 18 },
4959         'virtioscsi18' => { bus => 3, addr => 19 },
4960         'virtioscsi19' => { bus => 3, addr => 20 },
4961         'virtioscsi20' => { bus => 3, addr => 21 },
4962         'virtioscsi21' => { bus => 3, addr => 22 },
4963         'virtioscsi22' => { bus => 3, addr => 23 },
4964         'virtioscsi23' => { bus => 3, addr => 24 },
4965         'virtioscsi24' => { bus => 3, addr => 25 },
4966         'virtioscsi25' => { bus => 3, addr => 26 },
4967         'virtioscsi26' => { bus => 3, addr => 27 },
4968         'virtioscsi27' => { bus => 3, addr => 28 },
4969         'virtioscsi28' => { bus => 3, addr => 29 },
4970         'virtioscsi29' => { bus => 3, addr => 30 },
4971         'virtioscsi30' => { bus => 3, addr => 31 },
4972
4973     };
4974
4975     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4976            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4977            my $bus = $devices->{$id}->{bus};
4978            $res = ",bus=pci.$bus,addr=$addr";
4979            $bridges->{$bus} = 1 if $bridges;
4980     }
4981     return $res;
4982
4983 }
4984
4985 sub print_pcie_addr {
4986     my ($id) = @_;
4987
4988     my $res = '';
4989     my $devices = {
4990         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4991         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4992         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4993         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4994     };
4995
4996     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4997            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4998            my $bus = $devices->{$id}->{bus};
4999            $res = ",bus=$bus,addr=$addr";
5000     }
5001     return $res;
5002
5003 }
5004
5005 # vzdump restore implementaion
5006
5007 sub tar_archive_read_firstfile {
5008     my $archive = shift;
5009
5010     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5011
5012     # try to detect archive type first
5013     my $pid = open (TMP, "tar tf '$archive'|") ||
5014         die "unable to open file '$archive'\n";
5015     my $firstfile = <TMP>;
5016     kill 15, $pid;
5017     close TMP;
5018
5019     die "ERROR: archive contaions no data\n" if !$firstfile;
5020     chomp $firstfile;
5021
5022     return $firstfile;
5023 }
5024
5025 sub tar_restore_cleanup {
5026     my ($storecfg, $statfile) = @_;
5027
5028     print STDERR "starting cleanup\n";
5029
5030     if (my $fd = IO::File->new($statfile, "r")) {
5031         while (defined(my $line = <$fd>)) {
5032             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5033                 my $volid = $2;
5034                 eval {
5035                     if ($volid =~ m|^/|) {
5036                         unlink $volid || die 'unlink failed\n';
5037                     } else {
5038                         PVE::Storage::vdisk_free($storecfg, $volid);
5039                     }
5040                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5041                 };
5042                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5043             } else {
5044                 print STDERR "unable to parse line in statfile - $line";
5045             }
5046         }
5047         $fd->close();
5048     }
5049 }
5050
5051 sub restore_archive {
5052     my ($archive, $vmid, $user, $opts) = @_;
5053
5054     my $format = $opts->{format};
5055     my $comp;
5056
5057     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5058         $format = 'tar' if !$format;
5059         $comp = 'gzip';
5060     } elsif ($archive =~ m/\.tar$/) {
5061         $format = 'tar' if !$format;
5062     } elsif ($archive =~ m/.tar.lzo$/) {
5063         $format = 'tar' if !$format;
5064         $comp = 'lzop';
5065     } elsif ($archive =~ m/\.vma$/) {
5066         $format = 'vma' if !$format;
5067     } elsif ($archive =~ m/\.vma\.gz$/) {
5068         $format = 'vma' if !$format;
5069         $comp = 'gzip';
5070     } elsif ($archive =~ m/\.vma\.lzo$/) {
5071         $format = 'vma' if !$format;
5072         $comp = 'lzop';
5073     } else {
5074         $format = 'vma' if !$format; # default
5075     }
5076
5077     # try to detect archive format
5078     if ($format eq 'tar') {
5079         return restore_tar_archive($archive, $vmid, $user, $opts);
5080     } else {
5081         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5082     }
5083 }
5084
5085 sub restore_update_config_line {
5086     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5087
5088     return if $line =~ m/^\#qmdump\#/;
5089     return if $line =~ m/^\#vzdump\#/;
5090     return if $line =~ m/^lock:/;
5091     return if $line =~ m/^unused\d+:/;
5092     return if $line =~ m/^parent:/;
5093     return if $line =~ m/^template:/; # restored VM is never a template
5094
5095     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5096         # try to convert old 1.X settings
5097         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5098         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5099             my ($model, $macaddr) = split(/\=/, $devconfig);
5100             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5101             my $net = {
5102                 model => $model,
5103                 bridge => "vmbr$ind",
5104                 macaddr => $macaddr,
5105             };
5106             my $netstr = print_net($net);
5107
5108             print $outfd "net$cookie->{netcount}: $netstr\n";
5109             $cookie->{netcount}++;
5110         }
5111     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5112         my ($id, $netstr) = ($1, $2);
5113         my $net = parse_net($netstr);
5114         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5115         $netstr = print_net($net);
5116         print $outfd "$id: $netstr\n";
5117     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5118         my $virtdev = $1;
5119         my $value = $3;
5120         if ($line =~ m/backup=no/) {
5121             print $outfd "#$line";
5122         } elsif ($virtdev && $map->{$virtdev}) {
5123             my $di = parse_drive($virtdev, $value);
5124             delete $di->{format}; # format can change on restore
5125             $di->{file} = $map->{$virtdev};
5126             $value = print_drive($vmid, $di);
5127             print $outfd "$virtdev: $value\n";
5128         } else {
5129             print $outfd $line;
5130         }
5131     } else {
5132         print $outfd $line;
5133     }
5134 }
5135
5136 sub scan_volids {
5137     my ($cfg, $vmid) = @_;
5138
5139     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5140
5141     my $volid_hash = {};
5142     foreach my $storeid (keys %$info) {
5143         foreach my $item (@{$info->{$storeid}}) {
5144             next if !($item->{volid} && $item->{size});
5145             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5146             $volid_hash->{$item->{volid}} = $item;
5147         }
5148     }
5149
5150     return $volid_hash;
5151 }
5152
5153 sub is_volume_in_use {
5154     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5155
5156     my $path = PVE::Storage::path($storecfg, $volid);
5157
5158     my $scan_config = sub {
5159         my ($cref, $snapname) = @_;
5160
5161         foreach my $key (keys %$cref) {
5162             my $value = $cref->{$key};
5163             if (valid_drivename($key)) {
5164                 next if $skip_drive && $key eq $skip_drive;
5165                 my $drive = parse_drive($key, $value);
5166                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5167                 return 1 if $volid eq $drive->{file};
5168                 if ($drive->{file} =~ m!^/!) {
5169                     return 1 if $drive->{file} eq $path;
5170                 } else {
5171                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5172                     next if !$storeid;
5173                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5174                     next if !$scfg;
5175                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5176                 }
5177             }
5178         }
5179
5180         return 0;
5181     };
5182
5183     return 1 if &$scan_config($conf);
5184
5185     undef $skip_drive;
5186
5187     foreach my $snapname (keys %{$conf->{snapshots}}) {
5188         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5189     }
5190
5191     return 0;
5192 }
5193
5194 sub update_disksize {
5195     my ($vmid, $conf, $volid_hash) = @_;
5196
5197     my $changes;
5198
5199     my $used = {};
5200
5201     # Note: it is allowed to define multiple storages with same path (alias), so
5202     # we need to check both 'volid' and real 'path' (two different volid can point
5203     # to the same path).
5204
5205     my $usedpath = {};
5206
5207     # update size info
5208     foreach my $opt (keys %$conf) {
5209         if (valid_drivename($opt)) {
5210             my $drive = parse_drive($opt, $conf->{$opt});
5211             my $volid = $drive->{file};
5212             next if !$volid;
5213
5214             $used->{$volid} = 1;
5215             if ($volid_hash->{$volid} &&
5216                 (my $path = $volid_hash->{$volid}->{path})) {
5217                 $usedpath->{$path} = 1;
5218             }
5219
5220             next if drive_is_cdrom($drive);
5221             next if !$volid_hash->{$volid};
5222
5223             $drive->{size} = $volid_hash->{$volid}->{size};
5224             my $new = print_drive($vmid, $drive);
5225             if ($new ne $conf->{$opt}) {
5226                 $changes = 1;
5227                 $conf->{$opt} = $new;
5228             }
5229         }
5230     }
5231
5232     # remove 'unusedX' entry if volume is used
5233     foreach my $opt (keys %$conf) {
5234         next if $opt !~ m/^unused\d+$/;
5235         my $volid = $conf->{$opt};
5236         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5237         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5238             $changes = 1;
5239             delete $conf->{$opt};
5240         }
5241     }
5242
5243     foreach my $volid (sort keys %$volid_hash) {
5244         next if $volid =~ m/vm-$vmid-state-/;
5245         next if $used->{$volid};
5246         my $path = $volid_hash->{$volid}->{path};
5247         next if !$path; # just to be sure
5248         next if $usedpath->{$path};
5249         $changes = 1;
5250         add_unused_volume($conf, $volid);
5251         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5252     }
5253
5254     return $changes;
5255 }
5256
5257 sub rescan {
5258     my ($vmid, $nolock) = @_;
5259
5260     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5261
5262     my $volid_hash = scan_volids($cfg, $vmid);
5263
5264     my $updatefn =  sub {
5265         my ($vmid) = @_;
5266
5267         my $conf = load_config($vmid);
5268
5269         check_lock($conf);
5270
5271         my $vm_volids = {};
5272         foreach my $volid (keys %$volid_hash) {
5273             my $info = $volid_hash->{$volid};
5274             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5275         }
5276
5277         my $changes = update_disksize($vmid, $conf, $vm_volids);
5278
5279         update_config_nolock($vmid, $conf, 1) if $changes;
5280     };
5281
5282     if (defined($vmid)) {
5283         if ($nolock) {
5284             &$updatefn($vmid);
5285         } else {
5286             lock_config($vmid, $updatefn, $vmid);
5287         }
5288     } else {
5289         my $vmlist = config_list();
5290         foreach my $vmid (keys %$vmlist) {
5291             if ($nolock) {
5292                 &$updatefn($vmid);
5293             } else {
5294                 lock_config($vmid, $updatefn, $vmid);
5295             }
5296         }
5297     }
5298 }
5299
5300 sub restore_vma_archive {
5301     my ($archive, $vmid, $user, $opts, $comp) = @_;
5302
5303     my $input = $archive eq '-' ? "<&STDIN" : undef;
5304     my $readfrom = $archive;
5305
5306     my $uncomp = '';
5307     if ($comp) {
5308         $readfrom = '-';
5309         my $qarchive = PVE::Tools::shellquote($archive);
5310         if ($comp eq 'gzip') {
5311             $uncomp = "zcat $qarchive|";
5312         } elsif ($comp eq 'lzop') {
5313             $uncomp = "lzop -d -c $qarchive|";
5314         } else {
5315             die "unknown compression method '$comp'\n";
5316         }
5317
5318     }
5319
5320     my $tmpdir = "/var/tmp/vzdumptmp$$";
5321     rmtree $tmpdir;
5322
5323     # disable interrupts (always do cleanups)
5324     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5325         warn "got interrupt - ignored\n";
5326     };
5327
5328     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5329     POSIX::mkfifo($mapfifo, 0600);
5330     my $fifofh;
5331
5332     my $openfifo = sub {
5333         open($fifofh, '>', $mapfifo) || die $!;
5334     };
5335
5336     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5337
5338     my $oldtimeout;
5339     my $timeout = 5;
5340
5341     my $devinfo = {};
5342
5343     my $rpcenv = PVE::RPCEnvironment::get();
5344
5345     my $conffile = config_file($vmid);
5346     my $tmpfn = "$conffile.$$.tmp";
5347
5348     # Note: $oldconf is undef if VM does not exists
5349     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5350
5351     my $print_devmap = sub {
5352         my $virtdev_hash = {};
5353
5354         my $cfgfn = "$tmpdir/qemu-server.conf";
5355
5356         # we can read the config - that is already extracted
5357         my $fh = IO::File->new($cfgfn, "r") ||
5358             "unable to read qemu-server.conf - $!\n";
5359
5360         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5361         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5362             if -f $fwcfgfn;
5363
5364         while (defined(my $line = <$fh>)) {
5365             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5366                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5367                 die "archive does not contain data for drive '$virtdev'\n"
5368                     if !$devinfo->{$devname};
5369                 if (defined($opts->{storage})) {
5370                     $storeid = $opts->{storage} || 'local';
5371                 } elsif (!$storeid) {
5372                     $storeid = 'local';
5373                 }
5374                 $format = 'raw' if !$format;
5375                 $devinfo->{$devname}->{devname} = $devname;
5376                 $devinfo->{$devname}->{virtdev} = $virtdev;
5377                 $devinfo->{$devname}->{format} = $format;
5378                 $devinfo->{$devname}->{storeid} = $storeid;
5379
5380                 # check permission on storage
5381                 my $pool = $opts->{pool}; # todo: do we need that?
5382                 if ($user ne 'root@pam') {
5383                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5384                 }
5385
5386                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5387             }
5388         }
5389
5390         foreach my $devname (keys %$devinfo) {
5391             die "found no device mapping information for device '$devname'\n"
5392                 if !$devinfo->{$devname}->{virtdev};
5393         }
5394
5395         my $cfg = cfs_read_file('storage.cfg');
5396
5397         # create empty/temp config
5398         if ($oldconf) {
5399             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5400             foreach_drive($oldconf, sub {
5401                 my ($ds, $drive) = @_;
5402
5403                 return if drive_is_cdrom($drive);
5404
5405                 my $volid = $drive->{file};
5406
5407                 return if !$volid || $volid =~ m|^/|;
5408
5409                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5410                 return if !$path || !$owner || ($owner != $vmid);
5411
5412                 # Note: only delete disk we want to restore
5413                 # other volumes will become unused
5414                 if ($virtdev_hash->{$ds}) {
5415                     PVE::Storage::vdisk_free($cfg, $volid);
5416                 }
5417             });
5418         }
5419
5420         my $map = {};
5421         foreach my $virtdev (sort keys %$virtdev_hash) {
5422             my $d = $virtdev_hash->{$virtdev};
5423             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5424             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5425
5426             # test if requested format is supported
5427             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5428             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5429             $d->{format} = $defFormat if !$supported;
5430
5431             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5432                                                   $d->{format}, undef, $alloc_size);
5433             print STDERR "new volume ID is '$volid'\n";
5434             $d->{volid} = $volid;
5435             my $path = PVE::Storage::path($cfg, $volid);
5436
5437             PVE::Storage::activate_volumes($cfg,[$volid]);
5438
5439             my $write_zeros = 1;
5440             # fixme: what other storages types initialize volumes with zero?
5441             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5442                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5443                 $write_zeros = 0;
5444             }
5445
5446             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5447
5448             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5449             $map->{$virtdev} = $volid;
5450         }
5451
5452         $fh->seek(0, 0) || die "seek failed - $!\n";
5453
5454         my $outfd = new IO::File ($tmpfn, "w") ||
5455             die "unable to write config for VM $vmid\n";
5456
5457         my $cookie = { netcount => 0 };
5458         while (defined(my $line = <$fh>)) {
5459             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5460         }
5461
5462         $fh->close();
5463         $outfd->close();
5464     };
5465
5466     eval {
5467         # enable interrupts
5468         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5469             die "interrupted by signal\n";
5470         };
5471         local $SIG{ALRM} = sub { die "got timeout\n"; };
5472
5473         $oldtimeout = alarm($timeout);
5474
5475         my $parser = sub {
5476             my $line = shift;
5477
5478             print "$line\n";
5479
5480             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5481                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5482                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5483             } elsif ($line =~ m/^CTIME: /) {
5484                 # we correctly received the vma config, so we can disable
5485                 # the timeout now for disk allocation (set to 10 minutes, so
5486                 # that we always timeout if something goes wrong)
5487                 alarm(600);
5488                 &$print_devmap();
5489                 print $fifofh "done\n";
5490                 my $tmp = $oldtimeout || 0;
5491                 $oldtimeout = undef;
5492                 alarm($tmp);
5493                 close($fifofh);
5494             }
5495         };
5496
5497         print "restore vma archive: $cmd\n";
5498         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5499     };
5500     my $err = $@;
5501
5502     alarm($oldtimeout) if $oldtimeout;
5503
5504     my $vollist = [];
5505     foreach my $devname (keys %$devinfo) {
5506         my $volid = $devinfo->{$devname}->{volid};
5507         push @$vollist, $volid if $volid;
5508     }
5509
5510     my $cfg = cfs_read_file('storage.cfg');
5511     PVE::Storage::deactivate_volumes($cfg, $vollist);
5512
5513     unlink $mapfifo;
5514
5515     if ($err) {
5516         rmtree $tmpdir;
5517         unlink $tmpfn;
5518
5519         foreach my $devname (keys %$devinfo) {
5520             my $volid = $devinfo->{$devname}->{volid};
5521             next if !$volid;
5522             eval {
5523                 if ($volid =~ m|^/|) {
5524                     unlink $volid || die 'unlink failed\n';
5525                 } else {
5526                     PVE::Storage::vdisk_free($cfg, $volid);
5527                 }
5528                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5529             };
5530             print STDERR "unable to cleanup '$volid' - $@" if $@;
5531         }
5532         die $err;
5533     }
5534
5535     rmtree $tmpdir;
5536
5537     rename($tmpfn, $conffile) ||
5538         die "unable to commit configuration file '$conffile'\n";
5539
5540     PVE::Cluster::cfs_update(); # make sure we read new file
5541
5542     eval { rescan($vmid, 1); };
5543     warn $@ if $@;
5544 }
5545
5546 sub restore_tar_archive {
5547     my ($archive, $vmid, $user, $opts) = @_;
5548
5549     if ($archive ne '-') {
5550         my $firstfile = tar_archive_read_firstfile($archive);
5551         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5552             if $firstfile ne 'qemu-server.conf';
5553     }
5554
5555     my $storecfg = cfs_read_file('storage.cfg');
5556
5557     # destroy existing data - keep empty config
5558     my $vmcfgfn = config_file($vmid);
5559     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5560
5561     my $tocmd = "/usr/lib/qemu-server/qmextract";
5562
5563     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5564     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5565     $tocmd .= ' --prealloc' if $opts->{prealloc};
5566     $tocmd .= ' --info' if $opts->{info};
5567
5568     # tar option "xf" does not autodetect compression when read from STDIN,
5569     # so we pipe to zcat
5570     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5571         PVE::Tools::shellquote("--to-command=$tocmd");
5572
5573     my $tmpdir = "/var/tmp/vzdumptmp$$";
5574     mkpath $tmpdir;
5575
5576     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5577     local $ENV{VZDUMP_VMID} = $vmid;
5578     local $ENV{VZDUMP_USER} = $user;
5579
5580     my $conffile = config_file($vmid);
5581     my $tmpfn = "$conffile.$$.tmp";
5582
5583     # disable interrupts (always do cleanups)
5584     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5585         print STDERR "got interrupt - ignored\n";
5586     };
5587
5588     eval {
5589         # enable interrupts
5590         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5591             die "interrupted by signal\n";
5592         };
5593
5594         if ($archive eq '-') {
5595             print "extracting archive from STDIN\n";
5596             run_command($cmd, input => "<&STDIN");
5597         } else {
5598             print "extracting archive '$archive'\n";
5599             run_command($cmd);
5600         }
5601
5602         return if $opts->{info};
5603
5604         # read new mapping
5605         my $map = {};
5606         my $statfile = "$tmpdir/qmrestore.stat";
5607         if (my $fd = IO::File->new($statfile, "r")) {
5608             while (defined (my $line = <$fd>)) {
5609                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5610                     $map->{$1} = $2 if $1;
5611                 } else {
5612                     print STDERR "unable to parse line in statfile - $line\n";
5613                 }
5614             }
5615             $fd->close();
5616         }
5617
5618         my $confsrc = "$tmpdir/qemu-server.conf";
5619
5620         my $srcfd = new IO::File($confsrc, "r") ||
5621             die "unable to open file '$confsrc'\n";
5622
5623         my $outfd = new IO::File ($tmpfn, "w") ||
5624             die "unable to write config for VM $vmid\n";
5625
5626         my $cookie = { netcount => 0 };
5627         while (defined (my $line = <$srcfd>)) {
5628             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5629         }
5630
5631         $srcfd->close();
5632         $outfd->close();
5633     };
5634     my $err = $@;
5635
5636     if ($err) {
5637
5638         unlink $tmpfn;
5639
5640         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5641
5642         die $err;
5643     }
5644
5645     rmtree $tmpdir;
5646
5647     rename $tmpfn, $conffile ||
5648         die "unable to commit configuration file '$conffile'\n";
5649
5650     PVE::Cluster::cfs_update(); # make sure we read new file
5651
5652     eval { rescan($vmid, 1); };
5653     warn $@ if $@;
5654 };
5655
5656
5657 # Internal snapshots
5658
5659 # NOTE: Snapshot create/delete involves several non-atomic
5660 # action, and can take a long time.
5661 # So we try to avoid locking the file and use 'lock' variable
5662 # inside the config file instead.
5663
5664 my $snapshot_copy_config = sub {
5665     my ($source, $dest) = @_;
5666
5667     foreach my $k (keys %$source) {
5668         next if $k eq 'snapshots';
5669         next if $k eq 'snapstate';
5670         next if $k eq 'snaptime';
5671         next if $k eq 'vmstate';
5672         next if $k eq 'lock';
5673         next if $k eq 'digest';
5674         next if $k eq 'description';
5675         next if $k =~ m/^unused\d+$/;
5676
5677         $dest->{$k} = $source->{$k};
5678     }
5679 };
5680
5681 my $snapshot_apply_config = sub {
5682     my ($conf, $snap) = @_;
5683
5684     # copy snapshot list
5685     my $newconf = {
5686         snapshots => $conf->{snapshots},
5687     };
5688
5689     # keep description and list of unused disks
5690     foreach my $k (keys %$conf) {
5691         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5692         $newconf->{$k} = $conf->{$k};
5693     }
5694
5695     &$snapshot_copy_config($snap, $newconf);
5696
5697     return $newconf;
5698 };
5699
5700 sub foreach_writable_storage {
5701     my ($conf, $func) = @_;
5702
5703     my $sidhash = {};
5704
5705     foreach my $ds (keys %$conf) {
5706         next if !valid_drivename($ds);
5707
5708         my $drive = parse_drive($ds, $conf->{$ds});
5709         next if !$drive;
5710         next if drive_is_cdrom($drive);
5711
5712         my $volid = $drive->{file};
5713
5714         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5715         $sidhash->{$sid} = $sid if $sid;
5716     }
5717
5718     foreach my $sid (sort keys %$sidhash) {
5719         &$func($sid);
5720     }
5721 }
5722
5723 my $alloc_vmstate_volid = sub {
5724     my ($storecfg, $vmid, $conf, $snapname) = @_;
5725
5726     # Note: we try to be smart when selecting a $target storage
5727
5728     my $target;
5729
5730     # search shared storage first
5731     foreach_writable_storage($conf, sub {
5732         my ($sid) = @_;
5733         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5734         return if !$scfg->{shared};
5735
5736         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5737     });
5738
5739     if (!$target) {
5740         # now search local storage
5741         foreach_writable_storage($conf, sub {
5742             my ($sid) = @_;
5743             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5744             return if $scfg->{shared};
5745
5746             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5747         });
5748     }
5749
5750     $target = 'local' if !$target;
5751
5752     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5753     # we abort live save after $conf->{memory}, so we need at max twice that space
5754     my $size = $conf->{memory}*2 + $driver_state_size;
5755
5756     my $name = "vm-$vmid-state-$snapname";
5757     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5758     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5759     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5760
5761     return $volid;
5762 };
5763
5764 my $snapshot_prepare = sub {
5765     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5766
5767     my $snap;
5768
5769     my $updatefn =  sub {
5770
5771         my $conf = load_config($vmid);
5772
5773         die "you can't take a snapshot if it's a template\n"
5774             if is_template($conf);
5775
5776         check_lock($conf);
5777
5778         $conf->{lock} = 'snapshot';
5779
5780         die "snapshot name '$snapname' already used\n"
5781             if defined($conf->{snapshots}->{$snapname});
5782
5783         my $storecfg = PVE::Storage::config();
5784         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5785
5786         $snap = $conf->{snapshots}->{$snapname} = {};
5787
5788         if ($save_vmstate && check_running($vmid)) {
5789             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5790         }
5791
5792         &$snapshot_copy_config($conf, $snap);
5793
5794         $snap->{snapstate} = "prepare";
5795         $snap->{snaptime} = time();
5796         $snap->{description} = $comment if $comment;
5797
5798         # always overwrite machine if we save vmstate. This makes sure we
5799         # can restore it later using correct machine type
5800         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5801
5802         update_config_nolock($vmid, $conf, 1);
5803     };
5804
5805     lock_config($vmid, $updatefn);
5806
5807     return $snap;
5808 };
5809
5810 my $snapshot_commit = sub {
5811     my ($vmid, $snapname) = @_;
5812
5813     my $updatefn = sub {
5814
5815         my $conf = load_config($vmid);
5816
5817         die "missing snapshot lock\n"
5818             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5819
5820         my $has_machine_config = defined($conf->{machine});
5821
5822         my $snap = $conf->{snapshots}->{$snapname};
5823
5824         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5825
5826         die "wrong snapshot state\n"
5827             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5828
5829         delete $snap->{snapstate};
5830         delete $conf->{lock};
5831
5832         my $newconf = &$snapshot_apply_config($conf, $snap);
5833
5834         delete $newconf->{machine} if !$has_machine_config;
5835
5836         $newconf->{parent} = $snapname;
5837
5838         update_config_nolock($vmid, $newconf, 1);
5839     };
5840
5841     lock_config($vmid, $updatefn);
5842 };
5843
5844 sub snapshot_rollback {
5845     my ($vmid, $snapname) = @_;
5846
5847     my $prepare = 1;
5848
5849     my $storecfg = PVE::Storage::config();
5850
5851     my $conf = load_config($vmid);
5852
5853     my $get_snapshot_config = sub {
5854
5855         die "you can't rollback if vm is a template\n" if is_template($conf);
5856
5857         my $res = $conf->{snapshots}->{$snapname};
5858
5859         die "snapshot '$snapname' does not exist\n" if !defined($res);
5860
5861         return $res;
5862     };
5863
5864     my $snap = &$get_snapshot_config();
5865
5866     foreach_drive($snap, sub {
5867         my ($ds, $drive) = @_;
5868
5869         return if drive_is_cdrom($drive);
5870
5871         my $volid = $drive->{file};
5872
5873         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5874     });
5875
5876     my $updatefn = sub {
5877
5878         $conf = load_config($vmid);
5879
5880         $snap = &$get_snapshot_config();
5881
5882         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5883             if $snap->{snapstate};
5884
5885         if ($prepare) {
5886             check_lock($conf);
5887             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5888         }
5889
5890         die "unable to rollback vm $vmid: vm is running\n"
5891             if check_running($vmid);
5892
5893         if ($prepare) {
5894             $conf->{lock} = 'rollback';
5895         } else {
5896             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5897             delete $conf->{lock};
5898         }
5899
5900         my $forcemachine;
5901
5902         if (!$prepare) {
5903             my $has_machine_config = defined($conf->{machine});
5904
5905             # copy snapshot config to current config
5906             $conf = &$snapshot_apply_config($conf, $snap);
5907             $conf->{parent} = $snapname;
5908
5909             # Note: old code did not store 'machine', so we try to be smart
5910             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5911             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5912             # we remove the 'machine' configuration if not explicitly specified
5913             # in the original config.
5914             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5915         }
5916
5917         update_config_nolock($vmid, $conf, 1);
5918
5919         if (!$prepare && $snap->{vmstate}) {
5920             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5921             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5922         }
5923     };
5924
5925     lock_config($vmid, $updatefn);
5926
5927     foreach_drive($snap, sub {
5928         my ($ds, $drive) = @_;
5929
5930         return if drive_is_cdrom($drive);
5931
5932         my $volid = $drive->{file};
5933         my $device = "drive-$ds";
5934
5935         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5936     });
5937
5938     $prepare = 0;
5939     lock_config($vmid, $updatefn);
5940 }
5941
5942 my $savevm_wait = sub {
5943     my ($vmid) = @_;
5944
5945     for(;;) {
5946         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5947         if (!$stat->{status}) {
5948             die "savevm not active\n";
5949         } elsif ($stat->{status} eq 'active') {
5950             sleep(1);
5951             next;
5952         } elsif ($stat->{status} eq 'completed') {
5953             last;
5954         } else {
5955             die "query-savevm returned status '$stat->{status}'\n";
5956         }
5957     }
5958 };
5959
5960 sub do_snapshots_with_qemu {
5961     my ($storecfg, $volid) = @_;
5962
5963     my $storage_name = PVE::Storage::parse_volume_id($volid);
5964
5965     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5966         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5967         return 1;
5968     }
5969
5970     if ($volid =~ m/\.(qcow2|qed)$/){
5971         return 1;
5972     }
5973
5974     return undef;
5975 }
5976
5977 sub snapshot_create {
5978     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5979
5980     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5981
5982     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5983
5984     my $config = load_config($vmid);
5985
5986     my $running = check_running($vmid);
5987
5988     my $freezefs = $running && $config->{agent};
5989     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5990
5991     my $drivehash = {};
5992
5993     if ($freezefs) {
5994         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5995         warn "guest-fsfreeze-freeze problems - $@" if $@;
5996     }
5997
5998     eval {
5999         # create internal snapshots of all drives
6000
6001         my $storecfg = PVE::Storage::config();
6002
6003         if ($running) {
6004             if ($snap->{vmstate}) {
6005                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
6006                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
6007                 &$savevm_wait($vmid);
6008             } else {
6009                 vm_mon_cmd($vmid, "savevm-start");
6010             }
6011         };
6012
6013         foreach_drive($snap, sub {
6014             my ($ds, $drive) = @_;
6015
6016             return if drive_is_cdrom($drive);
6017
6018             my $volid = $drive->{file};
6019             my $device = "drive-$ds";
6020
6021             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6022             $drivehash->{$ds} = 1;
6023        });
6024     };
6025     my $err = $@;
6026
6027     if ($running) {
6028         eval { vm_mon_cmd($vmid, "savevm-end")  };
6029         warn $@ if $@;
6030
6031         if ($freezefs) {
6032             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6033             warn "guest-fsfreeze-thaw problems - $@" if $@;
6034         }
6035
6036         # savevm-end is async, we need to wait
6037         for (;;) {
6038             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6039             if (!$stat->{bytes}) {
6040                 last;
6041             } else {
6042                 print "savevm not yet finished\n";
6043                 sleep(1);
6044                 next;
6045             }
6046         }
6047     }
6048
6049     if ($err) {
6050         warn "snapshot create failed: starting cleanup\n";
6051         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6052         warn $@ if $@;
6053         die $err;
6054     }
6055
6056     &$snapshot_commit($vmid, $snapname);
6057 }
6058
6059 # Note: $drivehash is only set when called from snapshot_create.
6060 sub snapshot_delete {
6061     my ($vmid, $snapname, $force, $drivehash) = @_;
6062
6063     my $prepare = 1;
6064
6065     my $snap;
6066     my $unused = [];
6067
6068     my $unlink_parent = sub {
6069         my ($confref, $new_parent) = @_;
6070
6071         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6072             if ($new_parent) {
6073                 $confref->{parent} = $new_parent;
6074             } else {
6075                 delete $confref->{parent};
6076             }
6077         }
6078     };
6079
6080     my $updatefn =  sub {
6081         my ($remove_drive) = @_;
6082
6083         my $conf = load_config($vmid);
6084
6085         if (!$drivehash) {
6086             check_lock($conf);
6087             die "you can't delete a snapshot if vm is a template\n"
6088                 if is_template($conf);
6089         }
6090
6091         $snap = $conf->{snapshots}->{$snapname};
6092
6093         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6094
6095         # remove parent refs
6096         if (!$prepare) {
6097             &$unlink_parent($conf, $snap->{parent});
6098             foreach my $sn (keys %{$conf->{snapshots}}) {
6099                 next if $sn eq $snapname;
6100                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6101             }
6102         }
6103
6104         if ($remove_drive) {
6105             if ($remove_drive eq 'vmstate') {
6106                 delete $snap->{$remove_drive};
6107             } else {
6108                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6109                 my $volid = $drive->{file};
6110                 delete $snap->{$remove_drive};
6111                 add_unused_volume($conf, $volid);
6112             }
6113         }
6114
6115         if ($prepare) {
6116             $snap->{snapstate} = 'delete';
6117         } else {
6118             delete $conf->{snapshots}->{$snapname};
6119             delete $conf->{lock} if $drivehash;
6120             foreach my $volid (@$unused) {
6121                 add_unused_volume($conf, $volid);
6122             }
6123         }
6124
6125         update_config_nolock($vmid, $conf, 1);
6126     };
6127
6128     lock_config($vmid, $updatefn);
6129
6130     # now remove vmstate file
6131
6132     my $storecfg = PVE::Storage::config();
6133
6134     if ($snap->{vmstate}) {
6135         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6136         if (my $err = $@) {
6137             die $err if !$force;
6138             warn $err;
6139         }
6140         # save changes (remove vmstate from snapshot)
6141         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6142     };
6143
6144     # now remove all internal snapshots
6145     foreach_drive($snap, sub {
6146         my ($ds, $drive) = @_;
6147
6148         return if drive_is_cdrom($drive);
6149
6150         my $volid = $drive->{file};
6151         my $device = "drive-$ds";
6152
6153         if (!$drivehash || $drivehash->{$ds}) {
6154             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6155             if (my $err = $@) {
6156                 die $err if !$force;
6157                 warn $err;
6158             }
6159         }
6160
6161         # save changes (remove drive fron snapshot)
6162         lock_config($vmid, $updatefn, $ds) if !$force;
6163         push @$unused, $volid;
6164     });
6165
6166     # now cleanup config
6167     $prepare = 0;
6168     lock_config($vmid, $updatefn);
6169 }
6170
6171 sub has_feature {
6172     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6173
6174     my $err;
6175     foreach_drive($conf, sub {
6176         my ($ds, $drive) = @_;
6177
6178         return if drive_is_cdrom($drive);
6179         my $volid = $drive->{file};
6180         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6181     });
6182
6183     return $err ? 0 : 1;
6184 }
6185
6186 sub template_create {
6187     my ($vmid, $conf, $disk) = @_;
6188
6189     my $storecfg = PVE::Storage::config();
6190
6191     foreach_drive($conf, sub {
6192         my ($ds, $drive) = @_;
6193
6194         return if drive_is_cdrom($drive);
6195         return if $disk && $ds ne $disk;
6196
6197         my $volid = $drive->{file};
6198         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6199
6200         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6201         $drive->{file} = $voliddst;
6202         $conf->{$ds} = print_drive($vmid, $drive);
6203         update_config_nolock($vmid, $conf, 1);
6204     });
6205 }
6206
6207 sub is_template {
6208     my ($conf) = @_;
6209
6210     return 1 if defined $conf->{template} && $conf->{template} == 1;
6211 }
6212
6213 sub qemu_img_convert {
6214     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6215
6216     my $storecfg = PVE::Storage::config();
6217     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6218     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6219
6220     if ($src_storeid && $dst_storeid) {
6221
6222         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6223
6224         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6225         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6226
6227         my $src_format = qemu_img_format($src_scfg, $src_volname);
6228         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6229
6230         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6231         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6232
6233         my $cmd = [];
6234         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6235         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6236         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6237
6238         my $parser = sub {
6239             my $line = shift;
6240             if($line =~ m/\((\S+)\/100\%\)/){
6241                 my $percent = $1;
6242                 my $transferred = int($size * $percent / 100);
6243                 my $remaining = $size - $transferred;
6244
6245                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6246             }
6247
6248         };
6249
6250         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6251         my $err = $@;
6252         die "copy failed: $err" if $err;
6253     }
6254 }
6255
6256 sub qemu_img_format {
6257     my ($scfg, $volname) = @_;
6258
6259     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6260         return $1;
6261     } else {
6262         return "raw";
6263     }
6264 }
6265
6266 sub qemu_drive_mirror {
6267     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6268
6269     my $storecfg = PVE::Storage::config();
6270     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6271
6272     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6273
6274     my $format = qemu_img_format($dst_scfg, $dst_volname);
6275
6276     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6277
6278     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6279     $opts->{format} = $format if $format;
6280
6281     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6282
6283     eval {
6284     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6285         while (1) {
6286             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6287             my $stat = @$stats[0];
6288             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6289             die "error job is not mirroring" if $stat->{type} ne "mirror";
6290
6291             my $busy = $stat->{busy};
6292             my $ready = $stat->{ready};
6293
6294             if (my $total = $stat->{len}) {
6295                 my $transferred = $stat->{offset} || 0;
6296                 my $remaining = $total - $transferred;
6297                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6298
6299                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6300             }
6301
6302
6303             if ($stat->{ready} eq 'true') {
6304
6305                 last if $vmiddst != $vmid;
6306
6307                 # try to switch the disk if source and destination are on the same guest
6308                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6309                 last if !$@;
6310                 die $@ if $@ !~ m/cannot be completed/;
6311             }
6312             sleep 1;
6313         }
6314
6315
6316     };
6317     my $err = $@;
6318
6319     my $cancel_job = sub {
6320         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6321         while (1) {
6322             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6323             my $stat = @$stats[0];
6324             last if !$stat;
6325             sleep 1;
6326         }
6327     };
6328
6329     if ($err) {
6330         eval { &$cancel_job(); };
6331         die "mirroring error: $err";
6332     }
6333
6334     if ($vmiddst != $vmid) {
6335         # if we clone a disk for a new target vm, we don't switch the disk
6336         &$cancel_job(); # so we call block-job-cancel
6337     }
6338 }
6339
6340 sub clone_disk {
6341     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6342         $newvmid, $storage, $format, $full, $newvollist) = @_;
6343
6344     my $newvolid;
6345
6346     if (!$full) {
6347         print "create linked clone of drive $drivename ($drive->{file})\n";
6348         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6349         push @$newvollist, $newvolid;
6350     } else {
6351         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6352         $storeid = $storage if $storage;
6353
6354         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6355         if (!$format) {
6356             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6357             $format = qemu_img_format($scfg, $volname);
6358         }
6359
6360         # test if requested format is supported - else use default
6361         my $supported = grep { $_ eq $format } @$validFormats;
6362         $format = $defFormat if !$supported;
6363
6364         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6365
6366         print "create full clone of drive $drivename ($drive->{file})\n";
6367         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6368         push @$newvollist, $newvolid;
6369
6370         PVE::Storage::activate_volumes($storecfg, $newvollist);
6371
6372         if (!$running || $snapname) {
6373             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6374         } else {
6375             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6376         }
6377     }
6378
6379     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6380
6381     my $disk = $drive;
6382     $disk->{format} = undef;
6383     $disk->{file} = $newvolid;
6384     $disk->{size} = $size;
6385
6386     return $disk;
6387 }
6388
6389 # this only works if VM is running
6390 sub get_current_qemu_machine {
6391     my ($vmid) = @_;
6392
6393     my $cmd = { execute => 'query-machines', arguments => {} };
6394     my $res = vm_qmp_command($vmid, $cmd);
6395
6396     my ($current, $default);
6397     foreach my $e (@$res) {
6398         $default = $e->{name} if $e->{'is-default'};
6399         $current = $e->{name} if $e->{'is-current'};
6400     }
6401
6402     # fallback to the default machine if current is not supported by qemu
6403     return $current || $default || 'pc';
6404 }
6405
6406 sub qemu_machine_feature_enabled {
6407     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6408
6409     my $current_major;
6410     my $current_minor;
6411
6412     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6413
6414         $current_major = $3;
6415         $current_minor = $4;
6416
6417     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6418
6419         $current_major = $1;
6420         $current_minor = $2;
6421     }
6422
6423     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6424
6425
6426 }
6427
6428 sub qemu_machine_pxe {
6429     my ($vmid, $conf, $machine) = @_;
6430
6431     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6432
6433     foreach my $opt (keys %$conf) {
6434         next if $opt !~ m/^net(\d+)$/;
6435         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6436         next if !$net;
6437         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6438         return $machine.".pxe" if $romfile =~ m/pxe/;
6439         last;
6440     }
6441
6442 }
6443
6444 sub qemu_use_old_bios_files {
6445     my ($machine_type) = @_;
6446
6447     return if !$machine_type;
6448
6449     my $use_old_bios_files = undef;
6450
6451     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6452         $machine_type = $1;
6453         $use_old_bios_files = 1;
6454     } else {
6455         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6456         # load new efi bios files on migration. So this hack is required to allow
6457         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6458         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6459         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6460     }
6461
6462     return ($use_old_bios_files, $machine_type);
6463 }
6464
6465 sub lspci {
6466
6467     my $devices = {};
6468
6469     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6470             my (undef, $id, $function) = @_;
6471             my $res = { id => $id, function => $function};
6472             push @{$devices->{$id}}, $res;
6473     });
6474
6475     return $devices;
6476 }
6477
6478 sub vm_iothreads_list {
6479     my ($vmid) = @_;
6480
6481     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6482
6483     my $iothreads = {};
6484     foreach my $iothread (@$res) {
6485         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6486     }
6487
6488     return $iothreads;
6489 }
6490
6491 sub scsihw_infos {
6492     my ($conf, $drive) = @_;
6493
6494     my $maxdev = 0;
6495
6496     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6497         $maxdev = 7;
6498     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6499         $maxdev = 1;
6500     } else {
6501         $maxdev = 256;
6502     }
6503
6504     my $controller = int($drive->{index} / $maxdev);
6505     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6506
6507     return ($maxdev, $controller, $controller_prefix);
6508 }
6509
6510 # bash completion helper
6511
6512 sub complete_backup_archives {
6513     my ($cmdname, $pname, $cvalue) = @_;
6514
6515     my $cfg = PVE::Storage::config();
6516
6517     my $storeid;
6518
6519     if ($cvalue =~ m/^([^:]+):/) {
6520         $storeid = $1;
6521     }
6522
6523     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6524
6525     my $res = [];
6526     foreach my $id (keys %$data) {
6527         foreach my $item (@{$data->{$id}}) {
6528             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6529             push @$res, $item->{volid} if defined($item->{volid});
6530         }
6531     }
6532
6533     return $res;
6534 }
6535
6536 my $complete_vmid_full = sub {
6537     my ($running) = @_;
6538
6539     my $idlist = vmstatus();
6540
6541     my $res = [];
6542
6543     foreach my $id (keys %$idlist) {
6544         my $d = $idlist->{$id};
6545         if (defined($running)) {
6546             next if $d->{template};
6547             next if $running && $d->{status} ne 'running';
6548             next if !$running && $d->{status} eq 'running';
6549         }
6550         push @$res, $id;
6551
6552     }
6553     return $res;
6554 };
6555
6556 sub complete_vmid {
6557     return &$complete_vmid_full();
6558 }
6559
6560 sub complete_vmid_stopped {
6561     return &$complete_vmid_full(0);
6562 }
6563
6564 sub complete_vmid_running {
6565     return &$complete_vmid_full(1);
6566 }
6567
6568 sub complete_storage {
6569
6570     my $cfg = PVE::Storage::config();
6571     my $ids = $cfg->{ids};
6572
6573     my $res = [];
6574     foreach my $sid (keys %$ids) {
6575         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6576         next if !$ids->{$sid}->{content}->{images};
6577         push @$res, $sid;
6578     }
6579
6580     return $res;
6581 }
6582
6583 1;