PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 385         maxLength => 256,
 386         optional => 1,
 387     },
 388     protection => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 392         default => 0,
 393     },
 394     bios => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [ qw(seabios ovmf) ],
 398         description => "Select BIOS implementation.",
 399         default => 'seabios',
 400     },
 401 };
 402
 403 # what about other qemu settings ?
 404 #cpu => 'string',
 405 #machine => 'string',
 406 #fda => 'file',
 407 #fdb => 'file',
 408 #mtdblock => 'file',
 409 #sd => 'file',
 410 #pflash => 'file',
 411 #snapshot => 'bool',
 412 #bootp => 'file',
 413 ##tftp => 'dir',
 414 ##smb => 'dir',
 415 #kernel => 'file',
 416 #append => 'string',
 417 #initrd => 'file',
 418 ##soundhw => 'string',
 419
 420 while (my ($k, $v) = each %$confdesc) {
 421     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 422 }
 423
 424 my $MAX_IDE_DISKS = 4;
 425 my $MAX_SCSI_DISKS = 14;
 426 my $MAX_VIRTIO_DISKS = 16;
 427 my $MAX_SATA_DISKS = 6;
 428 my $MAX_USB_DEVICES = 5;
 429 my $MAX_NETS = 32;
 430 my $MAX_UNUSED_DISKS = 8;
 431 my $MAX_HOSTPCI_DEVICES = 4;
 432 my $MAX_SERIAL_PORTS = 4;
 433 my $MAX_PARALLEL_PORTS = 3;
 434 my $MAX_NUMA = 8;
 435 my $MAX_MEM = 4194304;
 436 my $STATICMEM = 1024;
 437
 438 my $numadesc = {
 439     optional => 1,
 440     type => 'string', format => 'pve-qm-numanode',
 441     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 442     description => "numa topology",
 443 };
 444 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 445
 446 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 447     $confdesc->{"numa$i"} = $numadesc;
 448 }
 449
 450 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 451                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 452                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 453 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 454
 455 my $netdesc = {
 456     optional => 1,
 457     type => 'string', format => 'pve-qm-net',
 458     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 459     description => <<EODESCR,
 460 Specify network devices.
 461
 462 MODEL is one of: $nic_model_list_txt
 463
 464 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 465 automatically generated if not specified.
 466
 467 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 468
 469 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 470
 471 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 472
 473 10.0.2.2   Gateway
 474 10.0.2.3   DNS Server
 475 10.0.2.4   SMB Server
 476
 477 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 478
 479 EODESCR
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 482
 483 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 484     $confdesc->{"net$i"} = $netdesc;
 485 }
 486
 487 my $drivename_hash;
 488
 489 my $idedesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial][,model=model]',
 493     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 496
 497 my $scsidesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 501     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 504
 505 my $satadesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial]',
 509     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 512
 513 my $virtiodesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-drive',
 516     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,serial=serial]',
 517     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 518 };
 519 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 520
 521 my $usbdesc = {
 522     optional => 1,
 523     type => 'string', format => 'pve-qm-usb-device',
 524     typetext => 'host=HOSTUSBDEVICE|spice',
 525     description => <<EODESCR,
 526 Configure an USB device (n is 0 to 4). This can be used to
 527 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 528
 529 'bus-port(.port)*' (decimal numbers) or
 530 'vendor_id:product_id' (hexadeciaml numbers)
 531
 532 You can use the 'lsusb -t' command to list existing usb devices.
 533
 534 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 535
 536 The value 'spice' can be used to add a usb redirection devices for spice.
 537
 538 EODESCR
 539 };
 540 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 541
 542 my $hostpcidesc = {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-hostpci',
 545         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 546         description => <<EODESCR,
 547 Map host pci devices. HOSTPCIDEVICE syntax is:
 548
 549 'bus:dev.func' (hexadecimal numbers)
 550
 551 You can us the 'lspci' command to list existing pci devices.
 552
 553 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 561
 562 my $serialdesc = {
 563         optional => 1,
 564         type => 'string',
 565         pattern => '(/dev/.+|socket)',
 566         description =>  <<EODESCR,
 567 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 568
 569 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 570
 571 Experimental: user reported problems with this option.
 572 EODESCR
 573 };
 574
 575 my $paralleldesc= {
 576         optional => 1,
 577         type => 'string',
 578         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 579         description =>  <<EODESCR,
 580 Map host parallel devices (n is 0 to 2).
 581
 582 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 583
 584 Experimental: user reported problems with this option.
 585 EODESCR
 586 };
 587
 588 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 589     $confdesc->{"parallel$i"} = $paralleldesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 593     $confdesc->{"serial$i"} = $serialdesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 597     $confdesc->{"hostpci$i"} = $hostpcidesc;
 598 }
 599
 600 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 601     $drivename_hash->{"ide$i"} = 1;
 602     $confdesc->{"ide$i"} = $idedesc;
 603 }
 604
 605 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 606     $drivename_hash->{"sata$i"} = 1;
 607     $confdesc->{"sata$i"} = $satadesc;
 608 }
 609
 610 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 611     $drivename_hash->{"scsi$i"} = 1;
 612     $confdesc->{"scsi$i"} = $scsidesc ;
 613 }
 614
 615 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 616     $drivename_hash->{"virtio$i"} = 1;
 617     $confdesc->{"virtio$i"} = $virtiodesc;
 618 }
 619
 620 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 621     $confdesc->{"usb$i"} = $usbdesc;
 622 }
 623
 624 my $unuseddesc = {
 625     optional => 1,
 626     type => 'string', format => 'pve-volume-id',
 627     description => "Reference to unused volumes.",
 628 };
 629
 630 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 631     $confdesc->{"unused$i"} = $unuseddesc;
 632 }
 633
 634 my $kvm_api_version = 0;
 635
 636 sub kvm_version {
 637
 638     return $kvm_api_version if $kvm_api_version;
 639
 640     my $fh = IO::File->new("</dev/kvm") ||
 641         return 0;
 642
 643     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 644         $kvm_api_version = $v;
 645     }
 646
 647     $fh->close();
 648
 649     return  $kvm_api_version;
 650 }
 651
 652 my $kvm_user_version;
 653
 654 sub kvm_user_version {
 655
 656     return $kvm_user_version if $kvm_user_version;
 657
 658     $kvm_user_version = 'unknown';
 659
 660     my $tmp = `kvm -help 2>/dev/null`;
 661
 662     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 663         $kvm_user_version = $2;
 664     }
 665
 666     return $kvm_user_version;
 667
 668 }
 669
 670 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 671
 672 sub disknames {
 673     # order is important - used to autoselect boot disk
 674     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 675             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 676             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 677             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 678 }
 679
 680 sub valid_drivename {
 681     my $dev = shift;
 682
 683     return defined($drivename_hash->{$dev});
 684 }
 685
 686 sub option_exists {
 687     my $key = shift;
 688     return defined($confdesc->{$key});
 689 }
 690
 691 sub nic_models {
 692     return $nic_model_list;
 693 }
 694
 695 sub os_list_description {
 696
 697     return {
 698         other => 'Other',
 699         wxp => 'Windows XP',
 700         w2k => 'Windows 2000',
 701         w2k3 =>, 'Windows 2003',
 702         w2k8 => 'Windows 2008',
 703         wvista => 'Windows Vista',
 704         win7 => 'Windows 7',
 705         win8 => 'Windows 8/2012',
 706         l24 => 'Linux 2.4',
 707         l26 => 'Linux 2.6',
 708     };
 709 }
 710
 711 my $cdrom_path;
 712
 713 sub get_cdrom_path {
 714
 715     return  $cdrom_path if $cdrom_path;
 716
 717     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 718     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 719     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 720 }
 721
 722 sub get_iso_path {
 723     my ($storecfg, $vmid, $cdrom) = @_;
 724
 725     if ($cdrom eq 'cdrom') {
 726         return get_cdrom_path();
 727     } elsif ($cdrom eq 'none') {
 728         return '';
 729     } elsif ($cdrom =~ m|^/|) {
 730         return $cdrom;
 731     } else {
 732         return PVE::Storage::path($storecfg, $cdrom);
 733     }
 734 }
 735
 736 # try to convert old style file names to volume IDs
 737 sub filename_to_volume_id {
 738     my ($vmid, $file, $media) = @_;
 739
 740     if (!($file eq 'none' || $file eq 'cdrom' ||
 741           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 742
 743         return undef if $file =~ m|/|;
 744
 745         if ($media && $media eq 'cdrom') {
 746             $file = "local:iso/$file";
 747         } else {
 748             $file = "local:$vmid/$file";
 749         }
 750     }
 751
 752     return $file;
 753 }
 754
 755 sub verify_media_type {
 756     my ($opt, $vtype, $media) = @_;
 757
 758     return if !$media;
 759
 760     my $etype;
 761     if ($media eq 'disk') {
 762         $etype = 'images';
 763     } elsif ($media eq 'cdrom') {
 764         $etype = 'iso';
 765     } else {
 766         die "internal error";
 767     }
 768
 769     return if ($vtype eq $etype);
 770
 771     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 772 }
 773
 774 sub cleanup_drive_path {
 775     my ($opt, $storecfg, $drive) = @_;
 776
 777     # try to convert filesystem paths to volume IDs
 778
 779     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 780         ($drive->{file} !~ m|^/dev/.+|) &&
 781         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 782         ($drive->{file} !~ m/^\d+$/)) {
 783         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 784         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 785         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 786         verify_media_type($opt, $vtype, $drive->{media});
 787         $drive->{file} = $volid;
 788     }
 789
 790     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 791 }
 792
 793 sub create_conf_nolock {
 794     my ($vmid, $settings) = @_;
 795
 796     my $filename = config_file($vmid);
 797
 798     die "configuration file '$filename' already exists\n" if -f $filename;
 799
 800     my $defaults = load_defaults();
 801
 802     $settings->{name} = "vm$vmid" if !$settings->{name};
 803     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 804
 805     my $data = '';
 806     foreach my $opt (keys %$settings) {
 807         next if !$confdesc->{$opt};
 808
 809         my $value = $settings->{$opt};
 810         next if !$value;
 811
 812         $data .= "$opt: $value\n";
 813     }
 814
 815     PVE::Tools::file_set_contents($filename, $data);
 816 }
 817
 818 sub parse_hotplug_features {
 819     my ($data) = @_;
 820
 821     my $res = {};
 822
 823     return $res if $data eq '0';
 824
 825     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 826
 827     foreach my $feature (PVE::Tools::split_list($data)) {
 828         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 829             $res->{$1} = 1;
 830         } else {
 831             warn "ignoring unknown hotplug feature '$feature'\n";
 832         }
 833     }
 834     return $res;
 835 }
 836
 837 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 838 sub pve_verify_hotplug_features {
 839     my ($value, $noerr) = @_;
 840
 841     return $value if parse_hotplug_features($value);
 842
 843     return undef if $noerr;
 844
 845     die "unable to parse hotplug option\n";
 846 }
 847
 848 my $parse_size = sub {
 849     my ($value) = @_;
 850
 851     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 852     my ($size, $unit) = ($1, $3);
 853     if ($unit) {
 854         if ($unit eq 'K') {
 855             $size = $size * 1024;
 856         } elsif ($unit eq 'M') {
 857             $size = $size * 1024 * 1024;
 858         } elsif ($unit eq 'G') {
 859             $size = $size * 1024 * 1024 * 1024;
 860         }
 861     }
 862     return int($size);
 863 };
 864
 865 my $format_size = sub {
 866     my ($size) = @_;
 867
 868     $size = int($size);
 869
 870     my $kb = int($size/1024);
 871     return $size if $kb*1024 != $size;
 872
 873     my $mb = int($kb/1024);
 874     return "${kb}K" if $mb*1024 != $kb;
 875
 876     my $gb = int($mb/1024);
 877     return "${mb}M" if $gb*1024 != $mb;
 878
 879     return "${gb}G";
 880 };
 881
 882 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 883 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 884 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 885 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
 886 #        [,iothread=on][,serial=serial][,model=model]
 887
 888 sub parse_drive {
 889     my ($key, $data) = @_;
 890
 891     my $res = {};
 892
 893     # $key may be undefined - used to verify JSON parameters
 894     if (!defined($key)) {
 895         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 896         $res->{index} = 0;
 897     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 898         $res->{interface} = $1;
 899         $res->{index} = $2;
 900     } else {
 901         return undef;
 902     }
 903
 904     foreach my $p (split (/,/, $data)) {
 905         next if $p =~ m/^\s*$/;
 906
 907         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|detect_zeroes|iothread|queues|serial|model)=(.+)$/) {
 908             my ($k, $v) = ($1, $2);
 909
 910             $k = 'file' if $k eq 'volume';
 911
 912             return undef if defined $res->{$k};
 913
 914             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 915                 return undef if !$v || $v !~ m/^\d+/;
 916                 $k = "m$k";
 917                 $v = sprintf("%.3f", $v / (1024*1024));
 918             }
 919             $res->{$k} = $v;
 920         } else {
 921             if (!$res->{file} && $p !~ m/=/) {
 922                 $res->{file} = $p;
 923             } else {
 924                 return undef;
 925             }
 926         }
 927     }
 928
 929     return undef if !$res->{file};
 930
 931     return undef if $res->{cache} &&
 932         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 933     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 934     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 935     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 936     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 937     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 938     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 939     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 940     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 941     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 942     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 943     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 944     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 945     return undef if $res->{detect_zeroes} && $res->{detect_zeroes} !~ m/^(on|off)$/;
 946     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 947     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 948
 949     return undef if $res->{mbps_rd} && $res->{mbps};
 950     return undef if $res->{mbps_wr} && $res->{mbps};
 951
 952     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 957     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 958
 959     return undef if $res->{iops_rd} && $res->{iops};
 960     return undef if $res->{iops_wr} && $res->{iops};
 961
 962
 963     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 964     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 966     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 968     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 969
 970     if ($res->{size}) {
 971         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 972     }
 973
 974     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 975         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 976         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 977         return undef if $res->{interface} eq 'virtio';
 978     }
 979
 980     # rerror does not work with scsi drives
 981     if ($res->{rerror}) {
 982         return undef if $res->{interface} eq 'scsi';
 983     }
 984
 985     return $res;
 986 }
 987
 988 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 989
 990 sub print_drive {
 991     my ($vmid, $drive) = @_;
 992
 993     my $opts = '';
 994     foreach my $o (@qemu_drive_options, qw(mbps mbps_rd mbps_wr mbps_max mbps_rd_max mbps_wr_max backup iothread queues detect_zeroes)) {
 995         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 996     }
 997
 998     if ($drive->{size}) {
 999         $opts .= ",size=" . &$format_size($drive->{size});
1000     }
1001
1002     if (my $model = $drive->{model}) {
1003         $opts .= ",model=$model";
1004     }
1005
1006     return "$drive->{file}$opts";
1007 }
1008
1009 sub scsi_inquiry {
1010     my($fh, $noerr) = @_;
1011
1012     my $SG_IO = 0x2285;
1013     my $SG_GET_VERSION_NUM = 0x2282;
1014
1015     my $versionbuf = "\x00" x 8;
1016     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1017     if (!$ret) {
1018         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1019         return undef;
1020     }
1021     my $version = unpack("I", $versionbuf);
1022     if ($version < 30000) {
1023         die "scsi generic interface too old\n"  if !$noerr;
1024         return undef;
1025     }
1026
1027     my $buf = "\x00" x 36;
1028     my $sensebuf = "\x00" x 8;
1029     my $cmd = pack("C x3 C x1", 0x12, 36);
1030
1031     # see /usr/include/scsi/sg.h
1032     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1033
1034     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1035                       length($sensebuf), 0, length($buf), $buf,
1036                       $cmd, $sensebuf, 6000);
1037
1038     $ret = ioctl($fh, $SG_IO, $packet);
1039     if (!$ret) {
1040         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1041         return undef;
1042     }
1043
1044     my @res = unpack($sg_io_hdr_t, $packet);
1045     if ($res[17] || $res[18]) {
1046         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1047         return undef;
1048     }
1049
1050     my $res = {};
1051     (my $byte0, my $byte1, $res->{vendor},
1052      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1053
1054     $res->{removable} = $byte1 & 128 ? 1 : 0;
1055     $res->{type} = $byte0 & 31;
1056
1057     return $res;
1058 }
1059
1060 sub path_is_scsi {
1061     my ($path) = @_;
1062
1063     my $fh = IO::File->new("+<$path") || return undef;
1064     my $res = scsi_inquiry($fh, 1);
1065     close($fh);
1066
1067     return $res;
1068 }
1069
1070 sub machine_type_is_q35 {
1071     my ($conf) = @_;
1072
1073     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1074 }
1075
1076 sub print_tabletdevice_full {
1077     my ($conf) = @_;
1078
1079     my $q35 = machine_type_is_q35($conf);
1080
1081     # we use uhci for old VMs because tablet driver was buggy in older qemu
1082     my $usbbus = $q35 ? "ehci" : "uhci";
1083
1084     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1085 }
1086
1087 sub print_drivedevice_full {
1088     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1089
1090     my $device = '';
1091     my $maxdev = 0;
1092
1093     if ($drive->{interface} eq 'virtio') {
1094         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1095         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1096         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1097     } elsif ($drive->{interface} eq 'scsi') {
1098
1099         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1100         my $unit = $drive->{index} % $maxdev;
1101         my $devicetype = 'hd';
1102         my $path = '';
1103         if (drive_is_cdrom($drive)) {
1104               $devicetype = 'cd';
1105         } else {
1106               if ($drive->{file} =~ m|^/|) {
1107                   $path = $drive->{file};
1108               } else {
1109                   $path = PVE::Storage::path($storecfg, $drive->{file});
1110               }
1111
1112               if($path =~ m/^iscsi\:\/\//){
1113                   $devicetype = 'generic';
1114               } else {
1115                   if (my $info = path_is_scsi($path)) {
1116                       if ($info->{type} == 0) {
1117                           $devicetype = 'block';
1118                       } elsif ($info->{type} == 1) { # tape
1119                           $devicetype = 'generic';
1120                       }
1121                   }
1122               }
1123          }
1124
1125         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1126             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1127         } else {
1128             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         }
1130
1131     } elsif ($drive->{interface} eq 'ide'){
1132         $maxdev = 2;
1133         my $controller = int($drive->{index} / $maxdev);
1134         my $unit = $drive->{index} % $maxdev;
1135         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1136
1137         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1138         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1139             $device .= ",model=$model";
1140         }
1141     } elsif ($drive->{interface} eq 'sata'){
1142         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1143         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1144         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1145     } elsif ($drive->{interface} eq 'usb') {
1146         die "implement me";
1147         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1148     } else {
1149         die "unsupported interface type";
1150     }
1151
1152     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1153
1154     return $device;
1155 }
1156
1157 sub get_initiator_name {
1158     my $initiator;
1159
1160     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1161     while (defined(my $line = <$fh>)) {
1162         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1163         $initiator = $1;
1164         last;
1165     }
1166     $fh->close();
1167
1168     return $initiator;
1169 }
1170
1171 sub print_drive_full {
1172     my ($storecfg, $vmid, $drive) = @_;
1173
1174     my $path;
1175     my $volid = $drive->{file};
1176     my $format;
1177
1178     if (drive_is_cdrom($drive)) {
1179         $path = get_iso_path($storecfg, $vmid, $volid);
1180     } else {
1181         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1182         if ($storeid) {
1183             $path = PVE::Storage::path($storecfg, $volid);
1184             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1185             $format = qemu_img_format($scfg, $volname);
1186         } else {
1187             $path = $volid;
1188         }
1189    }
1190
1191     my $opts = '';
1192     foreach my $o (@qemu_drive_options) {
1193         next if $o eq 'bootindex';
1194         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1195     }
1196
1197     $opts .= ",format=$format" if $format && !$drive->{format};
1198
1199     foreach my $o (qw(bps bps_rd bps_wr)) {
1200         my $v = $drive->{"m$o"};
1201         $opts .= ",$o=" . int($v*1024*1024) if $v;
1202     }
1203
1204     my $cache_direct = 0;
1205
1206     if (my $cache = $drive->{cache}) {
1207         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1208     } elsif (!drive_is_cdrom($drive)) {
1209         $opts .= ",cache=none";
1210         $cache_direct = 1;
1211     }
1212
1213     # aio native works only with O_DIRECT
1214     if (!$drive->{aio}) {
1215         if($cache_direct) {
1216             $opts .= ",aio=native";
1217         } else {
1218             $opts .= ",aio=threads";
1219         }
1220     }
1221
1222     if (!drive_is_cdrom($drive)) {
1223         my $detectzeroes;
1224         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1225             $detectzeroes = 'off';
1226         } elsif ($drive->{discard}) {
1227             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1228         } else {
1229             # This used to be our default with discard not being specified:
1230             $detectzeroes = 'on';
1231         }
1232         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1233     }
1234
1235     my $pathinfo = $path ? "file=$path," : '';
1236
1237     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1238 }
1239
1240 sub print_netdevice_full {
1241     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1242
1243     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1244
1245     my $device = $net->{model};
1246     if ($net->{model} eq 'virtio') {
1247          $device = 'virtio-net-pci';
1248      };
1249
1250     my $pciaddr = print_pci_addr("$netid", $bridges);
1251     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1252     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1253         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1254         my $vectors = $net->{queues} * 2 + 2;
1255         $tmpstr .= ",vectors=$vectors,mq=on";
1256     }
1257     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1258
1259     if ($use_old_bios_files) {
1260         my $romfile;
1261         if ($device eq 'virtio-net-pci') {
1262             $romfile = 'pxe-virtio.rom';
1263         } elsif ($device eq 'e1000') {
1264             $romfile = 'pxe-e1000.rom';
1265         } elsif ($device eq 'ne2k') {
1266             $romfile = 'pxe-ne2k_pci.rom';
1267         } elsif ($device eq 'pcnet') {
1268             $romfile = 'pxe-pcnet.rom';
1269         } elsif ($device eq 'rtl8139') {
1270             $romfile = 'pxe-rtl8139.rom';
1271         }
1272         $tmpstr .= ",romfile=$romfile" if $romfile;
1273     }
1274
1275     return $tmpstr;
1276 }
1277
1278 sub print_netdev_full {
1279     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1280
1281     my $i = '';
1282     if ($netid =~ m/^net(\d+)$/) {
1283         $i = int($1);
1284     }
1285
1286     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1287
1288     my $ifname = "tap${vmid}i$i";
1289
1290     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1291     die "interface name '$ifname' is too long (max 15 character)\n"
1292         if length($ifname) >= 16;
1293
1294     my $vhostparam = '';
1295     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1296
1297     my $vmname = $conf->{name} || "vm$vmid";
1298
1299     my $netdev = "";
1300     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1301
1302     if ($net->{bridge}) {
1303         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1304     } else {
1305         $netdev = "type=user,id=$netid,hostname=$vmname";
1306     }
1307
1308     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1309
1310     return $netdev;
1311 }
1312
1313 sub drive_is_cdrom {
1314     my ($drive) = @_;
1315
1316     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1317
1318 }
1319
1320 sub parse_numa {
1321     my ($data) = @_;
1322
1323     my $res = {};
1324
1325     foreach my $kvp (split(/,/, $data)) {
1326
1327         if ($kvp =~ m/^memory=(\S+)$/) {
1328             $res->{memory} = $1;
1329         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1330             $res->{policy} = $1;
1331         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1332             $res->{cpus}->{start} = $1;
1333             $res->{cpus}->{end} = $3;
1334         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1335             $res->{hostnodes}->{start} = $1;
1336             $res->{hostnodes}->{end} = $3;
1337         } else {
1338             return undef;
1339         }
1340     }
1341
1342     return $res;
1343 }
1344
1345 sub parse_hostpci {
1346     my ($value) = @_;
1347
1348     return undef if !$value;
1349
1350
1351     my @list = split(/,/, $value);
1352     my $found;
1353
1354     my $res = {};
1355     foreach my $kv (@list) {
1356
1357         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1358             $found = 1;
1359             if(defined($4)){
1360                 push @{$res->{pciid}}, { id => $2 , function => $4};
1361
1362             }else{
1363                 my $pcidevices = lspci($2);
1364                 $res->{pciid} = $pcidevices->{$2};
1365             }
1366         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1367             $res->{rombar} = $1;
1368         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1369             $res->{'x-vga'} = $1;
1370         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1371             $res->{pcie} = 1 if $1 == 1;
1372         } else {
1373             warn "unknown hostpci setting '$kv'\n";
1374         }
1375     }
1376
1377     return undef if !$found;
1378
1379     return $res;
1380 }
1381
1382 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1383 sub parse_net {
1384     my ($data) = @_;
1385
1386     my $res = {};
1387
1388     foreach my $kvp (split(/,/, $data)) {
1389
1390         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1391             my $model = lc($1);
1392             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1393             $res->{model} = $model;
1394             $res->{macaddr} = $mac;
1395         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1396             $res->{bridge} = $1;
1397         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1398             $res->{queues} = $1;
1399         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1400             $res->{rate} = $1;
1401         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1402             $res->{tag} = $1;
1403         } elsif ($kvp =~ m/^firewall=([01])$/) {
1404             $res->{firewall} = $1;
1405         } elsif ($kvp =~ m/^link_down=([01])$/) {
1406             $res->{link_down} = $1;
1407         } else {
1408             return undef;
1409         }
1410
1411     }
1412
1413     return undef if !$res->{model};
1414
1415     return $res;
1416 }
1417
1418 sub print_net {
1419     my $net = shift;
1420
1421     my $res = "$net->{model}";
1422     $res .= "=$net->{macaddr}" if $net->{macaddr};
1423     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1424     $res .= ",rate=$net->{rate}" if $net->{rate};
1425     $res .= ",tag=$net->{tag}" if $net->{tag};
1426     $res .= ",firewall=1" if $net->{firewall};
1427     $res .= ",link_down=1" if $net->{link_down};
1428     $res .= ",queues=$net->{queues}" if $net->{queues};
1429
1430     return $res;
1431 }
1432
1433 sub add_random_macs {
1434     my ($settings) = @_;
1435
1436     foreach my $opt (keys %$settings) {
1437         next if $opt !~ m/^net(\d+)$/;
1438         my $net = parse_net($settings->{$opt});
1439         next if !$net;
1440         $settings->{$opt} = print_net($net);
1441     }
1442 }
1443
1444 sub add_unused_volume {
1445     my ($config, $volid) = @_;
1446
1447     my $key;
1448     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1449         my $test = "unused$ind";
1450         if (my $vid = $config->{$test}) {
1451             return if $vid eq $volid; # do not add duplicates
1452         } else {
1453             $key = $test;
1454         }
1455     }
1456
1457     die "To many unused volume - please delete them first.\n" if !$key;
1458
1459     $config->{$key} = $volid;
1460
1461     return $key;
1462 }
1463
1464 sub vm_is_volid_owner {
1465     my ($storecfg, $vmid, $volid) = @_;
1466
1467     if ($volid !~  m|^/|) {
1468         my ($path, $owner);
1469         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1470         if ($owner && ($owner == $vmid)) {
1471             return 1;
1472         }
1473     }
1474
1475     return undef;
1476 }
1477
1478 sub split_flagged_list {
1479     my $text = shift || '';
1480     $text =~ s/[,;]/ /g;
1481     $text =~ s/^\s+//;
1482     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1483 }
1484
1485 sub join_flagged_list {
1486     my ($how, $lst) = @_;
1487     join $how, map { $lst->{$_} . $_ } keys %$lst;
1488 }
1489
1490 sub vmconfig_delete_pending_option {
1491     my ($conf, $key, $force) = @_;
1492
1493     delete $conf->{pending}->{$key};
1494     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1495     $pending_delete_hash->{$key} = $force ? '!' : '';
1496     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1497 }
1498
1499 sub vmconfig_undelete_pending_option {
1500     my ($conf, $key) = @_;
1501
1502     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1503     delete $pending_delete_hash->{$key};
1504
1505     if (%$pending_delete_hash) {
1506         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1507     } else {
1508         delete $conf->{pending}->{delete};
1509     }
1510 }
1511
1512 sub vmconfig_register_unused_drive {
1513     my ($storecfg, $vmid, $conf, $drive) = @_;
1514
1515     if (!drive_is_cdrom($drive)) {
1516         my $volid = $drive->{file};
1517         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1518             add_unused_volume($conf, $volid, $vmid);
1519         }
1520     }
1521 }
1522
1523 sub vmconfig_cleanup_pending {
1524     my ($conf) = @_;
1525
1526     # remove pending changes when nothing changed
1527     my $changes;
1528     foreach my $opt (keys %{$conf->{pending}}) {
1529         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1530             $changes = 1;
1531             delete $conf->{pending}->{$opt};
1532         }
1533     }
1534
1535     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1536     my $pending_delete_hash = {};
1537     while (my ($opt, $force) = each %$current_delete_hash) {
1538         if (defined($conf->{$opt})) {
1539             $pending_delete_hash->{$opt} = $force;
1540         } else {
1541             $changes = 1;
1542         }
1543     }
1544
1545     if (%$pending_delete_hash) {
1546         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1547     } else {
1548         delete $conf->{pending}->{delete};
1549     }
1550
1551     return $changes;
1552 }
1553
1554 my $valid_smbios1_options = {
1555     manufacturer => '\S+',
1556     product => '\S+',
1557     version => '\S+',
1558     serial => '\S+',
1559     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1560     sku => '\S+',
1561     family => '\S+',
1562 };
1563
1564 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1565 sub parse_smbios1 {
1566     my ($data) = @_;
1567
1568     my $res = {};
1569
1570     foreach my $kvp (split(/,/, $data)) {
1571         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1572         my ($k, $v) = split(/=/, $kvp);
1573         return undef if !defined($k) || !defined($v);
1574         return undef if !$valid_smbios1_options->{$k};
1575         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1576         $res->{$k} = $v;
1577     }
1578
1579     return $res;
1580 }
1581
1582 sub print_smbios1 {
1583     my ($smbios1) = @_;
1584
1585     my $data = '';
1586     foreach my $k (keys %$smbios1) {
1587         next if !defined($smbios1->{$k});
1588         next if !$valid_smbios1_options->{$k};
1589         $data .= ',' if $data;
1590         $data .= "$k=$smbios1->{$k}";
1591     }
1592     return $data;
1593 }
1594
1595 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1596 sub verify_smbios1 {
1597     my ($value, $noerr) = @_;
1598
1599     return $value if parse_smbios1($value);
1600
1601     return undef if $noerr;
1602
1603     die "unable to parse smbios (type 1) options\n";
1604 }
1605
1606 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1607 sub verify_bootdisk {
1608     my ($value, $noerr) = @_;
1609
1610     return $value if valid_drivename($value);
1611
1612     return undef if $noerr;
1613
1614     die "invalid boot disk '$value'\n";
1615 }
1616
1617 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1618 sub verify_numa {
1619     my ($value, $noerr) = @_;
1620
1621     return $value if parse_numa($value);
1622
1623     return undef if $noerr;
1624
1625     die "unable to parse numa options\n";
1626 }
1627
1628 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1629 sub verify_net {
1630     my ($value, $noerr) = @_;
1631
1632     return $value if parse_net($value);
1633
1634     return undef if $noerr;
1635
1636     die "unable to parse network options\n";
1637 }
1638
1639 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1640 sub verify_drive {
1641     my ($value, $noerr) = @_;
1642
1643     return $value if parse_drive(undef, $value);
1644
1645     return undef if $noerr;
1646
1647     die "unable to parse drive options\n";
1648 }
1649
1650 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1651 sub verify_hostpci {
1652     my ($value, $noerr) = @_;
1653
1654     return $value if parse_hostpci($value);
1655
1656     return undef if $noerr;
1657
1658     die "unable to parse pci id\n";
1659 }
1660
1661 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1662 sub verify_watchdog {
1663     my ($value, $noerr) = @_;
1664
1665     return $value if parse_watchdog($value);
1666
1667     return undef if $noerr;
1668
1669     die "unable to parse watchdog options\n";
1670 }
1671
1672 sub parse_watchdog {
1673     my ($value) = @_;
1674
1675     return undef if !$value;
1676
1677     my $res = {};
1678
1679     foreach my $p (split(/,/, $value)) {
1680         next if $p =~ m/^\s*$/;
1681
1682         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1683             $res->{model} = $2;
1684         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1685             $res->{action} = $2;
1686         } else {
1687             return undef;
1688         }
1689     }
1690
1691     return $res;
1692 }
1693
1694 sub parse_usb_device {
1695     my ($value) = @_;
1696
1697     return undef if !$value;
1698
1699     my @dl = split(/,/, $value);
1700     my $found;
1701
1702     my $res = {};
1703     foreach my $v (@dl) {
1704         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1705             $found = 1;
1706             $res->{vendorid} = $2;
1707             $res->{productid} = $4;
1708         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1709             $found = 1;
1710             $res->{hostbus} = $1;
1711             $res->{hostport} = $2;
1712         } elsif ($v =~ m/^spice$/) {
1713             $found = 1;
1714             $res->{spice} = 1;
1715         } else {
1716             return undef;
1717         }
1718     }
1719     return undef if !$found;
1720
1721     return $res;
1722 }
1723
1724 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1725 sub verify_usb_device {
1726     my ($value, $noerr) = @_;
1727
1728     return $value if parse_usb_device($value);
1729
1730     return undef if $noerr;
1731
1732     die "unable to parse usb device\n";
1733 }
1734
1735 # add JSON properties for create and set function
1736 sub json_config_properties {
1737     my $prop = shift;
1738
1739     foreach my $opt (keys %$confdesc) {
1740         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1741         $prop->{$opt} = $confdesc->{$opt};
1742     }
1743
1744     return $prop;
1745 }
1746
1747 sub check_type {
1748     my ($key, $value) = @_;
1749
1750     die "unknown setting '$key'\n" if !$confdesc->{$key};
1751
1752     my $type = $confdesc->{$key}->{type};
1753
1754     if (!defined($value)) {
1755         die "got undefined value\n";
1756     }
1757
1758     if ($value =~ m/[\n\r]/) {
1759         die "property contains a line feed\n";
1760     }
1761
1762     if ($type eq 'boolean') {
1763         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1764         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1765         die "type check ('boolean') failed - got '$value'\n";
1766     } elsif ($type eq 'integer') {
1767         return int($1) if $value =~ m/^(\d+)$/;
1768         die "type check ('integer') failed - got '$value'\n";
1769     } elsif ($type eq 'number') {
1770         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1771         die "type check ('number') failed - got '$value'\n";
1772     } elsif ($type eq 'string') {
1773         if (my $fmt = $confdesc->{$key}->{format}) {
1774             if ($fmt eq 'pve-qm-drive') {
1775                 # special case - we need to pass $key to parse_drive()
1776                 my $drive = parse_drive($key, $value);
1777                 return $value if $drive;
1778                 die "unable to parse drive options\n";
1779             }
1780             PVE::JSONSchema::check_format($fmt, $value);
1781             return $value;
1782         }
1783         $value =~ s/^\"(.*)\"$/$1/;
1784         return $value;
1785     } else {
1786         die "internal error"
1787     }
1788 }
1789
1790 sub lock_config_full {
1791     my ($vmid, $timeout, $code, @param) = @_;
1792
1793     my $filename = config_file_lock($vmid);
1794
1795     my $res = lock_file($filename, $timeout, $code, @param);
1796
1797     die $@ if $@;
1798
1799     return $res;
1800 }
1801
1802 sub lock_config_mode {
1803     my ($vmid, $timeout, $shared, $code, @param) = @_;
1804
1805     my $filename = config_file_lock($vmid);
1806
1807     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1808
1809     die $@ if $@;
1810
1811     return $res;
1812 }
1813
1814 sub lock_config {
1815     my ($vmid, $code, @param) = @_;
1816
1817     return lock_config_full($vmid, 10, $code, @param);
1818 }
1819
1820 sub cfs_config_path {
1821     my ($vmid, $node) = @_;
1822
1823     $node = $nodename if !$node;
1824     return "nodes/$node/qemu-server/$vmid.conf";
1825 }
1826
1827 sub check_iommu_support{
1828     #fixme : need to check IOMMU support
1829     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1830
1831     my $iommu=1;
1832     return $iommu;
1833
1834 }
1835
1836 sub config_file {
1837     my ($vmid, $node) = @_;
1838
1839     my $cfspath = cfs_config_path($vmid, $node);
1840     return "/etc/pve/$cfspath";
1841 }
1842
1843 sub config_file_lock {
1844     my ($vmid) = @_;
1845
1846     return "$lock_dir/lock-$vmid.conf";
1847 }
1848
1849 sub touch_config {
1850     my ($vmid) = @_;
1851
1852     my $conf = config_file($vmid);
1853     utime undef, undef, $conf;
1854 }
1855
1856 sub destroy_vm {
1857     my ($storecfg, $vmid, $keep_empty_config) = @_;
1858
1859     my $conffile = config_file($vmid);
1860
1861     my $conf = load_config($vmid);
1862
1863     check_lock($conf);
1864
1865     # only remove disks owned by this VM
1866     foreach_drive($conf, sub {
1867         my ($ds, $drive) = @_;
1868
1869         return if drive_is_cdrom($drive);
1870
1871         my $volid = $drive->{file};
1872
1873         return if !$volid || $volid =~ m|^/|;
1874
1875         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1876         return if !$path || !$owner || ($owner != $vmid);
1877
1878         PVE::Storage::vdisk_free($storecfg, $volid);
1879     });
1880
1881     if ($keep_empty_config) {
1882         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1883     } else {
1884         unlink $conffile;
1885     }
1886
1887     # also remove unused disk
1888     eval {
1889         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1890
1891         eval {
1892             PVE::Storage::foreach_volid($dl, sub {
1893                 my ($volid, $sid, $volname, $d) = @_;
1894                 PVE::Storage::vdisk_free($storecfg, $volid);
1895             });
1896         };
1897         warn $@ if $@;
1898
1899     };
1900     warn $@ if $@;
1901 }
1902
1903 sub load_config {
1904     my ($vmid, $node) = @_;
1905
1906     my $cfspath = cfs_config_path($vmid, $node);
1907
1908     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1909
1910     die "no such VM ('$vmid')\n" if !defined($conf);
1911
1912     return $conf;
1913 }
1914
1915 sub parse_vm_config {
1916     my ($filename, $raw) = @_;
1917
1918     return undef if !defined($raw);
1919
1920     my $res = {
1921         digest => Digest::SHA::sha1_hex($raw),
1922         snapshots => {},
1923         pending => {},
1924     };
1925
1926     $filename =~ m|/qemu-server/(\d+)\.conf$|
1927         || die "got strange filename '$filename'";
1928
1929     my $vmid = $1;
1930
1931     my $conf = $res;
1932     my $descr;
1933     my $section = '';
1934
1935     my @lines = split(/\n/, $raw);
1936     foreach my $line (@lines) {
1937         next if $line =~ m/^\s*$/;
1938
1939         if ($line =~ m/^\[PENDING\]\s*$/i) {
1940             $section = 'pending';
1941             if (defined($descr)) {
1942                 $descr =~ s/\s+$//;
1943                 $conf->{description} = $descr;
1944             }
1945             $descr = undef;
1946             $conf = $res->{$section} = {};
1947             next;
1948
1949         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1950             $section = $1;
1951             if (defined($descr)) {
1952                 $descr =~ s/\s+$//;
1953                 $conf->{description} = $descr;
1954             }
1955             $descr = undef;
1956             $conf = $res->{snapshots}->{$section} = {};
1957             next;
1958         }
1959
1960         if ($line =~ m/^\#(.*)\s*$/) {
1961             $descr = '' if !defined($descr);
1962             $descr .= PVE::Tools::decode_text($1) . "\n";
1963             next;
1964         }
1965
1966         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1967             $descr = '' if !defined($descr);
1968             $descr .= PVE::Tools::decode_text($2);
1969         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1970             $conf->{snapstate} = $1;
1971         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1972             my $key = $1;
1973             my $value = $2;
1974             $conf->{$key} = $value;
1975         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1976             my $value = $1;
1977             if ($section eq 'pending') {
1978                 $conf->{delete} = $value; # we parse this later
1979             } else {
1980                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1981             }
1982         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1983             my $key = $1;
1984             my $value = $2;
1985             eval { $value = check_type($key, $value); };
1986             if ($@) {
1987                 warn "vm $vmid - unable to parse value of '$key' - $@";
1988             } else {
1989                 my $fmt = $confdesc->{$key}->{format};
1990                 if ($fmt && $fmt eq 'pve-qm-drive') {
1991                     my $v = parse_drive($key, $value);
1992                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1993                         $v->{file} = $volid;
1994                         $value = print_drive($vmid, $v);
1995                     } else {
1996                         warn "vm $vmid - unable to parse value of '$key'\n";
1997                         next;
1998                     }
1999                 }
2000
2001                 if ($key eq 'cdrom') {
2002                     $conf->{ide2} = $value;
2003                 } else {
2004                     $conf->{$key} = $value;
2005                 }
2006             }
2007         }
2008     }
2009
2010     if (defined($descr)) {
2011         $descr =~ s/\s+$//;
2012         $conf->{description} = $descr;
2013     }
2014     delete $res->{snapstate}; # just to be sure
2015
2016     return $res;
2017 }
2018
2019 sub write_vm_config {
2020     my ($filename, $conf) = @_;
2021
2022     delete $conf->{snapstate}; # just to be sure
2023
2024     if ($conf->{cdrom}) {
2025         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2026         $conf->{ide2} = $conf->{cdrom};
2027         delete $conf->{cdrom};
2028     }
2029
2030     # we do not use 'smp' any longer
2031     if ($conf->{sockets}) {
2032         delete $conf->{smp};
2033     } elsif ($conf->{smp}) {
2034         $conf->{sockets} = $conf->{smp};
2035         delete $conf->{cores};
2036         delete $conf->{smp};
2037     }
2038
2039     my $used_volids = {};
2040
2041     my $cleanup_config = sub {
2042         my ($cref, $pending, $snapname) = @_;
2043
2044         foreach my $key (keys %$cref) {
2045             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2046                 $key eq 'snapstate' || $key eq 'pending';
2047             my $value = $cref->{$key};
2048             if ($key eq 'delete') {
2049                 die "propertry 'delete' is only allowed in [PENDING]\n"
2050                     if !$pending;
2051                 # fixme: check syntax?
2052                 next;
2053             }
2054             eval { $value = check_type($key, $value); };
2055             die "unable to parse value of '$key' - $@" if $@;
2056
2057             $cref->{$key} = $value;
2058
2059             if (!$snapname && valid_drivename($key)) {
2060                 my $drive = parse_drive($key, $value);
2061                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2062             }
2063         }
2064     };
2065
2066     &$cleanup_config($conf);
2067
2068     &$cleanup_config($conf->{pending}, 1);
2069
2070     foreach my $snapname (keys %{$conf->{snapshots}}) {
2071         die "internal error" if $snapname eq 'pending';
2072         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2073     }
2074
2075     # remove 'unusedX' settings if we re-add a volume
2076     foreach my $key (keys %$conf) {
2077         my $value = $conf->{$key};
2078         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2079             delete $conf->{$key};
2080         }
2081     }
2082
2083     my $generate_raw_config = sub {
2084         my ($conf, $pending) = @_;
2085
2086         my $raw = '';
2087
2088         # add description as comment to top of file
2089         if (defined(my $descr = $conf->{description})) {
2090             if ($descr) {
2091                 foreach my $cl (split(/\n/, $descr)) {
2092                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2093                 }
2094             } else {
2095                 $raw .= "#\n" if $pending;
2096             }
2097         }
2098
2099         foreach my $key (sort keys %$conf) {
2100             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2101             $raw .= "$key: $conf->{$key}\n";
2102         }
2103         return $raw;
2104     };
2105
2106     my $raw = &$generate_raw_config($conf);
2107
2108     if (scalar(keys %{$conf->{pending}})){
2109         $raw .= "\n[PENDING]\n";
2110         $raw .= &$generate_raw_config($conf->{pending}, 1);
2111     }
2112
2113     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2114         $raw .= "\n[$snapname]\n";
2115         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2116     }
2117
2118     return $raw;
2119 }
2120
2121 sub update_config_nolock {
2122     my ($vmid, $conf, $skiplock) = @_;
2123
2124     check_lock($conf) if !$skiplock;
2125
2126     my $cfspath = cfs_config_path($vmid);
2127
2128     PVE::Cluster::cfs_write_file($cfspath, $conf);
2129 }
2130
2131 sub update_config {
2132     my ($vmid, $conf, $skiplock) = @_;
2133
2134     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2135 }
2136
2137 sub load_defaults {
2138
2139     my $res = {};
2140
2141     # we use static defaults from our JSON schema configuration
2142     foreach my $key (keys %$confdesc) {
2143         if (defined(my $default = $confdesc->{$key}->{default})) {
2144             $res->{$key} = $default;
2145         }
2146     }
2147
2148     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2149     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2150
2151     return $res;
2152 }
2153
2154 sub config_list {
2155     my $vmlist = PVE::Cluster::get_vmlist();
2156     my $res = {};
2157     return $res if !$vmlist || !$vmlist->{ids};
2158     my $ids = $vmlist->{ids};
2159
2160     foreach my $vmid (keys %$ids) {
2161         my $d = $ids->{$vmid};
2162         next if !$d->{node} || $d->{node} ne $nodename;
2163         next if !$d->{type} || $d->{type} ne 'qemu';
2164         $res->{$vmid}->{exists} = 1;
2165     }
2166     return $res;
2167 }
2168
2169 # test if VM uses local resources (to prevent migration)
2170 sub check_local_resources {
2171     my ($conf, $noerr) = @_;
2172
2173     my $loc_res = 0;
2174
2175     $loc_res = 1 if $conf->{hostusb}; # old syntax
2176     $loc_res = 1 if $conf->{hostpci}; # old syntax
2177
2178     foreach my $k (keys %$conf) {
2179         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2180         # sockets are safe: they will recreated be on the target side post-migrate
2181         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2182         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2183     }
2184
2185     die "VM uses local resources\n" if $loc_res && !$noerr;
2186
2187     return $loc_res;
2188 }
2189
2190 # check if used storages are available on all nodes (use by migrate)
2191 sub check_storage_availability {
2192     my ($storecfg, $conf, $node) = @_;
2193
2194     foreach_drive($conf, sub {
2195         my ($ds, $drive) = @_;
2196
2197         my $volid = $drive->{file};
2198         return if !$volid;
2199
2200         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2201         return if !$sid;
2202
2203         # check if storage is available on both nodes
2204         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2205         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2206    });
2207 }
2208
2209 # list nodes where all VM images are available (used by has_feature API)
2210 sub shared_nodes {
2211     my ($conf, $storecfg) = @_;
2212
2213     my $nodelist = PVE::Cluster::get_nodelist();
2214     my $nodehash = { map { $_ => 1 } @$nodelist };
2215     my $nodename = PVE::INotify::nodename();
2216
2217     foreach_drive($conf, sub {
2218         my ($ds, $drive) = @_;
2219
2220         my $volid = $drive->{file};
2221         return if !$volid;
2222
2223         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2224         if ($storeid) {
2225             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2226             if ($scfg->{disable}) {
2227                 $nodehash = {};
2228             } elsif (my $avail = $scfg->{nodes}) {
2229                 foreach my $node (keys %$nodehash) {
2230                     delete $nodehash->{$node} if !$avail->{$node};
2231                 }
2232             } elsif (!$scfg->{shared}) {
2233                 foreach my $node (keys %$nodehash) {
2234                     delete $nodehash->{$node} if $node ne $nodename
2235                 }
2236             }
2237         }
2238     });
2239
2240     return $nodehash
2241 }
2242
2243 sub check_lock {
2244     my ($conf) = @_;
2245
2246     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2247 }
2248
2249 sub check_cmdline {
2250     my ($pidfile, $pid) = @_;
2251
2252     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2253     if (defined($fh)) {
2254         my $line = <$fh>;
2255         $fh->close;
2256         return undef if !$line;
2257         my @param = split(/\0/, $line);
2258
2259         my $cmd = $param[0];
2260         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2261
2262         for (my $i = 0; $i < scalar (@param); $i++) {
2263             my $p = $param[$i];
2264             next if !$p;
2265             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2266                 my $p = $param[$i+1];
2267                 return 1 if $p && ($p eq $pidfile);
2268                 return undef;
2269             }
2270         }
2271     }
2272     return undef;
2273 }
2274
2275 sub check_running {
2276     my ($vmid, $nocheck, $node) = @_;
2277
2278     my $filename = config_file($vmid, $node);
2279
2280     die "unable to find configuration file for VM $vmid - no such machine\n"
2281         if !$nocheck && ! -f $filename;
2282
2283     my $pidfile = pidfile_name($vmid);
2284
2285     if (my $fd = IO::File->new("<$pidfile")) {
2286         my $st = stat($fd);
2287         my $line = <$fd>;
2288         close($fd);
2289
2290         my $mtime = $st->mtime;
2291         if ($mtime > time()) {
2292             warn "file '$filename' modified in future\n";
2293         }
2294
2295         if ($line =~ m/^(\d+)$/) {
2296             my $pid = $1;
2297             if (check_cmdline($pidfile, $pid)) {
2298                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2299                     return $pid;
2300                 }
2301             }
2302         }
2303     }
2304
2305     return undef;
2306 }
2307
2308 sub vzlist {
2309
2310     my $vzlist = config_list();
2311
2312     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2313
2314     while (defined(my $de = $fd->read)) {
2315         next if $de !~ m/^(\d+)\.pid$/;
2316         my $vmid = $1;
2317         next if !defined($vzlist->{$vmid});
2318         if (my $pid = check_running($vmid)) {
2319             $vzlist->{$vmid}->{pid} = $pid;
2320         }
2321     }
2322
2323     return $vzlist;
2324 }
2325
2326 sub disksize {
2327     my ($storecfg, $conf) = @_;
2328
2329     my $bootdisk = $conf->{bootdisk};
2330     return undef if !$bootdisk;
2331     return undef if !valid_drivename($bootdisk);
2332
2333     return undef if !$conf->{$bootdisk};
2334
2335     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2336     return undef if !defined($drive);
2337
2338     return undef if drive_is_cdrom($drive);
2339
2340     my $volid = $drive->{file};
2341     return undef if !$volid;
2342
2343     return $drive->{size};
2344 }
2345
2346 my $last_proc_pid_stat;
2347
2348 # get VM status information
2349 # This must be fast and should not block ($full == false)
2350 # We only query KVM using QMP if $full == true (this can be slow)
2351 sub vmstatus {
2352     my ($opt_vmid, $full) = @_;
2353
2354     my $res = {};
2355
2356     my $storecfg = PVE::Storage::config();
2357
2358     my $list = vzlist();
2359     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2360
2361     my $cpucount = $cpuinfo->{cpus} || 1;
2362
2363     foreach my $vmid (keys %$list) {
2364         next if $opt_vmid && ($vmid ne $opt_vmid);
2365
2366         my $cfspath = cfs_config_path($vmid);
2367         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2368
2369         my $d = {};
2370         $d->{pid} = $list->{$vmid}->{pid};
2371
2372         # fixme: better status?
2373         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2374
2375         my $size = disksize($storecfg, $conf);
2376         if (defined($size)) {
2377             $d->{disk} = 0; # no info available
2378             $d->{maxdisk} = $size;
2379         } else {
2380             $d->{disk} = 0;
2381             $d->{maxdisk} = 0;
2382         }
2383
2384         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2385         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2386         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2387
2388         $d->{name} = $conf->{name} || "VM $vmid";
2389         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2390
2391         if ($conf->{balloon}) {
2392             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2393             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2394         }
2395
2396         $d->{uptime} = 0;
2397         $d->{cpu} = 0;
2398         $d->{mem} = 0;
2399
2400         $d->{netout} = 0;
2401         $d->{netin} = 0;
2402
2403         $d->{diskread} = 0;
2404         $d->{diskwrite} = 0;
2405
2406         $d->{template} = is_template($conf);
2407
2408         $res->{$vmid} = $d;
2409     }
2410
2411     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2412     foreach my $dev (keys %$netdev) {
2413         next if $dev !~ m/^tap([1-9]\d*)i/;
2414         my $vmid = $1;
2415         my $d = $res->{$vmid};
2416         next if !$d;
2417
2418         $d->{netout} += $netdev->{$dev}->{receive};
2419         $d->{netin} += $netdev->{$dev}->{transmit};
2420
2421         if ($full) {
2422             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2423             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2424         }
2425
2426     }
2427
2428     my $ctime = gettimeofday;
2429
2430     foreach my $vmid (keys %$list) {
2431
2432         my $d = $res->{$vmid};
2433         my $pid = $d->{pid};
2434         next if !$pid;
2435
2436         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2437         next if !$pstat; # not running
2438
2439         my $used = $pstat->{utime} + $pstat->{stime};
2440
2441         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2442
2443         if ($pstat->{vsize}) {
2444             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2445         }
2446
2447         my $old = $last_proc_pid_stat->{$pid};
2448         if (!$old) {
2449             $last_proc_pid_stat->{$pid} = {
2450                 time => $ctime,
2451                 used => $used,
2452                 cpu => 0,
2453             };
2454             next;
2455         }
2456
2457         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2458
2459         if ($dtime > 1000) {
2460             my $dutime = $used -  $old->{used};
2461
2462             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2463             $last_proc_pid_stat->{$pid} = {
2464                 time => $ctime,
2465                 used => $used,
2466                 cpu => $d->{cpu},
2467             };
2468         } else {
2469             $d->{cpu} = $old->{cpu};
2470         }
2471     }
2472
2473     return $res if !$full;
2474
2475     my $qmpclient = PVE::QMPClient->new();
2476
2477     my $ballooncb = sub {
2478         my ($vmid, $resp) = @_;
2479
2480         my $info = $resp->{'return'};
2481         return if !$info->{max_mem};
2482
2483         my $d = $res->{$vmid};
2484
2485         # use memory assigned to VM
2486         $d->{maxmem} = $info->{max_mem};
2487         $d->{balloon} = $info->{actual};
2488
2489         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2490             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2491             $d->{freemem} = $info->{free_mem};
2492         }
2493
2494         $d->{ballooninfo} = $info;
2495     };
2496
2497     my $blockstatscb = sub {
2498         my ($vmid, $resp) = @_;
2499         my $data = $resp->{'return'} || [];
2500         my $totalrdbytes = 0;
2501         my $totalwrbytes = 0;
2502
2503         for my $blockstat (@$data) {
2504             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2505             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2506
2507             $blockstat->{device} =~ s/drive-//;
2508             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2509         }
2510         $res->{$vmid}->{diskread} = $totalrdbytes;
2511         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2512     };
2513
2514     my $statuscb = sub {
2515         my ($vmid, $resp) = @_;
2516
2517         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2518         # this fails if ballon driver is not loaded, so this must be
2519         # the last commnand (following command are aborted if this fails).
2520         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2521
2522         my $status = 'unknown';
2523         if (!defined($status = $resp->{'return'}->{status})) {
2524             warn "unable to get VM status\n";
2525             return;
2526         }
2527
2528         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2529     };
2530
2531     foreach my $vmid (keys %$list) {
2532         next if $opt_vmid && ($vmid ne $opt_vmid);
2533         next if !$res->{$vmid}->{pid}; # not running
2534         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2535     }
2536
2537     $qmpclient->queue_execute(undef, 1);
2538
2539     foreach my $vmid (keys %$list) {
2540         next if $opt_vmid && ($vmid ne $opt_vmid);
2541         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub foreach_dimm {
2548     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2549
2550     my $dimm_id = 0;
2551     my $current_size = 1024;
2552     my $dimm_size = 512;
2553     return if $current_size == $memory;
2554
2555     for (my $j = 0; $j < 8; $j++) {
2556         for (my $i = 0; $i < 32; $i++) {
2557             my $name = "dimm${dimm_id}";
2558             $dimm_id++;
2559             my $numanode = $i % $sockets;
2560             $current_size += $dimm_size;
2561             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2562             return  $current_size if $current_size >= $memory;
2563         }
2564         $dimm_size *= 2;
2565     }
2566 }
2567
2568 sub foreach_reverse_dimm {
2569     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2570
2571     my $dimm_id = 253;
2572     my $current_size = 4177920;
2573     my $dimm_size = 65536;
2574     return if $current_size == $memory;
2575
2576     for (my $j = 0; $j < 8; $j++) {
2577         for (my $i = 0; $i < 32; $i++) {
2578             my $name = "dimm${dimm_id}";
2579             $dimm_id--;
2580             my $numanode = $i % $sockets;
2581             $current_size -= $dimm_size;
2582             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2583             return  $current_size if $current_size <= $memory;
2584         }
2585         $dimm_size /= 2;
2586     }
2587 }
2588
2589 sub foreach_drive {
2590     my ($conf, $func) = @_;
2591
2592     foreach my $ds (keys %$conf) {
2593         next if !valid_drivename($ds);
2594
2595         my $drive = parse_drive($ds, $conf->{$ds});
2596         next if !$drive;
2597
2598         &$func($ds, $drive);
2599     }
2600 }
2601
2602 sub foreach_volid {
2603     my ($conf, $func) = @_;
2604
2605     my $volhash = {};
2606
2607     my $test_volid = sub {
2608         my ($volid, $is_cdrom) = @_;
2609
2610         return if !$volid;
2611
2612         $volhash->{$volid} = $is_cdrom || 0;
2613     };
2614
2615     foreach_drive($conf, sub {
2616         my ($ds, $drive) = @_;
2617         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2618     });
2619
2620     foreach my $snapname (keys %{$conf->{snapshots}}) {
2621         my $snap = $conf->{snapshots}->{$snapname};
2622         &$test_volid($snap->{vmstate}, 0);
2623         foreach_drive($snap, sub {
2624             my ($ds, $drive) = @_;
2625             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2626         });
2627     }
2628
2629     foreach my $volid (keys %$volhash) {
2630         &$func($volid, $volhash->{$volid});
2631     }
2632 }
2633
2634 sub vga_conf_has_spice {
2635     my ($vga) = @_;
2636
2637     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2638
2639     return $1 || 1;
2640 }
2641
2642 sub config_to_command {
2643     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2644
2645     my $cmd = [];
2646     my $globalFlags = [];
2647     my $machineFlags = [];
2648     my $rtcFlags = [];
2649     my $cpuFlags = [];
2650     my $devices = [];
2651     my $pciaddr = '';
2652     my $bridges = {};
2653     my $kvmver = kvm_user_version();
2654     my $vernum = 0; # unknown
2655     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2656         $vernum = $1*1000000+$2*1000;
2657     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2658         $vernum = $1*1000000+$2*1000+$3;
2659     }
2660
2661     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2662
2663     my $have_ovz = -f '/proc/vz/vestat';
2664
2665     my $q35 = machine_type_is_q35($conf);
2666     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2667     my $machine_type = $forcemachine || $conf->{machine};
2668     my $use_old_bios_files = undef;
2669     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2670
2671     my $cpuunits = defined($conf->{cpuunits}) ?
2672             $conf->{cpuunits} : $defaults->{cpuunits};
2673
2674     push @$cmd, '/usr/bin/systemd-run';
2675     push @$cmd, '--scope';
2676     push @$cmd, '--slice', "qemu";
2677     push @$cmd, '--unit', $vmid;
2678     # set KillMode=none, so that systemd don't kill those scopes
2679     # at shutdown (pve-manager service should stop the VMs instead)
2680     push @$cmd, '-p', "KillMode=none";
2681     push @$cmd, '-p', "CPUShares=$cpuunits";
2682     if ($conf->{cpulimit}) {
2683         my $cpulimit = int($conf->{cpulimit} * 100);
2684         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2685     }
2686
2687     push @$cmd, '/usr/bin/kvm';
2688
2689     push @$cmd, '-id', $vmid;
2690
2691     my $use_virtio = 0;
2692
2693     my $qmpsocket = qmp_socket($vmid);
2694     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2695     push @$cmd, '-mon', "chardev=qmp,mode=control";
2696
2697     my $socket = vnc_socket($vmid);
2698     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2699
2700     push @$cmd, '-pidfile' , pidfile_name($vmid);
2701
2702     push @$cmd, '-daemonize';
2703
2704     if ($conf->{smbios1}) {
2705         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2706     }
2707
2708     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2709         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2710         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2711         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2712         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2713         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2714         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2715     }
2716
2717     if ($q35) {
2718         # the q35 chipset support native usb2, so we enable usb controller
2719         # by default for this machine type
2720         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2721     } else {
2722         $pciaddr = print_pci_addr("piix3", $bridges);
2723         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2724
2725         my $use_usb2 = 0;
2726         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2727             next if !$conf->{"usb$i"};
2728             $use_usb2 = 1;
2729         }
2730         # include usb device config
2731         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2732     }
2733
2734     my $vga = $conf->{vga};
2735
2736     my $qxlnum = vga_conf_has_spice($vga);
2737     $vga = 'qxl' if $qxlnum;
2738
2739     if (!$vga) {
2740         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2741                                 $conf->{ostype} eq 'win7' ||
2742                                 $conf->{ostype} eq 'w2k8')) {
2743             $vga = 'std';
2744         } else {
2745             $vga = 'cirrus';
2746         }
2747     }
2748
2749     # enable absolute mouse coordinates (needed by vnc)
2750     my $tablet;
2751     if (defined($conf->{tablet})) {
2752         $tablet = $conf->{tablet};
2753     } else {
2754         $tablet = $defaults->{tablet};
2755         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2756         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2757     }
2758
2759     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2760
2761     my $nohyperv;
2762     # host pci devices
2763     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2764         my $d = parse_hostpci($conf->{"hostpci$i"});
2765         next if !$d;
2766
2767         my $pcie = $d->{pcie};
2768         if($pcie){
2769             die "q35 machine model is not enabled" if !$q35;
2770             $pciaddr = print_pcie_addr("hostpci$i");
2771         }else{
2772             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2773         }
2774
2775         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2776         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2777         if ($xvga && $xvga ne '') {
2778             push @$cpuFlags, 'kvm=off';
2779             $vga = 'none';
2780             $nohyperv = 1;
2781             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2782                 $xvga = "";
2783             }
2784         }
2785         my $pcidevices = $d->{pciid};
2786         my $multifunction = 1 if @$pcidevices > 1;
2787
2788         my $j=0;
2789         foreach my $pcidevice (@$pcidevices) {
2790
2791             my $id = "hostpci$i";
2792             $id .= ".$j" if $multifunction;
2793             my $addr = $pciaddr;
2794             $addr .= ".$j" if $multifunction;
2795             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2796
2797             if($j == 0){
2798                 $devicestr .= "$rombar$xvga";
2799                 $devicestr .= ",multifunction=on" if $multifunction;
2800             }
2801
2802             push @$devices, '-device', $devicestr;
2803             $j++;
2804         }
2805     }
2806
2807     # usb devices
2808     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2809         my $d = parse_usb_device($conf->{"usb$i"});
2810         next if !$d;
2811         if ($d->{vendorid} && $d->{productid}) {
2812             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2813         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2814             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2815         } elsif ($d->{spice}) {
2816             # usb redir support for spice
2817             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2818             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2819         }
2820     }
2821
2822     # serial devices
2823     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2824         if (my $path = $conf->{"serial$i"}) {
2825             if ($path eq 'socket') {
2826                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2827                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2828                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2829             } else {
2830                 die "no such serial device\n" if ! -c $path;
2831                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2832                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2833             }
2834         }
2835     }
2836
2837     # parallel devices
2838     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2839         if (my $path = $conf->{"parallel$i"}) {
2840             die "no such parallel device\n" if ! -c $path;
2841             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2842             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2843             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2844         }
2845     }
2846
2847     my $vmname = $conf->{name} || "vm$vmid";
2848
2849     push @$cmd, '-name', $vmname;
2850
2851     my $sockets = 1;
2852     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2853     $sockets = $conf->{sockets} if  $conf->{sockets};
2854
2855     my $cores = $conf->{cores} || 1;
2856
2857     my $maxcpus = $sockets * $cores;
2858
2859     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2860
2861     my $allowed_vcpus = $cpuinfo->{cpus};
2862
2863     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2864         if ($allowed_vcpus < $maxcpus);
2865
2866     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2867
2868     push @$cmd, '-nodefaults';
2869
2870     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2871
2872     my $bootindex_hash = {};
2873     my $i = 1;
2874     foreach my $o (split(//, $bootorder)) {
2875         $bootindex_hash->{$o} = $i*100;
2876         $i++;
2877     }
2878
2879     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2880
2881     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2882
2883     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2884
2885     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2886
2887     # time drift fix
2888     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2889
2890     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2891     my $useLocaltime = $conf->{localtime};
2892
2893     if (my $ost = $conf->{ostype}) {
2894         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2895
2896         if ($ost =~ m/^w/) { # windows
2897             $useLocaltime = 1 if !defined($conf->{localtime});
2898
2899             # use time drift fix when acpi is enabled
2900             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2901                 $tdf = 1 if !defined($conf->{tdf});
2902             }
2903         }
2904
2905         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2906             $ost eq 'wvista') {
2907             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2908             push @$cmd, '-no-hpet';
2909             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2910                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
2911                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
2912                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
2913
2914             } else {
2915                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
2916             }
2917         }
2918
2919         if ($ost eq 'win7' || $ost eq 'win8') {
2920             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
2921         }
2922     }
2923
2924     push @$rtcFlags, 'driftfix=slew' if $tdf;
2925
2926     if ($nokvm) {
2927         push @$machineFlags, 'accel=tcg';
2928     } else {
2929         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2930     }
2931
2932     if ($machine_type) {
2933         push @$machineFlags, "type=${machine_type}";
2934     }
2935
2936     if ($conf->{startdate}) {
2937         push @$rtcFlags, "base=$conf->{startdate}";
2938     } elsif ($useLocaltime) {
2939         push @$rtcFlags, 'base=localtime';
2940     }
2941
2942     my $cpu = $nokvm ? "qemu64" : "kvm64";
2943     $cpu = $conf->{cpu} if $conf->{cpu};
2944
2945     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2946
2947     push @$cpuFlags , '-x2apic'
2948         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2949
2950     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2951
2952     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2953
2954     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2955
2956         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2957         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2958     }
2959
2960     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2961
2962     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2963
2964     push @$cmd, '-cpu', $cpu;
2965
2966     my $memory = $conf->{memory} || $defaults->{memory};
2967     my $static_memory = 0;
2968     my $dimm_memory = 0;
2969
2970     if ($hotplug_features->{memory}) {
2971         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2972         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2973         $static_memory = $STATICMEM;
2974         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2975         $dimm_memory = $memory - $static_memory;
2976         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2977
2978     } else {
2979
2980         $static_memory = $memory;
2981         push @$cmd, '-m', $static_memory;
2982     }
2983
2984     if ($conf->{numa}) {
2985
2986         my $numa_totalmemory = undef;
2987         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2988             next if !$conf->{"numa$i"};
2989             my $numa = parse_numa($conf->{"numa$i"});
2990             next if !$numa;
2991             # memory
2992             die "missing numa node$i memory value\n" if !$numa->{memory};
2993             my $numa_memory = $numa->{memory};
2994             $numa_totalmemory += $numa_memory;
2995             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2996
2997             # cpus
2998             my $cpus_start = $numa->{cpus}->{start};
2999             die "missing numa node$i cpus\n" if !defined($cpus_start);
3000             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3001             my $cpus = $cpus_start;
3002             if (defined($cpus_end)) {
3003                 $cpus .= "-$cpus_end";
3004                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3005             }
3006
3007             # hostnodes
3008             my $hostnodes_start = $numa->{hostnodes}->{start};
3009             if (defined($hostnodes_start)) {
3010                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3011                 my $hostnodes = $hostnodes_start;
3012                 if (defined($hostnodes_end)) {
3013                     $hostnodes .= "-$hostnodes_end";
3014                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3015                 }
3016
3017                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3018                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3019                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3020                 }
3021
3022                 # policy
3023                 my $policy = $numa->{policy};
3024                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3025                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3026             }
3027
3028             push @$cmd, '-object', $numa_object;
3029             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3030         }
3031
3032         die "total memory for NUMA nodes must be equal to vm static memory\n"
3033             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3034
3035         #if no custom tology, we split memory and cores across numa nodes
3036         if(!$numa_totalmemory) {
3037
3038             my $numa_memory = ($static_memory / $sockets) . "M";
3039
3040             for (my $i = 0; $i < $sockets; $i++)  {
3041
3042                 my $cpustart = ($cores * $i);
3043                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3044                 my $cpus = $cpustart;
3045                 $cpus .= "-$cpuend" if $cpuend;
3046
3047                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3048                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3049             }
3050         }
3051     }
3052
3053     if ($hotplug_features->{memory}) {
3054         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3055             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3056             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3057             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3058
3059             #if dimm_memory is not aligned to dimm map
3060             if($current_size > $memory) {
3061                  $conf->{memory} = $current_size;
3062                  update_config_nolock($vmid, $conf, 1);
3063             }
3064         });
3065     }
3066
3067     push @$cmd, '-S' if $conf->{freeze};
3068
3069     # set keyboard layout
3070     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3071     push @$cmd, '-k', $kb if $kb;
3072
3073     # enable sound
3074     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3075     #push @$cmd, '-soundhw', 'es1370';
3076     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3077
3078     if($conf->{agent}) {
3079         my $qgasocket = qmp_socket($vmid, 1);
3080         my $pciaddr = print_pci_addr("qga0", $bridges);
3081         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3082         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3083         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3084     }
3085
3086     my $spice_port;
3087
3088     if ($qxlnum) {
3089         if ($qxlnum > 1) {
3090             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3091                 for(my $i = 1; $i < $qxlnum; $i++){
3092                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3093                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3094                 }
3095             } else {
3096                 # assume other OS works like Linux
3097                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3098                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3099             }
3100         }
3101
3102         my $pciaddr = print_pci_addr("spice", $bridges);
3103
3104         my $nodename = PVE::INotify::nodename();
3105         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3106         $spice_port = PVE::Tools::next_spice_port($pfamily);
3107
3108         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3109
3110         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3111         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3112         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3113     }
3114
3115     # enable balloon by default, unless explicitly disabled
3116     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3117         $pciaddr = print_pci_addr("balloon0", $bridges);
3118         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3119     }
3120
3121     if ($conf->{watchdog}) {
3122         my $wdopts = parse_watchdog($conf->{watchdog});
3123         $pciaddr = print_pci_addr("watchdog", $bridges);
3124         my $watchdog = $wdopts->{model} || 'i6300esb';
3125         push @$devices, '-device', "$watchdog$pciaddr";
3126         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3127     }
3128
3129     my $vollist = [];
3130     my $scsicontroller = {};
3131     my $ahcicontroller = {};
3132     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3133
3134     # Add iscsi initiator name if available
3135     if (my $initiator = get_initiator_name()) {
3136         push @$devices, '-iscsi', "initiator-name=$initiator";
3137     }
3138
3139     foreach_drive($conf, sub {
3140         my ($ds, $drive) = @_;
3141
3142         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3143             push @$vollist, $drive->{file};
3144         }
3145
3146         $use_virtio = 1 if $ds =~ m/^virtio/;
3147
3148         if (drive_is_cdrom ($drive)) {
3149             if ($bootindex_hash->{d}) {
3150                 $drive->{bootindex} = $bootindex_hash->{d};
3151                 $bootindex_hash->{d} += 1;
3152             }
3153         } else {
3154             if ($bootindex_hash->{c}) {
3155                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3156                 $bootindex_hash->{c} += 1;
3157             }
3158         }
3159
3160         if($drive->{interface} eq 'virtio'){
3161            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3162         }
3163
3164         if ($drive->{interface} eq 'scsi') {
3165
3166             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3167
3168             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3169             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3170
3171             my $iothread = '';
3172             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3173                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3174                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3175             }
3176
3177             my $queues = '';
3178             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3179                 $queues = ",num_queues=$drive->{queues}";
3180             }
3181
3182             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3183             $scsicontroller->{$controller}=1;
3184         }
3185
3186         if ($drive->{interface} eq 'sata') {
3187            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3188            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3189            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3190            $ahcicontroller->{$controller}=1;
3191         }
3192
3193         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3194         push @$devices, '-drive',$drive_cmd;
3195         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3196     });
3197
3198     for (my $i = 0; $i < $MAX_NETS; $i++) {
3199          next if !$conf->{"net$i"};
3200          my $d = parse_net($conf->{"net$i"});
3201          next if !$d;
3202
3203          $use_virtio = 1 if $d->{model} eq 'virtio';
3204
3205          if ($bootindex_hash->{n}) {
3206             $d->{bootindex} = $bootindex_hash->{n};
3207             $bootindex_hash->{n} += 1;
3208          }
3209
3210          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3211          push @$devices, '-netdev', $netdevfull;
3212
3213          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3214          push @$devices, '-device', $netdevicefull;
3215     }
3216
3217     if (!$q35) {
3218         # add pci bridges
3219         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3220            $bridges->{1} = 1;
3221            $bridges->{2} = 1;
3222         }
3223
3224         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3225
3226         while (my ($k, $v) = each %$bridges) {
3227             $pciaddr = print_pci_addr("pci.$k");
3228             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3229         }
3230     }
3231
3232     # add custom args
3233     if ($conf->{args}) {
3234         my $aa = PVE::Tools::split_args($conf->{args});
3235         push @$cmd, @$aa;
3236     }
3237
3238     push @$cmd, @$devices;
3239     push @$cmd, '-rtc', join(',', @$rtcFlags)
3240         if scalar(@$rtcFlags);
3241     push @$cmd, '-machine', join(',', @$machineFlags)
3242         if scalar(@$machineFlags);
3243     push @$cmd, '-global', join(',', @$globalFlags)
3244         if scalar(@$globalFlags);
3245
3246     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3247 }
3248
3249 sub vnc_socket {
3250     my ($vmid) = @_;
3251     return "${var_run_tmpdir}/$vmid.vnc";
3252 }
3253
3254 sub spice_port {
3255     my ($vmid) = @_;
3256
3257     my $res = vm_mon_cmd($vmid, 'query-spice');
3258
3259     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3260 }
3261
3262 sub qmp_socket {
3263     my ($vmid, $qga) = @_;
3264     my $sockettype = $qga ? 'qga' : 'qmp';
3265     return "${var_run_tmpdir}/$vmid.$sockettype";
3266 }
3267
3268 sub pidfile_name {
3269     my ($vmid) = @_;
3270     return "${var_run_tmpdir}/$vmid.pid";
3271 }
3272
3273 sub vm_devices_list {
3274     my ($vmid) = @_;
3275
3276     my $res = vm_mon_cmd($vmid, 'query-pci');
3277     my $devices = {};
3278     foreach my $pcibus (@$res) {
3279         foreach my $device (@{$pcibus->{devices}}) {
3280             next if !$device->{'qdev_id'};
3281             if ($device->{'pci_bridge'}) {
3282                 $devices->{$device->{'qdev_id'}} = 1;
3283                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3284                     next if !$bridge_device->{'qdev_id'};
3285                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3286                     $devices->{$device->{'qdev_id'}}++;
3287                 }
3288             } else {
3289                 $devices->{$device->{'qdev_id'}} = 1;
3290             }
3291         }
3292     }
3293
3294     my $resblock = vm_mon_cmd($vmid, 'query-block');
3295     foreach my $block (@$resblock) {
3296         if($block->{device} =~ m/^drive-(\S+)/){
3297                 $devices->{$1} = 1;
3298         }
3299     }
3300
3301     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3302     foreach my $mice (@$resmice) {
3303         if ($mice->{name} eq 'QEMU HID Tablet') {
3304             $devices->{tablet} = 1;
3305             last;
3306         }
3307     }
3308
3309     return $devices;
3310 }
3311
3312 sub vm_deviceplug {
3313     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3314
3315     my $q35 = machine_type_is_q35($conf);
3316
3317     my $devices_list = vm_devices_list($vmid);
3318     return 1 if defined($devices_list->{$deviceid});
3319
3320     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3321
3322     if ($deviceid eq 'tablet') {
3323
3324         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3325
3326     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3327
3328         qemu_iothread_add($vmid, $deviceid, $device);
3329
3330         qemu_driveadd($storecfg, $vmid, $device);
3331         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3332
3333         qemu_deviceadd($vmid, $devicefull);
3334         eval { qemu_deviceaddverify($vmid, $deviceid); };
3335         if (my $err = $@) {
3336             eval { qemu_drivedel($vmid, $deviceid); };
3337             warn $@ if $@;
3338             die $err;
3339         }
3340
3341     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3342
3343
3344         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3345         my $pciaddr = print_pci_addr($deviceid);
3346         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3347
3348         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3349
3350         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3351             qemu_iothread_add($vmid, $deviceid, $device);
3352             $devicefull .= ",iothread=iothread-$deviceid";
3353         }
3354
3355         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3356             $devicefull .= ",num_queues=$device->{queues}";
3357         }
3358
3359         qemu_deviceadd($vmid, $devicefull);
3360         qemu_deviceaddverify($vmid, $deviceid);
3361
3362     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3363
3364         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3365         qemu_driveadd($storecfg, $vmid, $device);
3366
3367         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3368         eval { qemu_deviceadd($vmid, $devicefull); };
3369         if (my $err = $@) {
3370             eval { qemu_drivedel($vmid, $deviceid); };
3371             warn $@ if $@;
3372             die $err;
3373         }
3374
3375     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3376
3377         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3378
3379         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3380         my $use_old_bios_files = undef;
3381         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3382
3383         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3384         qemu_deviceadd($vmid, $netdevicefull);
3385         eval { qemu_deviceaddverify($vmid, $deviceid); };
3386         if (my $err = $@) {
3387             eval { qemu_netdevdel($vmid, $deviceid); };
3388             warn $@ if $@;
3389             die $err;
3390         }
3391
3392     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3393
3394         my $bridgeid = $2;
3395         my $pciaddr = print_pci_addr($deviceid);
3396         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3397
3398         qemu_deviceadd($vmid, $devicefull);
3399         qemu_deviceaddverify($vmid, $deviceid);
3400
3401     } else {
3402         die "can't hotplug device '$deviceid'\n";
3403     }
3404
3405     return 1;
3406 }
3407
3408 # fixme: this should raise exceptions on error!
3409 sub vm_deviceunplug {
3410     my ($vmid, $conf, $deviceid) = @_;
3411
3412     my $devices_list = vm_devices_list($vmid);
3413     return 1 if !defined($devices_list->{$deviceid});
3414
3415     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3416
3417     if ($deviceid eq 'tablet') {
3418
3419         qemu_devicedel($vmid, $deviceid);
3420
3421     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3422
3423         qemu_devicedel($vmid, $deviceid);
3424         qemu_devicedelverify($vmid, $deviceid);
3425         qemu_drivedel($vmid, $deviceid);
3426         qemu_iothread_del($conf, $vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3429
3430         qemu_devicedel($vmid, $deviceid);
3431         qemu_devicedelverify($vmid, $deviceid);
3432         qemu_iothread_del($conf, $vmid, $deviceid);
3433
3434     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3435
3436         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3437         my $device = parse_drive($deviceid, $conf->{$deviceid});
3438         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3439
3440         qemu_devicedel($vmid, $deviceid);
3441         qemu_drivedel($vmid, $deviceid);
3442         qemu_deletescsihw($conf, $vmid, $deviceid);
3443
3444     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3445
3446         qemu_devicedel($vmid, $deviceid);
3447         qemu_devicedelverify($vmid, $deviceid);
3448         qemu_netdevdel($vmid, $deviceid);
3449
3450     } else {
3451         die "can't unplug device '$deviceid'\n";
3452     }
3453
3454     return 1;
3455 }
3456
3457 sub qemu_deviceadd {
3458     my ($vmid, $devicefull) = @_;
3459
3460     $devicefull = "driver=".$devicefull;
3461     my %options =  split(/[=,]/, $devicefull);
3462
3463     vm_mon_cmd($vmid, "device_add" , %options);
3464 }
3465
3466 sub qemu_devicedel {
3467     my ($vmid, $deviceid) = @_;
3468
3469     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3470 }
3471
3472 sub qemu_iothread_add {
3473     my($vmid, $deviceid, $device) = @_;
3474
3475     if ($device->{iothread}) {
3476         my $iothreads = vm_iothreads_list($vmid);
3477         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3478     }
3479 }
3480
3481 sub qemu_iothread_del {
3482     my($conf, $vmid, $deviceid) = @_;
3483
3484     my $device = parse_drive($deviceid, $conf->{$deviceid});
3485     if ($device->{iothread}) {
3486         my $iothreads = vm_iothreads_list($vmid);
3487         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3488     }
3489 }
3490
3491 sub qemu_objectadd {
3492     my($vmid, $objectid, $qomtype) = @_;
3493
3494     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3495
3496     return 1;
3497 }
3498
3499 sub qemu_objectdel {
3500     my($vmid, $objectid) = @_;
3501
3502     vm_mon_cmd($vmid, "object-del", id => $objectid);
3503
3504     return 1;
3505 }
3506
3507 sub qemu_driveadd {
3508     my ($storecfg, $vmid, $device) = @_;
3509
3510     my $drive = print_drive_full($storecfg, $vmid, $device);
3511     $drive =~ s/\\/\\\\/g;
3512     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3513
3514     # If the command succeeds qemu prints: "OK"
3515     return 1 if $ret =~ m/OK/s;
3516
3517     die "adding drive failed: $ret\n";
3518 }
3519
3520 sub qemu_drivedel {
3521     my($vmid, $deviceid) = @_;
3522
3523     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3524     $ret =~ s/^\s+//;
3525
3526     return 1 if $ret eq "";
3527
3528     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3529     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3530
3531     die "deleting drive $deviceid failed : $ret\n";
3532 }
3533
3534 sub qemu_deviceaddverify {
3535     my ($vmid, $deviceid) = @_;
3536
3537     for (my $i = 0; $i <= 5; $i++) {
3538          my $devices_list = vm_devices_list($vmid);
3539          return 1 if defined($devices_list->{$deviceid});
3540          sleep 1;
3541     }
3542
3543     die "error on hotplug device '$deviceid'\n";
3544 }
3545
3546
3547 sub qemu_devicedelverify {
3548     my ($vmid, $deviceid) = @_;
3549
3550     # need to verify that the device is correctly removed as device_del
3551     # is async and empty return is not reliable
3552
3553     for (my $i = 0; $i <= 5; $i++) {
3554          my $devices_list = vm_devices_list($vmid);
3555          return 1 if !defined($devices_list->{$deviceid});
3556          sleep 1;
3557     }
3558
3559     die "error on hot-unplugging device '$deviceid'\n";
3560 }
3561
3562 sub qemu_findorcreatescsihw {
3563     my ($storecfg, $conf, $vmid, $device) = @_;
3564
3565     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3566
3567     my $scsihwid="$controller_prefix$controller";
3568     my $devices_list = vm_devices_list($vmid);
3569
3570     if(!defined($devices_list->{$scsihwid})) {
3571         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3572     }
3573
3574     return 1;
3575 }
3576
3577 sub qemu_deletescsihw {
3578     my ($conf, $vmid, $opt) = @_;
3579
3580     my $device = parse_drive($opt, $conf->{$opt});
3581
3582     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3583         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3584         return 1;
3585     }
3586
3587     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3588
3589     my $devices_list = vm_devices_list($vmid);
3590     foreach my $opt (keys %{$devices_list}) {
3591         if (PVE::QemuServer::valid_drivename($opt)) {
3592             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3593             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3594                 return 1;
3595             }
3596         }
3597     }
3598
3599     my $scsihwid="scsihw$controller";
3600
3601     vm_deviceunplug($vmid, $conf, $scsihwid);
3602
3603     return 1;
3604 }
3605
3606 sub qemu_add_pci_bridge {
3607     my ($storecfg, $conf, $vmid, $device) = @_;
3608
3609     my $bridges = {};
3610
3611     my $bridgeid;
3612
3613     print_pci_addr($device, $bridges);
3614
3615     while (my ($k, $v) = each %$bridges) {
3616         $bridgeid = $k;
3617     }
3618     return 1 if !defined($bridgeid) || $bridgeid < 1;
3619
3620     my $bridge = "pci.$bridgeid";
3621     my $devices_list = vm_devices_list($vmid);
3622
3623     if (!defined($devices_list->{$bridge})) {
3624         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3625     }
3626
3627     return 1;
3628 }
3629
3630 sub qemu_set_link_status {
3631     my ($vmid, $device, $up) = @_;
3632
3633     vm_mon_cmd($vmid, "set_link", name => $device,
3634                up => $up ? JSON::true : JSON::false);
3635 }
3636
3637 sub qemu_netdevadd {
3638     my ($vmid, $conf, $device, $deviceid) = @_;
3639
3640     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3641     my %options =  split(/[=,]/, $netdev);
3642
3643     vm_mon_cmd($vmid, "netdev_add",  %options);
3644     return 1;
3645 }
3646
3647 sub qemu_netdevdel {
3648     my ($vmid, $deviceid) = @_;
3649
3650     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3651 }
3652
3653 sub qemu_cpu_hotplug {
3654     my ($vmid, $conf, $vcpus) = @_;
3655
3656     my $sockets = 1;
3657     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3658     $sockets = $conf->{sockets} if  $conf->{sockets};
3659     my $cores = $conf->{cores} || 1;
3660     my $maxcpus = $sockets * $cores;
3661
3662     $vcpus = $maxcpus if !$vcpus;
3663
3664     die "you can't add more vcpus than maxcpus\n"
3665         if $vcpus > $maxcpus;
3666
3667     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3668     die "online cpu unplug is not yet possible\n"
3669         if $vcpus < $currentvcpus;
3670
3671     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3672     die "vcpus in running vm is different than configuration\n"
3673         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3674
3675     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3676         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3677     }
3678 }
3679
3680 sub qemu_memory_hotplug {
3681     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3682
3683     return $value if !check_running($vmid);
3684
3685     my $memory = $conf->{memory} || $defaults->{memory};
3686     $value = $defaults->{memory} if !$value;
3687     return $value if $value == $memory;
3688
3689     my $static_memory = $STATICMEM;
3690     my $dimm_memory = $memory - $static_memory;
3691
3692     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3693     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3694
3695
3696     my $sockets = 1;
3697     $sockets = $conf->{sockets} if $conf->{sockets};
3698
3699     if($value > $memory) {
3700
3701         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3702             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3703
3704                 return if $current_size <= $conf->{memory};
3705
3706                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3707                 if (my $err = $@) {
3708                     eval { qemu_objectdel($vmid, "mem-$name"); };
3709                     die $err;
3710                 }
3711
3712                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3713                 if (my $err = $@) {
3714                     eval { qemu_objectdel($vmid, "mem-$name"); };
3715                     die $err;
3716                 }
3717                 #update conf after each succesful module hotplug
3718                 $conf->{memory} = $current_size;
3719                 update_config_nolock($vmid, $conf, 1);
3720         });
3721
3722     } else {
3723
3724         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3725             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3726
3727                 return if $current_size >= $conf->{memory};
3728                 print "try to unplug memory dimm $name\n";
3729
3730                 my $retry = 0;
3731                 while (1) {
3732                     eval { qemu_devicedel($vmid, $name) };
3733                     sleep 3;
3734                     my $dimm_list = qemu_dimm_list($vmid);
3735                     last if !$dimm_list->{$name};
3736                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3737                     $retry++;
3738                 }
3739
3740                 #update conf after each succesful module unplug
3741                 $conf->{memory} = $current_size;
3742
3743                 eval { qemu_objectdel($vmid, "mem-$name"); };
3744                 update_config_nolock($vmid, $conf, 1);
3745         });
3746     }
3747 }
3748
3749 sub qemu_dimm_list {
3750     my ($vmid) = @_;
3751
3752     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3753     my $dimms = {};
3754
3755     foreach my $dimm (@$dimmarray) {
3756
3757         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3758         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3759         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3760         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3761         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3762     }
3763     return $dimms;
3764 }
3765
3766 sub qemu_block_set_io_throttle {
3767     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3768
3769     return if !check_running($vmid) ;
3770
3771     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3772
3773 }
3774
3775 # old code, only used to shutdown old VM after update
3776 sub __read_avail {
3777     my ($fh, $timeout) = @_;
3778
3779     my $sel = new IO::Select;
3780     $sel->add($fh);
3781
3782     my $res = '';
3783     my $buf;
3784
3785     my @ready;
3786     while (scalar (@ready = $sel->can_read($timeout))) {
3787         my $count;
3788         if ($count = $fh->sysread($buf, 8192)) {
3789             if ($buf =~ /^(.*)\(qemu\) $/s) {
3790                 $res .= $1;
3791                 last;
3792             } else {
3793                 $res .= $buf;
3794             }
3795         } else {
3796             if (!defined($count)) {
3797                 die "$!\n";
3798             }
3799             last;
3800         }
3801     }
3802
3803     die "monitor read timeout\n" if !scalar(@ready);
3804
3805     return $res;
3806 }
3807
3808 # old code, only used to shutdown old VM after update
3809 sub vm_monitor_command {
3810     my ($vmid, $cmdstr, $nocheck) = @_;
3811
3812     my $res;
3813
3814     eval {
3815         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3816
3817         my $sname = "${var_run_tmpdir}/$vmid.mon";
3818
3819         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3820             die "unable to connect to VM $vmid socket - $!\n";
3821
3822         my $timeout = 3;
3823
3824         # hack: migrate sometime blocks the monitor (when migrate_downtime
3825         # is set)
3826         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3827             $timeout = 60*60; # 1 hour
3828         }
3829
3830         # read banner;
3831         my $data = __read_avail($sock, $timeout);
3832
3833         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3834             die "got unexpected qemu monitor banner\n";
3835         }
3836
3837         my $sel = new IO::Select;
3838         $sel->add($sock);
3839
3840         if (!scalar(my @ready = $sel->can_write($timeout))) {
3841             die "monitor write error - timeout";
3842         }
3843
3844         my $fullcmd = "$cmdstr\r";
3845
3846         # syslog('info', "VM $vmid monitor command: $cmdstr");
3847
3848         my $b;
3849         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3850             die "monitor write error - $!";
3851         }
3852
3853         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3854
3855         $timeout = 20;
3856
3857         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3858             $timeout = 60*60; # 1 hour
3859         } elsif ($cmdstr =~ m/^(eject|change)/) {
3860             $timeout = 60; # note: cdrom mount command is slow
3861         }
3862         if ($res = __read_avail($sock, $timeout)) {
3863
3864             my @lines = split("\r?\n", $res);
3865
3866             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3867
3868             $res = join("\n", @lines);
3869             $res .= "\n";
3870         }
3871     };
3872
3873     my $err = $@;
3874
3875     if ($err) {
3876         syslog("err", "VM $vmid monitor command failed - $err");
3877         die $err;
3878     }
3879
3880     return $res;
3881 }
3882
3883 sub qemu_block_resize {
3884     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3885
3886     my $running = check_running($vmid);
3887
3888     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3889
3890     return if !$running;
3891
3892     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3893
3894 }
3895
3896 sub qemu_volume_snapshot {
3897     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3898
3899     my $running = check_running($vmid);
3900
3901     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3902         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3903     } else {
3904         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3905     }
3906 }
3907
3908 sub qemu_volume_snapshot_delete {
3909     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3910
3911     my $running = check_running($vmid);
3912
3913     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3914
3915     return if !$running;
3916
3917     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3918 }
3919
3920 sub set_migration_caps {
3921     my ($vmid) = @_;
3922
3923     my $cap_ref = [];
3924
3925     my $enabled_cap = {
3926         "auto-converge" => 1,
3927         "xbzrle" => 1,
3928         "x-rdma-pin-all" => 0,
3929         "zero-blocks" => 0,
3930         "compress" => 0
3931     };
3932
3933     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3934
3935     for my $supported_capability (@$supported_capabilities) {
3936         push @$cap_ref, {
3937             capability => $supported_capability->{capability},
3938             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3939         };
3940     }
3941
3942     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3943 }
3944
3945 my $fast_plug_option = {
3946     'lock' => 1,
3947     'name' => 1,
3948     'onboot' => 1,
3949     'shares' => 1,
3950     'startup' => 1,
3951     'description' => 1,
3952 };
3953
3954 # hotplug changes in [PENDING]
3955 # $selection hash can be used to only apply specified options, for
3956 # example: { cores => 1 } (only apply changed 'cores')
3957 # $errors ref is used to return error messages
3958 sub vmconfig_hotplug_pending {
3959     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3960
3961     my $defaults = load_defaults();
3962
3963     # commit values which do not have any impact on running VM first
3964     # Note: those option cannot raise errors, we we do not care about
3965     # $selection and always apply them.
3966
3967     my $add_error = sub {
3968         my ($opt, $msg) = @_;
3969         $errors->{$opt} = "hotplug problem - $msg";
3970     };
3971
3972     my $changes = 0;
3973     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3974         if ($fast_plug_option->{$opt}) {
3975             $conf->{$opt} = $conf->{pending}->{$opt};
3976             delete $conf->{pending}->{$opt};
3977             $changes = 1;
3978         }
3979     }
3980
3981     if ($changes) {
3982         update_config_nolock($vmid, $conf, 1);
3983         $conf = load_config($vmid); # update/reload
3984     }
3985
3986     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3987
3988     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3989     while (my ($opt, $force) = each %$pending_delete_hash) {
3990         next if $selection && !$selection->{$opt};
3991         eval {
3992             if ($opt eq 'hotplug') {
3993                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3994             } elsif ($opt eq 'tablet') {
3995                 die "skip\n" if !$hotplug_features->{usb};
3996                 if ($defaults->{tablet}) {
3997                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3998                 } else {
3999                     vm_deviceunplug($vmid, $conf, $opt);
4000                 }
4001             } elsif ($opt eq 'vcpus') {
4002                 die "skip\n" if !$hotplug_features->{cpu};
4003                 qemu_cpu_hotplug($vmid, $conf, undef);
4004             } elsif ($opt eq 'balloon') {
4005                 # enable balloon device is not hotpluggable
4006                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4007             } elsif ($fast_plug_option->{$opt}) {
4008                 # do nothing
4009             } elsif ($opt =~ m/^net(\d+)$/) {
4010                 die "skip\n" if !$hotplug_features->{network};
4011                 vm_deviceunplug($vmid, $conf, $opt);
4012             } elsif (valid_drivename($opt)) {
4013                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4014                 vm_deviceunplug($vmid, $conf, $opt);
4015                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4016             } elsif ($opt =~ m/^memory$/) {
4017                 die "skip\n" if !$hotplug_features->{memory};
4018                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4019             } elsif ($opt eq 'cpuunits') {
4020                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4021             } elsif ($opt eq 'cpulimit') {
4022                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4023             } else {
4024                 die "skip\n";
4025             }
4026         };
4027         if (my $err = $@) {
4028             &$add_error($opt, $err) if $err ne "skip\n";
4029         } else {
4030             # save new config if hotplug was successful
4031             delete $conf->{$opt};
4032             vmconfig_undelete_pending_option($conf, $opt);
4033             update_config_nolock($vmid, $conf, 1);
4034             $conf = load_config($vmid); # update/reload
4035         }
4036     }
4037
4038     foreach my $opt (keys %{$conf->{pending}}) {
4039         next if $selection && !$selection->{$opt};
4040         my $value = $conf->{pending}->{$opt};
4041         eval {
4042             if ($opt eq 'hotplug') {
4043                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4044             } elsif ($opt eq 'tablet') {
4045                 die "skip\n" if !$hotplug_features->{usb};
4046                 if ($value == 1) {
4047                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4048                 } elsif ($value == 0) {
4049                     vm_deviceunplug($vmid, $conf, $opt);
4050                 }
4051             } elsif ($opt eq 'vcpus') {
4052                 die "skip\n" if !$hotplug_features->{cpu};
4053                 qemu_cpu_hotplug($vmid, $conf, $value);
4054             } elsif ($opt eq 'balloon') {
4055                 # enable/disable balloning device is not hotpluggable
4056                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4057                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4058                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4059
4060                 # allow manual ballooning if shares is set to zero
4061                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4062                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4063                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4064                 }
4065             } elsif ($opt =~ m/^net(\d+)$/) {
4066                 # some changes can be done without hotplug
4067                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4068                                     $vmid, $opt, $value);
4069             } elsif (valid_drivename($opt)) {
4070                 # some changes can be done without hotplug
4071                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4072                                      $vmid, $opt, $value, 1);
4073             } elsif ($opt =~ m/^memory$/) { #dimms
4074                 die "skip\n" if !$hotplug_features->{memory};
4075                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4076             } elsif ($opt eq 'cpuunits') {
4077                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4078             } elsif ($opt eq 'cpulimit') {
4079                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4080                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4081             } else {
4082                 die "skip\n";  # skip non-hot-pluggable options
4083             }
4084         };
4085         if (my $err = $@) {
4086             &$add_error($opt, $err) if $err ne "skip\n";
4087         } else {
4088             # save new config if hotplug was successful
4089             $conf->{$opt} = $value;
4090             delete $conf->{pending}->{$opt};
4091             update_config_nolock($vmid, $conf, 1);
4092             $conf = load_config($vmid); # update/reload
4093         }
4094     }
4095 }
4096
4097 sub try_deallocate_drive {
4098     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4099
4100     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4101         my $volid = $drive->{file};
4102         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4103             my $sid = PVE::Storage::parse_volume_id($volid);
4104             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4105
4106             # check if the disk is really unused
4107             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4108                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4109             PVE::Storage::vdisk_free($storecfg, $volid);
4110             return 1;
4111         } else {
4112             # If vm is not owner of this disk remove from config
4113             return 1;
4114         }
4115     }
4116
4117     return undef;
4118 }
4119
4120 sub vmconfig_delete_or_detach_drive {
4121     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4122
4123     my $drive = parse_drive($opt, $conf->{$opt});
4124
4125     my $rpcenv = PVE::RPCEnvironment::get();
4126     my $authuser = $rpcenv->get_user();
4127
4128     if ($force) {
4129         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4130         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4131     } else {
4132         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4133     }
4134 }
4135
4136 sub vmconfig_apply_pending {
4137     my ($vmid, $conf, $storecfg) = @_;
4138
4139     # cold plug
4140
4141     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4142     while (my ($opt, $force) = each %$pending_delete_hash) {
4143         die "internal error" if $opt =~ m/^unused/;
4144         $conf = load_config($vmid); # update/reload
4145         if (!defined($conf->{$opt})) {
4146             vmconfig_undelete_pending_option($conf, $opt);
4147             update_config_nolock($vmid, $conf, 1);
4148         } elsif (valid_drivename($opt)) {
4149             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4150             vmconfig_undelete_pending_option($conf, $opt);
4151             delete $conf->{$opt};
4152             update_config_nolock($vmid, $conf, 1);
4153         } else {
4154             vmconfig_undelete_pending_option($conf, $opt);
4155             delete $conf->{$opt};
4156             update_config_nolock($vmid, $conf, 1);
4157         }
4158     }
4159
4160     $conf = load_config($vmid); # update/reload
4161
4162     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4163         $conf = load_config($vmid); # update/reload
4164
4165         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4166             # skip if nothing changed
4167         } elsif (valid_drivename($opt)) {
4168             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4169                 if defined($conf->{$opt});
4170             $conf->{$opt} = $conf->{pending}->{$opt};
4171         } else {
4172             $conf->{$opt} = $conf->{pending}->{$opt};
4173         }
4174
4175         delete $conf->{pending}->{$opt};
4176         update_config_nolock($vmid, $conf, 1);
4177     }
4178 }
4179
4180 my $safe_num_ne = sub {
4181     my ($a, $b) = @_;
4182
4183     return 0 if !defined($a) && !defined($b);
4184     return 1 if !defined($a);
4185     return 1 if !defined($b);
4186
4187     return $a != $b;
4188 };
4189
4190 my $safe_string_ne = sub {
4191     my ($a, $b) = @_;
4192
4193     return 0 if !defined($a) && !defined($b);
4194     return 1 if !defined($a);
4195     return 1 if !defined($b);
4196
4197     return $a ne $b;
4198 };
4199
4200 sub vmconfig_update_net {
4201     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4202
4203     my $newnet = parse_net($value);
4204
4205     if ($conf->{$opt}) {
4206         my $oldnet = parse_net($conf->{$opt});
4207
4208         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4209             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4210             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4211             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4212
4213             # for non online change, we try to hot-unplug
4214             die "skip\n" if !$hotplug;
4215             vm_deviceunplug($vmid, $conf, $opt);
4216         } else {
4217
4218             die "internal error" if $opt !~ m/net(\d+)/;
4219             my $iface = "tap${vmid}i$1";
4220
4221             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4222                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4223             }
4224
4225             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4226                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4227                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4228                 PVE::Network::tap_unplug($iface);
4229                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4230             }
4231
4232             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4233                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4234             }
4235
4236             return 1;
4237         }
4238     }
4239
4240     if ($hotplug) {
4241         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4242     } else {
4243         die "skip\n";
4244     }
4245 }
4246
4247 sub vmconfig_update_disk {
4248     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4249
4250     # fixme: do we need force?
4251
4252     my $drive = parse_drive($opt, $value);
4253
4254     if ($conf->{$opt}) {
4255
4256         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4257
4258             my $media = $drive->{media} || 'disk';
4259             my $oldmedia = $old_drive->{media} || 'disk';
4260             die "unable to change media type\n" if $media ne $oldmedia;
4261
4262             if (!drive_is_cdrom($old_drive)) {
4263
4264                 if ($drive->{file} ne $old_drive->{file}) {
4265
4266                     die "skip\n" if !$hotplug;
4267
4268                     # unplug and register as unused
4269                     vm_deviceunplug($vmid, $conf, $opt);
4270                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4271
4272                 } else {
4273                     # update existing disk
4274
4275                     # skip non hotpluggable value
4276                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4277                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4278                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4279                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4280                         die "skip\n";
4281                     }
4282
4283                     # apply throttle
4284                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4285                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4286                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4287                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4288                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4289                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4290                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4291                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4292                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4293                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4294                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4295                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4296
4297                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4298                                                    ($drive->{mbps} || 0)*1024*1024,
4299                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4300                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4301                                                    $drive->{iops} || 0,
4302                                                    $drive->{iops_rd} || 0,
4303                                                    $drive->{iops_wr} || 0,
4304                                                    ($drive->{mbps_max} || 0)*1024*1024,
4305                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4306                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4307                                                    $drive->{iops_max} || 0,
4308                                                    $drive->{iops_rd_max} || 0,
4309                                                    $drive->{iops_wr_max} || 0);
4310
4311                     }
4312
4313                     return 1;
4314                 }
4315
4316             } else { # cdrom
4317
4318                 if ($drive->{file} eq 'none') {
4319                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4320                 } else {
4321                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4322                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4323                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4324                 }
4325
4326                 return 1;
4327             }
4328         }
4329     }
4330
4331     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4332     # hotplug new disks
4333     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4334     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4335 }
4336
4337 sub vm_start {
4338     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4339         $forcemachine, $spice_ticket) = @_;
4340
4341     lock_config($vmid, sub {
4342         my $conf = load_config($vmid, $migratedfrom);
4343
4344         die "you can't start a vm if it's a template\n" if is_template($conf);
4345
4346         check_lock($conf) if !$skiplock;
4347
4348         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4349
4350         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4351             vmconfig_apply_pending($vmid, $conf, $storecfg);
4352             $conf = load_config($vmid); # update/reload
4353         }
4354
4355         my $defaults = load_defaults();
4356
4357         # set environment variable useful inside network script
4358         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4359
4360         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4361
4362         my $migrate_port = 0;
4363         my $migrate_uri;
4364         if ($statefile) {
4365             if ($statefile eq 'tcp') {
4366                 my $localip = "localhost";
4367                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4368                 my $nodename = PVE::INotify::nodename();
4369                 if ($datacenterconf->{migration_unsecure}) {
4370                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4371                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4372                 }
4373                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4374                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4375                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4376                 push @$cmd, '-incoming', $migrate_uri;
4377                 push @$cmd, '-S';
4378             } else {
4379                 push @$cmd, '-loadstate', $statefile;
4380             }
4381         } elsif ($paused) {
4382             push @$cmd, '-S';
4383         }
4384
4385         # host pci devices
4386         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4387           my $d = parse_hostpci($conf->{"hostpci$i"});
4388           next if !$d;
4389           my $pcidevices = $d->{pciid};
4390           foreach my $pcidevice (@$pcidevices) {
4391                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4392
4393                 my $info = pci_device_info("0000:$pciid");
4394                 die "IOMMU not present\n" if !check_iommu_support();
4395                 die "no pci device info for device '$pciid'\n" if !$info;
4396                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4397                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4398           }
4399         }
4400
4401         PVE::Storage::activate_volumes($storecfg, $vollist);
4402
4403         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4404                     umask => 0077); };
4405         my $err = $@;
4406         die "start failed: $err" if $err;
4407
4408         print "migration listens on $migrate_uri\n" if $migrate_uri;
4409
4410         if ($statefile && $statefile ne 'tcp')  {
4411             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4412             warn $@ if $@;
4413         }
4414
4415         if ($migratedfrom) {
4416
4417             eval {
4418                 set_migration_caps($vmid);
4419             };
4420             warn $@ if $@;
4421
4422             if ($spice_port) {
4423                 print "spice listens on port $spice_port\n";
4424                 if ($spice_ticket) {
4425                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4426                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4427                 }
4428             }
4429
4430         } else {
4431
4432             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4433                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4434                     if $conf->{balloon};
4435             }
4436
4437             foreach my $opt (keys %$conf) {
4438                 next if $opt !~  m/^net\d+$/;
4439                 my $nicconf = parse_net($conf->{$opt});
4440                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4441             }
4442         }
4443
4444         vm_mon_cmd_nocheck($vmid, 'qom-set',
4445                     path => "machine/peripheral/balloon0",
4446                     property => "guest-stats-polling-interval",
4447                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4448
4449     });
4450 }
4451
4452 sub vm_mon_cmd {
4453     my ($vmid, $execute, %params) = @_;
4454
4455     my $cmd = { execute => $execute, arguments => \%params };
4456     vm_qmp_command($vmid, $cmd);
4457 }
4458
4459 sub vm_mon_cmd_nocheck {
4460     my ($vmid, $execute, %params) = @_;
4461
4462     my $cmd = { execute => $execute, arguments => \%params };
4463     vm_qmp_command($vmid, $cmd, 1);
4464 }
4465
4466 sub vm_qmp_command {
4467     my ($vmid, $cmd, $nocheck) = @_;
4468
4469     my $res;
4470
4471     my $timeout;
4472     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4473         $timeout = $cmd->{arguments}->{timeout};
4474         delete $cmd->{arguments}->{timeout};
4475     }
4476
4477     eval {
4478         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4479         my $sname = qmp_socket($vmid);
4480         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4481             my $qmpclient = PVE::QMPClient->new();
4482
4483             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4484         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4485             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4486                 if scalar(%{$cmd->{arguments}});
4487             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4488         } else {
4489             die "unable to open monitor socket\n";
4490         }
4491     };
4492     if (my $err = $@) {
4493         syslog("err", "VM $vmid qmp command failed - $err");
4494         die $err;
4495     }
4496
4497     return $res;
4498 }
4499
4500 sub vm_human_monitor_command {
4501     my ($vmid, $cmdline) = @_;
4502
4503     my $res;
4504
4505     my $cmd = {
4506         execute => 'human-monitor-command',
4507         arguments => { 'command-line' => $cmdline},
4508     };
4509
4510     return vm_qmp_command($vmid, $cmd);
4511 }
4512
4513 sub vm_commandline {
4514     my ($storecfg, $vmid) = @_;
4515
4516     my $conf = load_config($vmid);
4517
4518     my $defaults = load_defaults();
4519
4520     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4521
4522     return join(' ', @$cmd);
4523 }
4524
4525 sub vm_reset {
4526     my ($vmid, $skiplock) = @_;
4527
4528     lock_config($vmid, sub {
4529
4530         my $conf = load_config($vmid);
4531
4532         check_lock($conf) if !$skiplock;
4533
4534         vm_mon_cmd($vmid, "system_reset");
4535     });
4536 }
4537
4538 sub get_vm_volumes {
4539     my ($conf) = @_;
4540
4541     my $vollist = [];
4542     foreach_volid($conf, sub {
4543         my ($volid, $is_cdrom) = @_;
4544
4545         return if $volid =~ m|^/|;
4546
4547         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4548         return if !$sid;
4549
4550         push @$vollist, $volid;
4551     });
4552
4553     return $vollist;
4554 }
4555
4556 sub vm_stop_cleanup {
4557     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4558
4559     eval {
4560
4561         if (!$keepActive) {
4562             my $vollist = get_vm_volumes($conf);
4563             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4564         }
4565
4566         foreach my $ext (qw(mon qmp pid vnc qga)) {
4567             unlink "/var/run/qemu-server/${vmid}.$ext";
4568         }
4569
4570         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4571     };
4572     warn $@ if $@; # avoid errors - just warn
4573 }
4574
4575 # Note: use $nockeck to skip tests if VM configuration file exists.
4576 # We need that when migration VMs to other nodes (files already moved)
4577 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4578 sub vm_stop {
4579     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4580
4581     $force = 1 if !defined($force) && !$shutdown;
4582
4583     if ($migratedfrom){
4584         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4585         kill 15, $pid if $pid;
4586         my $conf = load_config($vmid, $migratedfrom);
4587         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4588         return;
4589     }
4590
4591     lock_config($vmid, sub {
4592
4593         my $pid = check_running($vmid, $nocheck);
4594         return if !$pid;
4595
4596         my $conf;
4597         if (!$nocheck) {
4598             $conf = load_config($vmid);
4599             check_lock($conf) if !$skiplock;
4600             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4601                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4602                 $timeout = $opts->{down} if $opts->{down};
4603             }
4604         }
4605
4606         $timeout = 60 if !defined($timeout);
4607
4608         eval {
4609             if ($shutdown) {
4610                 if (defined($conf) && $conf->{agent}) {
4611                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4612                 } else {
4613                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4614                 }
4615             } else {
4616                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4617             }
4618         };
4619         my $err = $@;
4620
4621         if (!$err) {
4622             my $count = 0;
4623             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4624                 $count++;
4625                 sleep 1;
4626             }
4627
4628             if ($count >= $timeout) {
4629                 if ($force) {
4630                     warn "VM still running - terminating now with SIGTERM\n";
4631                     kill 15, $pid;
4632                 } else {
4633                     die "VM quit/powerdown failed - got timeout\n";
4634                 }
4635             } else {
4636                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4637                 return;
4638             }
4639         } else {
4640             if ($force) {
4641                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4642                 kill 15, $pid;
4643             } else {
4644                 die "VM quit/powerdown failed\n";
4645             }
4646         }
4647
4648         # wait again
4649         $timeout = 10;
4650
4651         my $count = 0;
4652         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4653             $count++;
4654             sleep 1;
4655         }
4656
4657         if ($count >= $timeout) {
4658             warn "VM still running - terminating now with SIGKILL\n";
4659             kill 9, $pid;
4660             sleep 1;
4661         }
4662
4663         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4664    });
4665 }
4666
4667 sub vm_suspend {
4668     my ($vmid, $skiplock) = @_;
4669
4670     lock_config($vmid, sub {
4671
4672         my $conf = load_config($vmid);
4673
4674         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4675
4676         vm_mon_cmd($vmid, "stop");
4677     });
4678 }
4679
4680 sub vm_resume {
4681     my ($vmid, $skiplock, $nocheck) = @_;
4682
4683     lock_config($vmid, sub {
4684
4685         if (!$nocheck) {
4686
4687             my $conf = load_config($vmid);
4688
4689             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4690
4691             vm_mon_cmd($vmid, "cont");
4692
4693         } else {
4694             vm_mon_cmd_nocheck($vmid, "cont");
4695         }
4696     });
4697 }
4698
4699 sub vm_sendkey {
4700     my ($vmid, $skiplock, $key) = @_;
4701
4702     lock_config($vmid, sub {
4703
4704         my $conf = load_config($vmid);
4705
4706         # there is no qmp command, so we use the human monitor command
4707         vm_human_monitor_command($vmid, "sendkey $key");
4708     });
4709 }
4710
4711 sub vm_destroy {
4712     my ($storecfg, $vmid, $skiplock) = @_;
4713
4714     lock_config($vmid, sub {
4715
4716         my $conf = load_config($vmid);
4717
4718         check_lock($conf) if !$skiplock;
4719
4720         if (!check_running($vmid)) {
4721             destroy_vm($storecfg, $vmid);
4722         } else {
4723             die "VM $vmid is running - destroy failed\n";
4724         }
4725     });
4726 }
4727
4728 # pci helpers
4729
4730 sub file_write {
4731     my ($filename, $buf) = @_;
4732
4733     my $fh = IO::File->new($filename, "w");
4734     return undef if !$fh;
4735
4736     my $res = print $fh $buf;
4737
4738     $fh->close();
4739
4740     return $res;
4741 }
4742
4743 sub pci_device_info {
4744     my ($name) = @_;
4745
4746     my $res;
4747
4748     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4749     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4750
4751     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4752     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4753
4754     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4755     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4756
4757     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4758     return undef if !defined($product) || $product !~ s/^0x//;
4759
4760     $res = {
4761         name => $name,
4762         vendor => $vendor,
4763         product => $product,
4764         domain => $domain,
4765         bus => $bus,
4766         slot => $slot,
4767         func => $func,
4768         irq => $irq,
4769         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4770     };
4771
4772     return $res;
4773 }
4774
4775 sub pci_dev_reset {
4776     my ($dev) = @_;
4777
4778     my $name = $dev->{name};
4779
4780     my $fn = "$pcisysfs/devices/$name/reset";
4781
4782     return file_write($fn, "1");
4783 }
4784
4785 sub pci_dev_bind_to_vfio {
4786     my ($dev) = @_;
4787
4788     my $name = $dev->{name};
4789
4790     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4791
4792     if (!-d $vfio_basedir) {
4793         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4794     }
4795     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4796
4797     my $testdir = "$vfio_basedir/$name";
4798     return 1 if -d $testdir;
4799
4800     my $data = "$dev->{vendor} $dev->{product}";
4801     return undef if !file_write("$vfio_basedir/new_id", $data);
4802
4803     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4804     if (!file_write($fn, $name)) {
4805         return undef if -f $fn;
4806     }
4807
4808     $fn = "$vfio_basedir/bind";
4809     if (! -d $testdir) {
4810         return undef if !file_write($fn, $name);
4811     }
4812
4813     return -d $testdir;
4814 }
4815
4816 sub pci_dev_group_bind_to_vfio {
4817     my ($pciid) = @_;
4818
4819     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4820
4821     if (!-d $vfio_basedir) {
4822         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4823     }
4824     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4825
4826     # get IOMMU group devices
4827     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4828       my @devs = grep /^0000:/, readdir($D);
4829     closedir($D);
4830
4831     foreach my $pciid (@devs) {
4832         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4833
4834         # pci bridges, switches or root ports are not supported
4835         # they have a pci_bus subdirectory so skip them
4836         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4837
4838         my $info = pci_device_info($1);
4839         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4840     }
4841
4842     return 1;
4843 }
4844
4845 sub print_pci_addr {
4846     my ($id, $bridges) = @_;
4847
4848     my $res = '';
4849     my $devices = {
4850         piix3 => { bus => 0, addr => 1 },
4851         #addr2 : first videocard
4852         balloon0 => { bus => 0, addr => 3 },
4853         watchdog => { bus => 0, addr => 4 },
4854         scsihw0 => { bus => 0, addr => 5 },
4855         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4856         scsihw1 => { bus => 0, addr => 6 },
4857         ahci0 => { bus => 0, addr => 7 },
4858         qga0 => { bus => 0, addr => 8 },
4859         spice => { bus => 0, addr => 9 },
4860         virtio0 => { bus => 0, addr => 10 },
4861         virtio1 => { bus => 0, addr => 11 },
4862         virtio2 => { bus => 0, addr => 12 },
4863         virtio3 => { bus => 0, addr => 13 },
4864         virtio4 => { bus => 0, addr => 14 },
4865         virtio5 => { bus => 0, addr => 15 },
4866         hostpci0 => { bus => 0, addr => 16 },
4867         hostpci1 => { bus => 0, addr => 17 },
4868         net0 => { bus => 0, addr => 18 },
4869         net1 => { bus => 0, addr => 19 },
4870         net2 => { bus => 0, addr => 20 },
4871         net3 => { bus => 0, addr => 21 },
4872         net4 => { bus => 0, addr => 22 },
4873         net5 => { bus => 0, addr => 23 },
4874         vga1 => { bus => 0, addr => 24 },
4875         vga2 => { bus => 0, addr => 25 },
4876         vga3 => { bus => 0, addr => 26 },
4877         hostpci2 => { bus => 0, addr => 27 },
4878         hostpci3 => { bus => 0, addr => 28 },
4879         #addr29 : usb-host (pve-usb.cfg)
4880         'pci.1' => { bus => 0, addr => 30 },
4881         'pci.2' => { bus => 0, addr => 31 },
4882         'net6' => { bus => 1, addr => 1 },
4883         'net7' => { bus => 1, addr => 2 },
4884         'net8' => { bus => 1, addr => 3 },
4885         'net9' => { bus => 1, addr => 4 },
4886         'net10' => { bus => 1, addr => 5 },
4887         'net11' => { bus => 1, addr => 6 },
4888         'net12' => { bus => 1, addr => 7 },
4889         'net13' => { bus => 1, addr => 8 },
4890         'net14' => { bus => 1, addr => 9 },
4891         'net15' => { bus => 1, addr => 10 },
4892         'net16' => { bus => 1, addr => 11 },
4893         'net17' => { bus => 1, addr => 12 },
4894         'net18' => { bus => 1, addr => 13 },
4895         'net19' => { bus => 1, addr => 14 },
4896         'net20' => { bus => 1, addr => 15 },
4897         'net21' => { bus => 1, addr => 16 },
4898         'net22' => { bus => 1, addr => 17 },
4899         'net23' => { bus => 1, addr => 18 },
4900         'net24' => { bus => 1, addr => 19 },
4901         'net25' => { bus => 1, addr => 20 },
4902         'net26' => { bus => 1, addr => 21 },
4903         'net27' => { bus => 1, addr => 22 },
4904         'net28' => { bus => 1, addr => 23 },
4905         'net29' => { bus => 1, addr => 24 },
4906         'net30' => { bus => 1, addr => 25 },
4907         'net31' => { bus => 1, addr => 26 },
4908         'virtio6' => { bus => 2, addr => 1 },
4909         'virtio7' => { bus => 2, addr => 2 },
4910         'virtio8' => { bus => 2, addr => 3 },
4911         'virtio9' => { bus => 2, addr => 4 },
4912         'virtio10' => { bus => 2, addr => 5 },
4913         'virtio11' => { bus => 2, addr => 6 },
4914         'virtio12' => { bus => 2, addr => 7 },
4915         'virtio13' => { bus => 2, addr => 8 },
4916         'virtio14' => { bus => 2, addr => 9 },
4917         'virtio15' => { bus => 2, addr => 10 },
4918         'virtioscsi0' => { bus => 3, addr => 1 },
4919         'virtioscsi1' => { bus => 3, addr => 2 },
4920         'virtioscsi2' => { bus => 3, addr => 3 },
4921         'virtioscsi3' => { bus => 3, addr => 4 },
4922         'virtioscsi4' => { bus => 3, addr => 5 },
4923         'virtioscsi5' => { bus => 3, addr => 6 },
4924         'virtioscsi6' => { bus => 3, addr => 7 },
4925         'virtioscsi7' => { bus => 3, addr => 8 },
4926         'virtioscsi8' => { bus => 3, addr => 9 },
4927         'virtioscsi9' => { bus => 3, addr => 10 },
4928         'virtioscsi10' => { bus => 3, addr => 11 },
4929         'virtioscsi11' => { bus => 3, addr => 12 },
4930         'virtioscsi12' => { bus => 3, addr => 13 },
4931         'virtioscsi13' => { bus => 3, addr => 14 },
4932         'virtioscsi14' => { bus => 3, addr => 15 },
4933         'virtioscsi15' => { bus => 3, addr => 16 },
4934         'virtioscsi16' => { bus => 3, addr => 17 },
4935         'virtioscsi17' => { bus => 3, addr => 18 },
4936         'virtioscsi18' => { bus => 3, addr => 19 },
4937         'virtioscsi19' => { bus => 3, addr => 20 },
4938         'virtioscsi20' => { bus => 3, addr => 21 },
4939         'virtioscsi21' => { bus => 3, addr => 22 },
4940         'virtioscsi22' => { bus => 3, addr => 23 },
4941         'virtioscsi23' => { bus => 3, addr => 24 },
4942         'virtioscsi24' => { bus => 3, addr => 25 },
4943         'virtioscsi25' => { bus => 3, addr => 26 },
4944         'virtioscsi26' => { bus => 3, addr => 27 },
4945         'virtioscsi27' => { bus => 3, addr => 28 },
4946         'virtioscsi28' => { bus => 3, addr => 29 },
4947         'virtioscsi29' => { bus => 3, addr => 30 },
4948         'virtioscsi30' => { bus => 3, addr => 31 },
4949
4950     };
4951
4952     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4953            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4954            my $bus = $devices->{$id}->{bus};
4955            $res = ",bus=pci.$bus,addr=$addr";
4956            $bridges->{$bus} = 1 if $bridges;
4957     }
4958     return $res;
4959
4960 }
4961
4962 sub print_pcie_addr {
4963     my ($id) = @_;
4964
4965     my $res = '';
4966     my $devices = {
4967         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4968         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4969         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4970         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4971     };
4972
4973     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4974            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4975            my $bus = $devices->{$id}->{bus};
4976            $res = ",bus=$bus,addr=$addr";
4977     }
4978     return $res;
4979
4980 }
4981
4982 # vzdump restore implementaion
4983
4984 sub tar_archive_read_firstfile {
4985     my $archive = shift;
4986
4987     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4988
4989     # try to detect archive type first
4990     my $pid = open (TMP, "tar tf '$archive'|") ||
4991         die "unable to open file '$archive'\n";
4992     my $firstfile = <TMP>;
4993     kill 15, $pid;
4994     close TMP;
4995
4996     die "ERROR: archive contaions no data\n" if !$firstfile;
4997     chomp $firstfile;
4998
4999     return $firstfile;
5000 }
5001
5002 sub tar_restore_cleanup {
5003     my ($storecfg, $statfile) = @_;
5004
5005     print STDERR "starting cleanup\n";
5006
5007     if (my $fd = IO::File->new($statfile, "r")) {
5008         while (defined(my $line = <$fd>)) {
5009             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5010                 my $volid = $2;
5011                 eval {
5012                     if ($volid =~ m|^/|) {
5013                         unlink $volid || die 'unlink failed\n';
5014                     } else {
5015                         PVE::Storage::vdisk_free($storecfg, $volid);
5016                     }
5017                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5018                 };
5019                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5020             } else {
5021                 print STDERR "unable to parse line in statfile - $line";
5022             }
5023         }
5024         $fd->close();
5025     }
5026 }
5027
5028 sub restore_archive {
5029     my ($archive, $vmid, $user, $opts) = @_;
5030
5031     my $format = $opts->{format};
5032     my $comp;
5033
5034     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5035         $format = 'tar' if !$format;
5036         $comp = 'gzip';
5037     } elsif ($archive =~ m/\.tar$/) {
5038         $format = 'tar' if !$format;
5039     } elsif ($archive =~ m/.tar.lzo$/) {
5040         $format = 'tar' if !$format;
5041         $comp = 'lzop';
5042     } elsif ($archive =~ m/\.vma$/) {
5043         $format = 'vma' if !$format;
5044     } elsif ($archive =~ m/\.vma\.gz$/) {
5045         $format = 'vma' if !$format;
5046         $comp = 'gzip';
5047     } elsif ($archive =~ m/\.vma\.lzo$/) {
5048         $format = 'vma' if !$format;
5049         $comp = 'lzop';
5050     } else {
5051         $format = 'vma' if !$format; # default
5052     }
5053
5054     # try to detect archive format
5055     if ($format eq 'tar') {
5056         return restore_tar_archive($archive, $vmid, $user, $opts);
5057     } else {
5058         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5059     }
5060 }
5061
5062 sub restore_update_config_line {
5063     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5064
5065     return if $line =~ m/^\#qmdump\#/;
5066     return if $line =~ m/^\#vzdump\#/;
5067     return if $line =~ m/^lock:/;
5068     return if $line =~ m/^unused\d+:/;
5069     return if $line =~ m/^parent:/;
5070     return if $line =~ m/^template:/; # restored VM is never a template
5071
5072     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5073         # try to convert old 1.X settings
5074         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5075         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5076             my ($model, $macaddr) = split(/\=/, $devconfig);
5077             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5078             my $net = {
5079                 model => $model,
5080                 bridge => "vmbr$ind",
5081                 macaddr => $macaddr,
5082             };
5083             my $netstr = print_net($net);
5084
5085             print $outfd "net$cookie->{netcount}: $netstr\n";
5086             $cookie->{netcount}++;
5087         }
5088     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5089         my ($id, $netstr) = ($1, $2);
5090         my $net = parse_net($netstr);
5091         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5092         $netstr = print_net($net);
5093         print $outfd "$id: $netstr\n";
5094     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5095         my $virtdev = $1;
5096         my $value = $3;
5097         if ($line =~ m/backup=no/) {
5098             print $outfd "#$line";
5099         } elsif ($virtdev && $map->{$virtdev}) {
5100             my $di = parse_drive($virtdev, $value);
5101             delete $di->{format}; # format can change on restore
5102             $di->{file} = $map->{$virtdev};
5103             $value = print_drive($vmid, $di);
5104             print $outfd "$virtdev: $value\n";
5105         } else {
5106             print $outfd $line;
5107         }
5108     } else {
5109         print $outfd $line;
5110     }
5111 }
5112
5113 sub scan_volids {
5114     my ($cfg, $vmid) = @_;
5115
5116     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5117
5118     my $volid_hash = {};
5119     foreach my $storeid (keys %$info) {
5120         foreach my $item (@{$info->{$storeid}}) {
5121             next if !($item->{volid} && $item->{size});
5122             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5123             $volid_hash->{$item->{volid}} = $item;
5124         }
5125     }
5126
5127     return $volid_hash;
5128 }
5129
5130 sub is_volume_in_use {
5131     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5132
5133     my $path = PVE::Storage::path($storecfg, $volid);
5134
5135     my $scan_config = sub {
5136         my ($cref, $snapname) = @_;
5137
5138         foreach my $key (keys %$cref) {
5139             my $value = $cref->{$key};
5140             if (valid_drivename($key)) {
5141                 next if $skip_drive && $key eq $skip_drive;
5142                 my $drive = parse_drive($key, $value);
5143                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5144                 return 1 if $volid eq $drive->{file};
5145                 if ($drive->{file} =~ m!^/!) {
5146                     return 1 if $drive->{file} eq $path;
5147                 } else {
5148                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5149                     next if !$storeid;
5150                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5151                     next if !$scfg;
5152                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5153                 }
5154             }
5155         }
5156
5157         return 0;
5158     };
5159
5160     return 1 if &$scan_config($conf);
5161
5162     undef $skip_drive;
5163
5164     foreach my $snapname (keys %{$conf->{snapshots}}) {
5165         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5166     }
5167
5168     return 0;
5169 }
5170
5171 sub update_disksize {
5172     my ($vmid, $conf, $volid_hash) = @_;
5173
5174     my $changes;
5175
5176     my $used = {};
5177
5178     # Note: it is allowed to define multiple storages with same path (alias), so
5179     # we need to check both 'volid' and real 'path' (two different volid can point
5180     # to the same path).
5181
5182     my $usedpath = {};
5183
5184     # update size info
5185     foreach my $opt (keys %$conf) {
5186         if (valid_drivename($opt)) {
5187             my $drive = parse_drive($opt, $conf->{$opt});
5188             my $volid = $drive->{file};
5189             next if !$volid;
5190
5191             $used->{$volid} = 1;
5192             if ($volid_hash->{$volid} &&
5193                 (my $path = $volid_hash->{$volid}->{path})) {
5194                 $usedpath->{$path} = 1;
5195             }
5196
5197             next if drive_is_cdrom($drive);
5198             next if !$volid_hash->{$volid};
5199
5200             $drive->{size} = $volid_hash->{$volid}->{size};
5201             my $new = print_drive($vmid, $drive);
5202             if ($new ne $conf->{$opt}) {
5203                 $changes = 1;
5204                 $conf->{$opt} = $new;
5205             }
5206         }
5207     }
5208
5209     # remove 'unusedX' entry if volume is used
5210     foreach my $opt (keys %$conf) {
5211         next if $opt !~ m/^unused\d+$/;
5212         my $volid = $conf->{$opt};
5213         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5214         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5215             $changes = 1;
5216             delete $conf->{$opt};
5217         }
5218     }
5219
5220     foreach my $volid (sort keys %$volid_hash) {
5221         next if $volid =~ m/vm-$vmid-state-/;
5222         next if $used->{$volid};
5223         my $path = $volid_hash->{$volid}->{path};
5224         next if !$path; # just to be sure
5225         next if $usedpath->{$path};
5226         $changes = 1;
5227         add_unused_volume($conf, $volid);
5228         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5229     }
5230
5231     return $changes;
5232 }
5233
5234 sub rescan {
5235     my ($vmid, $nolock) = @_;
5236
5237     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5238
5239     my $volid_hash = scan_volids($cfg, $vmid);
5240
5241     my $updatefn =  sub {
5242         my ($vmid) = @_;
5243
5244         my $conf = load_config($vmid);
5245
5246         check_lock($conf);
5247
5248         my $vm_volids = {};
5249         foreach my $volid (keys %$volid_hash) {
5250             my $info = $volid_hash->{$volid};
5251             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5252         }
5253
5254         my $changes = update_disksize($vmid, $conf, $vm_volids);
5255
5256         update_config_nolock($vmid, $conf, 1) if $changes;
5257     };
5258
5259     if (defined($vmid)) {
5260         if ($nolock) {
5261             &$updatefn($vmid);
5262         } else {
5263             lock_config($vmid, $updatefn, $vmid);
5264         }
5265     } else {
5266         my $vmlist = config_list();
5267         foreach my $vmid (keys %$vmlist) {
5268             if ($nolock) {
5269                 &$updatefn($vmid);
5270             } else {
5271                 lock_config($vmid, $updatefn, $vmid);
5272             }
5273         }
5274     }
5275 }
5276
5277 sub restore_vma_archive {
5278     my ($archive, $vmid, $user, $opts, $comp) = @_;
5279
5280     my $input = $archive eq '-' ? "<&STDIN" : undef;
5281     my $readfrom = $archive;
5282
5283     my $uncomp = '';
5284     if ($comp) {
5285         $readfrom = '-';
5286         my $qarchive = PVE::Tools::shellquote($archive);
5287         if ($comp eq 'gzip') {
5288             $uncomp = "zcat $qarchive|";
5289         } elsif ($comp eq 'lzop') {
5290             $uncomp = "lzop -d -c $qarchive|";
5291         } else {
5292             die "unknown compression method '$comp'\n";
5293         }
5294
5295     }
5296
5297     my $tmpdir = "/var/tmp/vzdumptmp$$";
5298     rmtree $tmpdir;
5299
5300     # disable interrupts (always do cleanups)
5301     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5302         warn "got interrupt - ignored\n";
5303     };
5304
5305     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5306     POSIX::mkfifo($mapfifo, 0600);
5307     my $fifofh;
5308
5309     my $openfifo = sub {
5310         open($fifofh, '>', $mapfifo) || die $!;
5311     };
5312
5313     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5314
5315     my $oldtimeout;
5316     my $timeout = 5;
5317
5318     my $devinfo = {};
5319
5320     my $rpcenv = PVE::RPCEnvironment::get();
5321
5322     my $conffile = config_file($vmid);
5323     my $tmpfn = "$conffile.$$.tmp";
5324
5325     # Note: $oldconf is undef if VM does not exists
5326     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5327
5328     my $print_devmap = sub {
5329         my $virtdev_hash = {};
5330
5331         my $cfgfn = "$tmpdir/qemu-server.conf";
5332
5333         # we can read the config - that is already extracted
5334         my $fh = IO::File->new($cfgfn, "r") ||
5335             "unable to read qemu-server.conf - $!\n";
5336
5337         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5338         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5339             if -f $fwcfgfn;
5340
5341         while (defined(my $line = <$fh>)) {
5342             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5343                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5344                 die "archive does not contain data for drive '$virtdev'\n"
5345                     if !$devinfo->{$devname};
5346                 if (defined($opts->{storage})) {
5347                     $storeid = $opts->{storage} || 'local';
5348                 } elsif (!$storeid) {
5349                     $storeid = 'local';
5350                 }
5351                 $format = 'raw' if !$format;
5352                 $devinfo->{$devname}->{devname} = $devname;
5353                 $devinfo->{$devname}->{virtdev} = $virtdev;
5354                 $devinfo->{$devname}->{format} = $format;
5355                 $devinfo->{$devname}->{storeid} = $storeid;
5356
5357                 # check permission on storage
5358                 my $pool = $opts->{pool}; # todo: do we need that?
5359                 if ($user ne 'root@pam') {
5360                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5361                 }
5362
5363                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5364             }
5365         }
5366
5367         foreach my $devname (keys %$devinfo) {
5368             die "found no device mapping information for device '$devname'\n"
5369                 if !$devinfo->{$devname}->{virtdev};
5370         }
5371
5372         my $cfg = cfs_read_file('storage.cfg');
5373
5374         # create empty/temp config
5375         if ($oldconf) {
5376             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5377             foreach_drive($oldconf, sub {
5378                 my ($ds, $drive) = @_;
5379
5380                 return if drive_is_cdrom($drive);
5381
5382                 my $volid = $drive->{file};
5383
5384                 return if !$volid || $volid =~ m|^/|;
5385
5386                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5387                 return if !$path || !$owner || ($owner != $vmid);
5388
5389                 # Note: only delete disk we want to restore
5390                 # other volumes will become unused
5391                 if ($virtdev_hash->{$ds}) {
5392                     PVE::Storage::vdisk_free($cfg, $volid);
5393                 }
5394             });
5395         }
5396
5397         my $map = {};
5398         foreach my $virtdev (sort keys %$virtdev_hash) {
5399             my $d = $virtdev_hash->{$virtdev};
5400             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5401             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5402
5403             # test if requested format is supported
5404             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5405             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5406             $d->{format} = $defFormat if !$supported;
5407
5408             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5409                                                   $d->{format}, undef, $alloc_size);
5410             print STDERR "new volume ID is '$volid'\n";
5411             $d->{volid} = $volid;
5412             my $path = PVE::Storage::path($cfg, $volid);
5413
5414             PVE::Storage::activate_volumes($cfg,[$volid]);
5415
5416             my $write_zeros = 1;
5417             # fixme: what other storages types initialize volumes with zero?
5418             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5419                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5420                 $write_zeros = 0;
5421             }
5422
5423             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5424
5425             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5426             $map->{$virtdev} = $volid;
5427         }
5428
5429         $fh->seek(0, 0) || die "seek failed - $!\n";
5430
5431         my $outfd = new IO::File ($tmpfn, "w") ||
5432             die "unable to write config for VM $vmid\n";
5433
5434         my $cookie = { netcount => 0 };
5435         while (defined(my $line = <$fh>)) {
5436             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5437         }
5438
5439         $fh->close();
5440         $outfd->close();
5441     };
5442
5443     eval {
5444         # enable interrupts
5445         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5446             die "interrupted by signal\n";
5447         };
5448         local $SIG{ALRM} = sub { die "got timeout\n"; };
5449
5450         $oldtimeout = alarm($timeout);
5451
5452         my $parser = sub {
5453             my $line = shift;
5454
5455             print "$line\n";
5456
5457             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5458                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5459                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5460             } elsif ($line =~ m/^CTIME: /) {
5461                 # we correctly received the vma config, so we can disable
5462                 # the timeout now for disk allocation (set to 10 minutes, so
5463                 # that we always timeout if something goes wrong)
5464                 alarm(600);
5465                 &$print_devmap();
5466                 print $fifofh "done\n";
5467                 my $tmp = $oldtimeout || 0;
5468                 $oldtimeout = undef;
5469                 alarm($tmp);
5470                 close($fifofh);
5471             }
5472         };
5473
5474         print "restore vma archive: $cmd\n";
5475         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5476     };
5477     my $err = $@;
5478
5479     alarm($oldtimeout) if $oldtimeout;
5480
5481     my $vollist = [];
5482     foreach my $devname (keys %$devinfo) {
5483         my $volid = $devinfo->{$devname}->{volid};
5484         push @$vollist, $volid if $volid;
5485     }
5486
5487     my $cfg = cfs_read_file('storage.cfg');
5488     PVE::Storage::deactivate_volumes($cfg, $vollist);
5489
5490     unlink $mapfifo;
5491
5492     if ($err) {
5493         rmtree $tmpdir;
5494         unlink $tmpfn;
5495
5496         foreach my $devname (keys %$devinfo) {
5497             my $volid = $devinfo->{$devname}->{volid};
5498             next if !$volid;
5499             eval {
5500                 if ($volid =~ m|^/|) {
5501                     unlink $volid || die 'unlink failed\n';
5502                 } else {
5503                     PVE::Storage::vdisk_free($cfg, $volid);
5504                 }
5505                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5506             };
5507             print STDERR "unable to cleanup '$volid' - $@" if $@;
5508         }
5509         die $err;
5510     }
5511
5512     rmtree $tmpdir;
5513
5514     rename($tmpfn, $conffile) ||
5515         die "unable to commit configuration file '$conffile'\n";
5516
5517     PVE::Cluster::cfs_update(); # make sure we read new file
5518
5519     eval { rescan($vmid, 1); };
5520     warn $@ if $@;
5521 }
5522
5523 sub restore_tar_archive {
5524     my ($archive, $vmid, $user, $opts) = @_;
5525
5526     if ($archive ne '-') {
5527         my $firstfile = tar_archive_read_firstfile($archive);
5528         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5529             if $firstfile ne 'qemu-server.conf';
5530     }
5531
5532     my $storecfg = cfs_read_file('storage.cfg');
5533
5534     # destroy existing data - keep empty config
5535     my $vmcfgfn = config_file($vmid);
5536     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5537
5538     my $tocmd = "/usr/lib/qemu-server/qmextract";
5539
5540     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5541     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5542     $tocmd .= ' --prealloc' if $opts->{prealloc};
5543     $tocmd .= ' --info' if $opts->{info};
5544
5545     # tar option "xf" does not autodetect compression when read from STDIN,
5546     # so we pipe to zcat
5547     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5548         PVE::Tools::shellquote("--to-command=$tocmd");
5549
5550     my $tmpdir = "/var/tmp/vzdumptmp$$";
5551     mkpath $tmpdir;
5552
5553     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5554     local $ENV{VZDUMP_VMID} = $vmid;
5555     local $ENV{VZDUMP_USER} = $user;
5556
5557     my $conffile = config_file($vmid);
5558     my $tmpfn = "$conffile.$$.tmp";
5559
5560     # disable interrupts (always do cleanups)
5561     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5562         print STDERR "got interrupt - ignored\n";
5563     };
5564
5565     eval {
5566         # enable interrupts
5567         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5568             die "interrupted by signal\n";
5569         };
5570
5571         if ($archive eq '-') {
5572             print "extracting archive from STDIN\n";
5573             run_command($cmd, input => "<&STDIN");
5574         } else {
5575             print "extracting archive '$archive'\n";
5576             run_command($cmd);
5577         }
5578
5579         return if $opts->{info};
5580
5581         # read new mapping
5582         my $map = {};
5583         my $statfile = "$tmpdir/qmrestore.stat";
5584         if (my $fd = IO::File->new($statfile, "r")) {
5585             while (defined (my $line = <$fd>)) {
5586                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5587                     $map->{$1} = $2 if $1;
5588                 } else {
5589                     print STDERR "unable to parse line in statfile - $line\n";
5590                 }
5591             }
5592             $fd->close();
5593         }
5594
5595         my $confsrc = "$tmpdir/qemu-server.conf";
5596
5597         my $srcfd = new IO::File($confsrc, "r") ||
5598             die "unable to open file '$confsrc'\n";
5599
5600         my $outfd = new IO::File ($tmpfn, "w") ||
5601             die "unable to write config for VM $vmid\n";
5602
5603         my $cookie = { netcount => 0 };
5604         while (defined (my $line = <$srcfd>)) {
5605             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5606         }
5607
5608         $srcfd->close();
5609         $outfd->close();
5610     };
5611     my $err = $@;
5612
5613     if ($err) {
5614
5615         unlink $tmpfn;
5616
5617         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5618
5619         die $err;
5620     }
5621
5622     rmtree $tmpdir;
5623
5624     rename $tmpfn, $conffile ||
5625         die "unable to commit configuration file '$conffile'\n";
5626
5627     PVE::Cluster::cfs_update(); # make sure we read new file
5628
5629     eval { rescan($vmid, 1); };
5630     warn $@ if $@;
5631 };
5632
5633
5634 # Internal snapshots
5635
5636 # NOTE: Snapshot create/delete involves several non-atomic
5637 # action, and can take a long time.
5638 # So we try to avoid locking the file and use 'lock' variable
5639 # inside the config file instead.
5640
5641 my $snapshot_copy_config = sub {
5642     my ($source, $dest) = @_;
5643
5644     foreach my $k (keys %$source) {
5645         next if $k eq 'snapshots';
5646         next if $k eq 'snapstate';
5647         next if $k eq 'snaptime';
5648         next if $k eq 'vmstate';
5649         next if $k eq 'lock';
5650         next if $k eq 'digest';
5651         next if $k eq 'description';
5652         next if $k =~ m/^unused\d+$/;
5653
5654         $dest->{$k} = $source->{$k};
5655     }
5656 };
5657
5658 my $snapshot_apply_config = sub {
5659     my ($conf, $snap) = @_;
5660
5661     # copy snapshot list
5662     my $newconf = {
5663         snapshots => $conf->{snapshots},
5664     };
5665
5666     # keep description and list of unused disks
5667     foreach my $k (keys %$conf) {
5668         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5669         $newconf->{$k} = $conf->{$k};
5670     }
5671
5672     &$snapshot_copy_config($snap, $newconf);
5673
5674     return $newconf;
5675 };
5676
5677 sub foreach_writable_storage {
5678     my ($conf, $func) = @_;
5679
5680     my $sidhash = {};
5681
5682     foreach my $ds (keys %$conf) {
5683         next if !valid_drivename($ds);
5684
5685         my $drive = parse_drive($ds, $conf->{$ds});
5686         next if !$drive;
5687         next if drive_is_cdrom($drive);
5688
5689         my $volid = $drive->{file};
5690
5691         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5692         $sidhash->{$sid} = $sid if $sid;
5693     }
5694
5695     foreach my $sid (sort keys %$sidhash) {
5696         &$func($sid);
5697     }
5698 }
5699
5700 my $alloc_vmstate_volid = sub {
5701     my ($storecfg, $vmid, $conf, $snapname) = @_;
5702
5703     # Note: we try to be smart when selecting a $target storage
5704
5705     my $target;
5706
5707     # search shared storage first
5708     foreach_writable_storage($conf, sub {
5709         my ($sid) = @_;
5710         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5711         return if !$scfg->{shared};
5712
5713         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5714     });
5715
5716     if (!$target) {
5717         # now search local storage
5718         foreach_writable_storage($conf, sub {
5719             my ($sid) = @_;
5720             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5721             return if $scfg->{shared};
5722
5723             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5724         });
5725     }
5726
5727     $target = 'local' if !$target;
5728
5729     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5730     # we abort live save after $conf->{memory}, so we need at max twice that space
5731     my $size = $conf->{memory}*2 + $driver_state_size;
5732
5733     my $name = "vm-$vmid-state-$snapname";
5734     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5735     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5736     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5737
5738     return $volid;
5739 };
5740
5741 my $snapshot_prepare = sub {
5742     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5743
5744     my $snap;
5745
5746     my $updatefn =  sub {
5747
5748         my $conf = load_config($vmid);
5749
5750         die "you can't take a snapshot if it's a template\n"
5751             if is_template($conf);
5752
5753         check_lock($conf);
5754
5755         $conf->{lock} = 'snapshot';
5756
5757         die "snapshot name '$snapname' already used\n"
5758             if defined($conf->{snapshots}->{$snapname});
5759
5760         my $storecfg = PVE::Storage::config();
5761         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5762
5763         $snap = $conf->{snapshots}->{$snapname} = {};
5764
5765         if ($save_vmstate && check_running($vmid)) {
5766             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5767         }
5768
5769         &$snapshot_copy_config($conf, $snap);
5770
5771         $snap->{snapstate} = "prepare";
5772         $snap->{snaptime} = time();
5773         $snap->{description} = $comment if $comment;
5774
5775         # always overwrite machine if we save vmstate. This makes sure we
5776         # can restore it later using correct machine type
5777         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5778
5779         update_config_nolock($vmid, $conf, 1);
5780     };
5781
5782     lock_config($vmid, $updatefn);
5783
5784     return $snap;
5785 };
5786
5787 my $snapshot_commit = sub {
5788     my ($vmid, $snapname) = @_;
5789
5790     my $updatefn = sub {
5791
5792         my $conf = load_config($vmid);
5793
5794         die "missing snapshot lock\n"
5795             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5796
5797         my $has_machine_config = defined($conf->{machine});
5798
5799         my $snap = $conf->{snapshots}->{$snapname};
5800
5801         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5802
5803         die "wrong snapshot state\n"
5804             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5805
5806         delete $snap->{snapstate};
5807         delete $conf->{lock};
5808
5809         my $newconf = &$snapshot_apply_config($conf, $snap);
5810
5811         delete $newconf->{machine} if !$has_machine_config;
5812
5813         $newconf->{parent} = $snapname;
5814
5815         update_config_nolock($vmid, $newconf, 1);
5816     };
5817
5818     lock_config($vmid, $updatefn);
5819 };
5820
5821 sub snapshot_rollback {
5822     my ($vmid, $snapname) = @_;
5823
5824     my $prepare = 1;
5825
5826     my $storecfg = PVE::Storage::config();
5827
5828     my $conf = load_config($vmid);
5829
5830     my $get_snapshot_config = sub {
5831
5832         die "you can't rollback if vm is a template\n" if is_template($conf);
5833
5834         my $res = $conf->{snapshots}->{$snapname};
5835
5836         die "snapshot '$snapname' does not exist\n" if !defined($res);
5837
5838         return $res;
5839     };
5840
5841     my $snap = &$get_snapshot_config();
5842
5843     foreach_drive($snap, sub {
5844         my ($ds, $drive) = @_;
5845
5846         return if drive_is_cdrom($drive);
5847
5848         my $volid = $drive->{file};
5849
5850         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5851     });
5852
5853     my $updatefn = sub {
5854
5855         $conf = load_config($vmid);
5856
5857         $snap = &$get_snapshot_config();
5858
5859         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5860             if $snap->{snapstate};
5861
5862         if ($prepare) {
5863             check_lock($conf);
5864             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5865         }
5866
5867         die "unable to rollback vm $vmid: vm is running\n"
5868             if check_running($vmid);
5869
5870         if ($prepare) {
5871             $conf->{lock} = 'rollback';
5872         } else {
5873             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5874             delete $conf->{lock};
5875         }
5876
5877         my $forcemachine;
5878
5879         if (!$prepare) {
5880             my $has_machine_config = defined($conf->{machine});
5881
5882             # copy snapshot config to current config
5883             $conf = &$snapshot_apply_config($conf, $snap);
5884             $conf->{parent} = $snapname;
5885
5886             # Note: old code did not store 'machine', so we try to be smart
5887             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5888             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5889             # we remove the 'machine' configuration if not explicitly specified
5890             # in the original config.
5891             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5892         }
5893
5894         update_config_nolock($vmid, $conf, 1);
5895
5896         if (!$prepare && $snap->{vmstate}) {
5897             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5898             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5899         }
5900     };
5901
5902     lock_config($vmid, $updatefn);
5903
5904     foreach_drive($snap, sub {
5905         my ($ds, $drive) = @_;
5906
5907         return if drive_is_cdrom($drive);
5908
5909         my $volid = $drive->{file};
5910         my $device = "drive-$ds";
5911
5912         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5913     });
5914
5915     $prepare = 0;
5916     lock_config($vmid, $updatefn);
5917 }
5918
5919 my $savevm_wait = sub {
5920     my ($vmid) = @_;
5921
5922     for(;;) {
5923         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5924         if (!$stat->{status}) {
5925             die "savevm not active\n";
5926         } elsif ($stat->{status} eq 'active') {
5927             sleep(1);
5928             next;
5929         } elsif ($stat->{status} eq 'completed') {
5930             last;
5931         } else {
5932             die "query-savevm returned status '$stat->{status}'\n";
5933         }
5934     }
5935 };
5936
5937 sub do_snapshots_with_qemu {
5938     my ($storecfg, $volid) = @_;
5939
5940     my $storage_name = PVE::Storage::parse_volume_id($volid);
5941
5942     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5943         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5944         return 1;
5945     }
5946
5947     if ($volid =~ m/\.(qcow2|qed)$/){
5948         return 1;
5949     }
5950
5951     return undef;
5952 }
5953
5954 sub snapshot_create {
5955     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5956
5957     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5958
5959     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5960
5961     my $config = load_config($vmid);
5962
5963     my $running = check_running($vmid);
5964
5965     my $freezefs = $running && $config->{agent};
5966     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5967
5968     my $drivehash = {};
5969
5970     if ($freezefs) {
5971         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5972         warn "guest-fsfreeze-freeze problems - $@" if $@;
5973     }
5974
5975     eval {
5976         # create internal snapshots of all drives
5977
5978         my $storecfg = PVE::Storage::config();
5979
5980         if ($running) {
5981             if ($snap->{vmstate}) {
5982                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5983                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5984                 &$savevm_wait($vmid);
5985             } else {
5986                 vm_mon_cmd($vmid, "savevm-start");
5987             }
5988         };
5989
5990         foreach_drive($snap, sub {
5991             my ($ds, $drive) = @_;
5992
5993             return if drive_is_cdrom($drive);
5994
5995             my $volid = $drive->{file};
5996             my $device = "drive-$ds";
5997
5998             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5999             $drivehash->{$ds} = 1;
6000        });
6001     };
6002     my $err = $@;
6003
6004     if ($running) {
6005         eval { vm_mon_cmd($vmid, "savevm-end")  };
6006         warn $@ if $@;
6007
6008         if ($freezefs) {
6009             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6010             warn "guest-fsfreeze-thaw problems - $@" if $@;
6011         }
6012
6013         # savevm-end is async, we need to wait
6014         for (;;) {
6015             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6016             if (!$stat->{bytes}) {
6017                 last;
6018             } else {
6019                 print "savevm not yet finished\n";
6020                 sleep(1);
6021                 next;
6022             }
6023         }
6024     }
6025
6026     if ($err) {
6027         warn "snapshot create failed: starting cleanup\n";
6028         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6029         warn $@ if $@;
6030         die $err;
6031     }
6032
6033     &$snapshot_commit($vmid, $snapname);
6034 }
6035
6036 # Note: $drivehash is only set when called from snapshot_create.
6037 sub snapshot_delete {
6038     my ($vmid, $snapname, $force, $drivehash) = @_;
6039
6040     my $prepare = 1;
6041
6042     my $snap;
6043     my $unused = [];
6044
6045     my $unlink_parent = sub {
6046         my ($confref, $new_parent) = @_;
6047
6048         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6049             if ($new_parent) {
6050                 $confref->{parent} = $new_parent;
6051             } else {
6052                 delete $confref->{parent};
6053             }
6054         }
6055     };
6056
6057     my $updatefn =  sub {
6058         my ($remove_drive) = @_;
6059
6060         my $conf = load_config($vmid);
6061
6062         if (!$drivehash) {
6063             check_lock($conf);
6064             die "you can't delete a snapshot if vm is a template\n"
6065                 if is_template($conf);
6066         }
6067
6068         $snap = $conf->{snapshots}->{$snapname};
6069
6070         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6071
6072         # remove parent refs
6073         if (!$prepare) {
6074             &$unlink_parent($conf, $snap->{parent});
6075             foreach my $sn (keys %{$conf->{snapshots}}) {
6076                 next if $sn eq $snapname;
6077                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6078             }
6079         }
6080
6081         if ($remove_drive) {
6082             if ($remove_drive eq 'vmstate') {
6083                 delete $snap->{$remove_drive};
6084             } else {
6085                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6086                 my $volid = $drive->{file};
6087                 delete $snap->{$remove_drive};
6088                 add_unused_volume($conf, $volid);
6089             }
6090         }
6091
6092         if ($prepare) {
6093             $snap->{snapstate} = 'delete';
6094         } else {
6095             delete $conf->{snapshots}->{$snapname};
6096             delete $conf->{lock} if $drivehash;
6097             foreach my $volid (@$unused) {
6098                 add_unused_volume($conf, $volid);
6099             }
6100         }
6101
6102         update_config_nolock($vmid, $conf, 1);
6103     };
6104
6105     lock_config($vmid, $updatefn);
6106
6107     # now remove vmstate file
6108
6109     my $storecfg = PVE::Storage::config();
6110
6111     if ($snap->{vmstate}) {
6112         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6113         if (my $err = $@) {
6114             die $err if !$force;
6115             warn $err;
6116         }
6117         # save changes (remove vmstate from snapshot)
6118         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6119     };
6120
6121     # now remove all internal snapshots
6122     foreach_drive($snap, sub {
6123         my ($ds, $drive) = @_;
6124
6125         return if drive_is_cdrom($drive);
6126
6127         my $volid = $drive->{file};
6128         my $device = "drive-$ds";
6129
6130         if (!$drivehash || $drivehash->{$ds}) {
6131             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6132             if (my $err = $@) {
6133                 die $err if !$force;
6134                 warn $err;
6135             }
6136         }
6137
6138         # save changes (remove drive fron snapshot)
6139         lock_config($vmid, $updatefn, $ds) if !$force;
6140         push @$unused, $volid;
6141     });
6142
6143     # now cleanup config
6144     $prepare = 0;
6145     lock_config($vmid, $updatefn);
6146 }
6147
6148 sub has_feature {
6149     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6150
6151     my $err;
6152     foreach_drive($conf, sub {
6153         my ($ds, $drive) = @_;
6154
6155         return if drive_is_cdrom($drive);
6156         my $volid = $drive->{file};
6157         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6158     });
6159
6160     return $err ? 0 : 1;
6161 }
6162
6163 sub template_create {
6164     my ($vmid, $conf, $disk) = @_;
6165
6166     my $storecfg = PVE::Storage::config();
6167
6168     foreach_drive($conf, sub {
6169         my ($ds, $drive) = @_;
6170
6171         return if drive_is_cdrom($drive);
6172         return if $disk && $ds ne $disk;
6173
6174         my $volid = $drive->{file};
6175         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6176
6177         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6178         $drive->{file} = $voliddst;
6179         $conf->{$ds} = print_drive($vmid, $drive);
6180         update_config_nolock($vmid, $conf, 1);
6181     });
6182 }
6183
6184 sub is_template {
6185     my ($conf) = @_;
6186
6187     return 1 if defined $conf->{template} && $conf->{template} == 1;
6188 }
6189
6190 sub qemu_img_convert {
6191     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6192
6193     my $storecfg = PVE::Storage::config();
6194     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6195     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6196
6197     if ($src_storeid && $dst_storeid) {
6198
6199         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6200
6201         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6202         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6203
6204         my $src_format = qemu_img_format($src_scfg, $src_volname);
6205         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6206
6207         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6208         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6209
6210         my $cmd = [];
6211         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6212         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6213         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6214
6215         my $parser = sub {
6216             my $line = shift;
6217             if($line =~ m/\((\S+)\/100\%\)/){
6218                 my $percent = $1;
6219                 my $transferred = int($size * $percent / 100);
6220                 my $remaining = $size - $transferred;
6221
6222                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6223             }
6224
6225         };
6226
6227         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6228         my $err = $@;
6229         die "copy failed: $err" if $err;
6230     }
6231 }
6232
6233 sub qemu_img_format {
6234     my ($scfg, $volname) = @_;
6235
6236     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6237         return $1;
6238     } else {
6239         return "raw";
6240     }
6241 }
6242
6243 sub qemu_drive_mirror {
6244     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6245
6246     my $storecfg = PVE::Storage::config();
6247     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6248
6249     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6250
6251     my $format = qemu_img_format($dst_scfg, $dst_volname);
6252
6253     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6254
6255     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6256     $opts->{format} = $format if $format;
6257
6258     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6259
6260     eval {
6261     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6262         while (1) {
6263             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6264             my $stat = @$stats[0];
6265             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6266             die "error job is not mirroring" if $stat->{type} ne "mirror";
6267
6268             my $busy = $stat->{busy};
6269             my $ready = $stat->{ready};
6270
6271             if (my $total = $stat->{len}) {
6272                 my $transferred = $stat->{offset} || 0;
6273                 my $remaining = $total - $transferred;
6274                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6275
6276                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6277             }
6278
6279
6280             if ($stat->{ready} eq 'true') {
6281
6282                 last if $vmiddst != $vmid;
6283
6284                 # try to switch the disk if source and destination are on the same guest
6285                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6286                 last if !$@;
6287                 die $@ if $@ !~ m/cannot be completed/;
6288             }
6289             sleep 1;
6290         }
6291
6292
6293     };
6294     my $err = $@;
6295
6296     my $cancel_job = sub {
6297         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6298         while (1) {
6299             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6300             my $stat = @$stats[0];
6301             last if !$stat;
6302             sleep 1;
6303         }
6304     };
6305
6306     if ($err) {
6307         eval { &$cancel_job(); };
6308         die "mirroring error: $err";
6309     }
6310
6311     if ($vmiddst != $vmid) {
6312         # if we clone a disk for a new target vm, we don't switch the disk
6313         &$cancel_job(); # so we call block-job-cancel
6314     }
6315 }
6316
6317 sub clone_disk {
6318     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6319         $newvmid, $storage, $format, $full, $newvollist) = @_;
6320
6321     my $newvolid;
6322
6323     if (!$full) {
6324         print "create linked clone of drive $drivename ($drive->{file})\n";
6325         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6326         push @$newvollist, $newvolid;
6327     } else {
6328         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6329         $storeid = $storage if $storage;
6330
6331         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6332         if (!$format) {
6333             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6334             $format = qemu_img_format($scfg, $volname);
6335         }
6336
6337         # test if requested format is supported - else use default
6338         my $supported = grep { $_ eq $format } @$validFormats;
6339         $format = $defFormat if !$supported;
6340
6341         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6342
6343         print "create full clone of drive $drivename ($drive->{file})\n";
6344         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6345         push @$newvollist, $newvolid;
6346
6347         PVE::Storage::activate_volumes($storecfg, $newvollist);
6348
6349         if (!$running || $snapname) {
6350             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6351         } else {
6352             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6353         }
6354     }
6355
6356     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6357
6358     my $disk = $drive;
6359     $disk->{format} = undef;
6360     $disk->{file} = $newvolid;
6361     $disk->{size} = $size;
6362
6363     return $disk;
6364 }
6365
6366 # this only works if VM is running
6367 sub get_current_qemu_machine {
6368     my ($vmid) = @_;
6369
6370     my $cmd = { execute => 'query-machines', arguments => {} };
6371     my $res = vm_qmp_command($vmid, $cmd);
6372
6373     my ($current, $default);
6374     foreach my $e (@$res) {
6375         $default = $e->{name} if $e->{'is-default'};
6376         $current = $e->{name} if $e->{'is-current'};
6377     }
6378
6379     # fallback to the default machine if current is not supported by qemu
6380     return $current || $default || 'pc';
6381 }
6382
6383 sub qemu_machine_feature_enabled {
6384     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6385
6386     my $current_major;
6387     my $current_minor;
6388
6389     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6390
6391         $current_major = $3;
6392         $current_minor = $4;
6393
6394     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6395
6396         $current_major = $1;
6397         $current_minor = $2;
6398     }
6399
6400     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6401
6402
6403 }
6404
6405 sub qemu_machine_pxe {
6406     my ($vmid, $conf, $machine) = @_;
6407
6408     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6409
6410     foreach my $opt (keys %$conf) {
6411         next if $opt !~ m/^net(\d+)$/;
6412         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6413         next if !$net;
6414         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6415         return $machine.".pxe" if $romfile =~ m/pxe/;
6416         last;
6417     }
6418
6419 }
6420
6421 sub qemu_use_old_bios_files {
6422     my ($machine_type) = @_;
6423
6424     return if !$machine_type;
6425
6426     my $use_old_bios_files = undef;
6427
6428     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6429         $machine_type = $1;
6430         $use_old_bios_files = 1;
6431     } else {
6432         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6433         # load new efi bios files on migration. So this hack is required to allow
6434         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6435         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6436         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6437     }
6438
6439     return ($use_old_bios_files, $machine_type);
6440 }
6441
6442 sub lspci {
6443
6444     my $devices = {};
6445
6446     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6447             my (undef, $id, $function) = @_;
6448             my $res = { id => $id, function => $function};
6449             push @{$devices->{$id}}, $res;
6450     });
6451
6452     return $devices;
6453 }
6454
6455 sub vm_iothreads_list {
6456     my ($vmid) = @_;
6457
6458     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6459
6460     my $iothreads = {};
6461     foreach my $iothread (@$res) {
6462         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6463     }
6464
6465     return $iothreads;
6466 }
6467
6468 sub scsihw_infos {
6469     my ($conf, $drive) = @_;
6470
6471     my $maxdev = 0;
6472
6473     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6474         $maxdev = 7;
6475     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6476         $maxdev = 1;
6477     } else {
6478         $maxdev = 256;
6479     }
6480
6481     my $controller = int($drive->{index} / $maxdev);
6482     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6483
6484     return ($maxdev, $controller, $controller_prefix);
6485 }
6486
6487 # bash completion helper
6488
6489 sub complete_backup_archives {
6490     my ($cmdname, $pname, $cvalue) = @_;
6491
6492     my $cfg = PVE::Storage::config();
6493
6494     my $storeid;
6495
6496     if ($cvalue =~ m/^([^:]+):/) {
6497         $storeid = $1;
6498     }
6499
6500     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6501
6502     my $res = [];
6503     foreach my $id (keys %$data) {
6504         foreach my $item (@{$data->{$id}}) {
6505             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6506             push @$res, $item->{volid} if defined($item->{volid});
6507         }
6508     }
6509
6510     return $res;
6511 }
6512
6513 my $complete_vmid_full = sub {
6514     my ($running) = @_;
6515
6516     my $idlist = vmstatus();
6517
6518     my $res = [];
6519
6520     foreach my $id (keys %$idlist) {
6521         my $d = $idlist->{$id};
6522         if (defined($running)) {
6523             next if $d->{template};
6524             next if $running && $d->{status} ne 'running';
6525             next if !$running && $d->{status} eq 'running';
6526         }
6527         push @$res, $id;
6528
6529     }
6530     return $res;
6531 };
6532
6533 sub complete_vmid {
6534     return &$complete_vmid_full();
6535 }
6536
6537 sub complete_vmid_stopped {
6538     return &$complete_vmid_full(0);
6539 }
6540
6541 sub complete_vmid_running {
6542     return &$complete_vmid_full(1);
6543 }
6544
6545 sub complete_storage {
6546
6547     my $cfg = PVE::Storage::config();
6548     my $ids = $cfg->{ids};
6549
6550     my $res = [];
6551     foreach my $sid (keys %$ids) {
6552         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6553         next if !$ids->{$sid}->{content}->{images};
6554         push @$res, $sid;
6555     }
6556
6557     return $res;
6558 }
6559
6560 1;