PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 385         maxLength => 256,
 386         optional => 1,
 387     },
 388     protection => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 392         default => 0,
 393     },
 394     bios => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [ qw(seabios ovmf) ],
 398         description => "Select BIOS implementation.",
 399         default => 'seabios',
 400     },
 401 };
 402
 403 # what about other qemu settings ?
 404 #cpu => 'string',
 405 #machine => 'string',
 406 #fda => 'file',
 407 #fdb => 'file',
 408 #mtdblock => 'file',
 409 #sd => 'file',
 410 #pflash => 'file',
 411 #snapshot => 'bool',
 412 #bootp => 'file',
 413 ##tftp => 'dir',
 414 ##smb => 'dir',
 415 #kernel => 'file',
 416 #append => 'string',
 417 #initrd => 'file',
 418 ##soundhw => 'string',
 419
 420 while (my ($k, $v) = each %$confdesc) {
 421     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 422 }
 423
 424 my $MAX_IDE_DISKS = 4;
 425 my $MAX_SCSI_DISKS = 14;
 426 my $MAX_VIRTIO_DISKS = 16;
 427 my $MAX_SATA_DISKS = 6;
 428 my $MAX_USB_DEVICES = 5;
 429 my $MAX_NETS = 32;
 430 my $MAX_UNUSED_DISKS = 8;
 431 my $MAX_HOSTPCI_DEVICES = 4;
 432 my $MAX_SERIAL_PORTS = 4;
 433 my $MAX_PARALLEL_PORTS = 3;
 434 my $MAX_NUMA = 8;
 435 my $MAX_MEM = 4194304;
 436 my $STATICMEM = 1024;
 437
 438 my $numadesc = {
 439     optional => 1,
 440     type => 'string', format => 'pve-qm-numanode',
 441     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 442     description => "numa topology",
 443 };
 444 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 445
 446 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 447     $confdesc->{"numa$i"} = $numadesc;
 448 }
 449
 450 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 451                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 452                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 453 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 454
 455 my $netdesc = {
 456     optional => 1,
 457     type => 'string', format => 'pve-qm-net',
 458     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 459     description => <<EODESCR,
 460 Specify network devices.
 461
 462 MODEL is one of: $nic_model_list_txt
 463
 464 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 465 automatically generated if not specified.
 466
 467 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 468
 469 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 470
 471 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 472
 473 10.0.2.2   Gateway
 474 10.0.2.3   DNS Server
 475 10.0.2.4   SMB Server
 476
 477 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 478
 479 EODESCR
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 482
 483 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 484     $confdesc->{"net$i"} = $netdesc;
 485 }
 486
 487 my $drivename_hash;
 488
 489 my $idedesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial][,model=model]',
 493     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 496
 497 my $scsidesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 501     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 504
 505 my $satadesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial]',
 509     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 512
 513 my $virtiodesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-drive',
 516     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,serial=serial]',
 517     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 518 };
 519 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 520
 521 my $usbdesc = {
 522     optional => 1,
 523     type => 'string', format => 'pve-qm-usb-device',
 524     typetext => 'host=HOSTUSBDEVICE|spice',
 525     description => <<EODESCR,
 526 Configure an USB device (n is 0 to 4). This can be used to
 527 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 528
 529 'bus-port(.port)*' (decimal numbers) or
 530 'vendor_id:product_id' (hexadeciaml numbers)
 531
 532 You can use the 'lsusb -t' command to list existing usb devices.
 533
 534 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 535
 536 The value 'spice' can be used to add a usb redirection devices for spice.
 537
 538 EODESCR
 539 };
 540 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 541
 542 my $hostpcidesc = {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-hostpci',
 545         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 546         description => <<EODESCR,
 547 Map host pci devices. HOSTPCIDEVICE syntax is:
 548
 549 'bus:dev.func' (hexadecimal numbers)
 550
 551 You can us the 'lspci' command to list existing pci devices.
 552
 553 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 561
 562 my $serialdesc = {
 563         optional => 1,
 564         type => 'string',
 565         pattern => '(/dev/.+|socket)',
 566         description =>  <<EODESCR,
 567 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 568
 569 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 570
 571 Experimental: user reported problems with this option.
 572 EODESCR
 573 };
 574
 575 my $paralleldesc= {
 576         optional => 1,
 577         type => 'string',
 578         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 579         description =>  <<EODESCR,
 580 Map host parallel devices (n is 0 to 2).
 581
 582 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 583
 584 Experimental: user reported problems with this option.
 585 EODESCR
 586 };
 587
 588 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 589     $confdesc->{"parallel$i"} = $paralleldesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 593     $confdesc->{"serial$i"} = $serialdesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 597     $confdesc->{"hostpci$i"} = $hostpcidesc;
 598 }
 599
 600 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 601     $drivename_hash->{"ide$i"} = 1;
 602     $confdesc->{"ide$i"} = $idedesc;
 603 }
 604
 605 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 606     $drivename_hash->{"sata$i"} = 1;
 607     $confdesc->{"sata$i"} = $satadesc;
 608 }
 609
 610 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 611     $drivename_hash->{"scsi$i"} = 1;
 612     $confdesc->{"scsi$i"} = $scsidesc ;
 613 }
 614
 615 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 616     $drivename_hash->{"virtio$i"} = 1;
 617     $confdesc->{"virtio$i"} = $virtiodesc;
 618 }
 619
 620 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 621     $confdesc->{"usb$i"} = $usbdesc;
 622 }
 623
 624 my $unuseddesc = {
 625     optional => 1,
 626     type => 'string', format => 'pve-volume-id',
 627     description => "Reference to unused volumes.",
 628 };
 629
 630 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 631     $confdesc->{"unused$i"} = $unuseddesc;
 632 }
 633
 634 my $kvm_api_version = 0;
 635
 636 sub kvm_version {
 637
 638     return $kvm_api_version if $kvm_api_version;
 639
 640     my $fh = IO::File->new("</dev/kvm") ||
 641         return 0;
 642
 643     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 644         $kvm_api_version = $v;
 645     }
 646
 647     $fh->close();
 648
 649     return  $kvm_api_version;
 650 }
 651
 652 my $kvm_user_version;
 653
 654 sub kvm_user_version {
 655
 656     return $kvm_user_version if $kvm_user_version;
 657
 658     $kvm_user_version = 'unknown';
 659
 660     my $tmp = `kvm -help 2>/dev/null`;
 661
 662     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 663         $kvm_user_version = $2;
 664     }
 665
 666     return $kvm_user_version;
 667
 668 }
 669
 670 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 671
 672 sub disknames {
 673     # order is important - used to autoselect boot disk
 674     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 675             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 676             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 677             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 678 }
 679
 680 sub valid_drivename {
 681     my $dev = shift;
 682
 683     return defined($drivename_hash->{$dev});
 684 }
 685
 686 sub option_exists {
 687     my $key = shift;
 688     return defined($confdesc->{$key});
 689 }
 690
 691 sub nic_models {
 692     return $nic_model_list;
 693 }
 694
 695 sub os_list_description {
 696
 697     return {
 698         other => 'Other',
 699         wxp => 'Windows XP',
 700         w2k => 'Windows 2000',
 701         w2k3 =>, 'Windows 2003',
 702         w2k8 => 'Windows 2008',
 703         wvista => 'Windows Vista',
 704         win7 => 'Windows 7',
 705         win8 => 'Windows 8/2012',
 706         l24 => 'Linux 2.4',
 707         l26 => 'Linux 2.6',
 708     };
 709 }
 710
 711 my $cdrom_path;
 712
 713 sub get_cdrom_path {
 714
 715     return  $cdrom_path if $cdrom_path;
 716
 717     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 718     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 719     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 720 }
 721
 722 sub get_iso_path {
 723     my ($storecfg, $vmid, $cdrom) = @_;
 724
 725     if ($cdrom eq 'cdrom') {
 726         return get_cdrom_path();
 727     } elsif ($cdrom eq 'none') {
 728         return '';
 729     } elsif ($cdrom =~ m|^/|) {
 730         return $cdrom;
 731     } else {
 732         return PVE::Storage::path($storecfg, $cdrom);
 733     }
 734 }
 735
 736 # try to convert old style file names to volume IDs
 737 sub filename_to_volume_id {
 738     my ($vmid, $file, $media) = @_;
 739
 740     if (!($file eq 'none' || $file eq 'cdrom' ||
 741           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 742
 743         return undef if $file =~ m|/|;
 744
 745         if ($media && $media eq 'cdrom') {
 746             $file = "local:iso/$file";
 747         } else {
 748             $file = "local:$vmid/$file";
 749         }
 750     }
 751
 752     return $file;
 753 }
 754
 755 sub verify_media_type {
 756     my ($opt, $vtype, $media) = @_;
 757
 758     return if !$media;
 759
 760     my $etype;
 761     if ($media eq 'disk') {
 762         $etype = 'images';
 763     } elsif ($media eq 'cdrom') {
 764         $etype = 'iso';
 765     } else {
 766         die "internal error";
 767     }
 768
 769     return if ($vtype eq $etype);
 770
 771     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 772 }
 773
 774 sub cleanup_drive_path {
 775     my ($opt, $storecfg, $drive) = @_;
 776
 777     # try to convert filesystem paths to volume IDs
 778
 779     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 780         ($drive->{file} !~ m|^/dev/.+|) &&
 781         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 782         ($drive->{file} !~ m/^\d+$/)) {
 783         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 784         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 785         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 786         verify_media_type($opt, $vtype, $drive->{media});
 787         $drive->{file} = $volid;
 788     }
 789
 790     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 791 }
 792
 793 sub create_conf_nolock {
 794     my ($vmid, $settings) = @_;
 795
 796     my $filename = config_file($vmid);
 797
 798     die "configuration file '$filename' already exists\n" if -f $filename;
 799
 800     my $defaults = load_defaults();
 801
 802     $settings->{name} = "vm$vmid" if !$settings->{name};
 803     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 804
 805     my $data = '';
 806     foreach my $opt (keys %$settings) {
 807         next if !$confdesc->{$opt};
 808
 809         my $value = $settings->{$opt};
 810         next if !$value;
 811
 812         $data .= "$opt: $value\n";
 813     }
 814
 815     PVE::Tools::file_set_contents($filename, $data);
 816 }
 817
 818 sub parse_hotplug_features {
 819     my ($data) = @_;
 820
 821     my $res = {};
 822
 823     return $res if $data eq '0';
 824
 825     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 826
 827     foreach my $feature (PVE::Tools::split_list($data)) {
 828         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 829             $res->{$1} = 1;
 830         } else {
 831             warn "ignoring unknown hotplug feature '$feature'\n";
 832         }
 833     }
 834     return $res;
 835 }
 836
 837 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 838 sub pve_verify_hotplug_features {
 839     my ($value, $noerr) = @_;
 840
 841     return $value if parse_hotplug_features($value);
 842
 843     return undef if $noerr;
 844
 845     die "unable to parse hotplug option\n";
 846 }
 847
 848 my $parse_size = sub {
 849     my ($value) = @_;
 850
 851     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 852     my ($size, $unit) = ($1, $3);
 853     if ($unit) {
 854         if ($unit eq 'K') {
 855             $size = $size * 1024;
 856         } elsif ($unit eq 'M') {
 857             $size = $size * 1024 * 1024;
 858         } elsif ($unit eq 'G') {
 859             $size = $size * 1024 * 1024 * 1024;
 860         }
 861     }
 862     return int($size);
 863 };
 864
 865 my $format_size = sub {
 866     my ($size) = @_;
 867
 868     $size = int($size);
 869
 870     my $kb = int($size/1024);
 871     return $size if $kb*1024 != $size;
 872
 873     my $mb = int($kb/1024);
 874     return "${kb}K" if $mb*1024 != $kb;
 875
 876     my $gb = int($mb/1024);
 877     return "${mb}M" if $gb*1024 != $mb;
 878
 879     return "${gb}G";
 880 };
 881
 882 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 883 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 884 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 885 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
 886 #        [,iothread=on][,serial=serial][,model=model]
 887
 888 sub parse_drive {
 889     my ($key, $data) = @_;
 890
 891     my $res = {};
 892
 893     # $key may be undefined - used to verify JSON parameters
 894     if (!defined($key)) {
 895         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 896         $res->{index} = 0;
 897     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 898         $res->{interface} = $1;
 899         $res->{index} = $2;
 900     } else {
 901         return undef;
 902     }
 903
 904     foreach my $p (split (/,/, $data)) {
 905         next if $p =~ m/^\s*$/;
 906
 907         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|detect_zeroes|iothread|queues|serial|model)=(.+)$/) {
 908             my ($k, $v) = ($1, $2);
 909
 910             $k = 'file' if $k eq 'volume';
 911
 912             return undef if defined $res->{$k};
 913
 914             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 915                 return undef if !$v || $v !~ m/^\d+/;
 916                 $k = "m$k";
 917                 $v = sprintf("%.3f", $v / (1024*1024));
 918             }
 919             $res->{$k} = $v;
 920         } else {
 921             if (!$res->{file} && $p !~ m/=/) {
 922                 $res->{file} = $p;
 923             } else {
 924                 return undef;
 925             }
 926         }
 927     }
 928
 929     return undef if !$res->{file};
 930
 931     return undef if $res->{cache} &&
 932         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 933     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 934     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 935     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 936     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 937     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 938     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 939     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 940     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 941     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 942     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 943     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 944     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 945     return undef if $res->{detect_zeroes} && $res->{detect_zeroes} !~ m/^(on|off)$/;
 946     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 947     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 948
 949     return undef if $res->{mbps_rd} && $res->{mbps};
 950     return undef if $res->{mbps_wr} && $res->{mbps};
 951
 952     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 957     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 958
 959     return undef if $res->{iops_rd} && $res->{iops};
 960     return undef if $res->{iops_wr} && $res->{iops};
 961
 962
 963     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 964     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 966     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 968     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 969
 970     if ($res->{size}) {
 971         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 972     }
 973
 974     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 975         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 976         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 977         return undef if $res->{interface} eq 'virtio';
 978     }
 979
 980     # rerror does not work with scsi drives
 981     if ($res->{rerror}) {
 982         return undef if $res->{interface} eq 'scsi';
 983     }
 984
 985     return $res;
 986 }
 987
 988 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 989
 990 sub print_drive {
 991     my ($vmid, $drive) = @_;
 992
 993     my $opts = '';
 994     foreach my $o (@qemu_drive_options, qw(mbps mbps_rd mbps_wr mbps_max mbps_rd_max mbps_wr_max backup iothread queues detect_zeroes)) {
 995         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 996     }
 997
 998     if ($drive->{size}) {
 999         $opts .= ",size=" . &$format_size($drive->{size});
1000     }
1001
1002     if (my $model = $drive->{model}) {
1003         $opts .= ",model=$model";
1004     }
1005
1006     return "$drive->{file}$opts";
1007 }
1008
1009 sub scsi_inquiry {
1010     my($fh, $noerr) = @_;
1011
1012     my $SG_IO = 0x2285;
1013     my $SG_GET_VERSION_NUM = 0x2282;
1014
1015     my $versionbuf = "\x00" x 8;
1016     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1017     if (!$ret) {
1018         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1019         return undef;
1020     }
1021     my $version = unpack("I", $versionbuf);
1022     if ($version < 30000) {
1023         die "scsi generic interface too old\n"  if !$noerr;
1024         return undef;
1025     }
1026
1027     my $buf = "\x00" x 36;
1028     my $sensebuf = "\x00" x 8;
1029     my $cmd = pack("C x3 C x1", 0x12, 36);
1030
1031     # see /usr/include/scsi/sg.h
1032     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1033
1034     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1035                       length($sensebuf), 0, length($buf), $buf,
1036                       $cmd, $sensebuf, 6000);
1037
1038     $ret = ioctl($fh, $SG_IO, $packet);
1039     if (!$ret) {
1040         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1041         return undef;
1042     }
1043
1044     my @res = unpack($sg_io_hdr_t, $packet);
1045     if ($res[17] || $res[18]) {
1046         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1047         return undef;
1048     }
1049
1050     my $res = {};
1051     (my $byte0, my $byte1, $res->{vendor},
1052      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1053
1054     $res->{removable} = $byte1 & 128 ? 1 : 0;
1055     $res->{type} = $byte0 & 31;
1056
1057     return $res;
1058 }
1059
1060 sub path_is_scsi {
1061     my ($path) = @_;
1062
1063     my $fh = IO::File->new("+<$path") || return undef;
1064     my $res = scsi_inquiry($fh, 1);
1065     close($fh);
1066
1067     return $res;
1068 }
1069
1070 sub machine_type_is_q35 {
1071     my ($conf) = @_;
1072
1073     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1074 }
1075
1076 sub print_tabletdevice_full {
1077     my ($conf) = @_;
1078
1079     my $q35 = machine_type_is_q35($conf);
1080
1081     # we use uhci for old VMs because tablet driver was buggy in older qemu
1082     my $usbbus = $q35 ? "ehci" : "uhci";
1083
1084     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1085 }
1086
1087 sub print_drivedevice_full {
1088     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1089
1090     my $device = '';
1091     my $maxdev = 0;
1092
1093     if ($drive->{interface} eq 'virtio') {
1094         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1095         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1096         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1097     } elsif ($drive->{interface} eq 'scsi') {
1098
1099         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1100         my $unit = $drive->{index} % $maxdev;
1101         my $devicetype = 'hd';
1102         my $path = '';
1103         if (drive_is_cdrom($drive)) {
1104               $devicetype = 'cd';
1105         } else {
1106               if ($drive->{file} =~ m|^/|) {
1107                   $path = $drive->{file};
1108               } else {
1109                   $path = PVE::Storage::path($storecfg, $drive->{file});
1110               }
1111
1112               if($path =~ m/^iscsi\:\/\//){
1113                   $devicetype = 'generic';
1114               } else {
1115                   if (my $info = path_is_scsi($path)) {
1116                       if ($info->{type} == 0) {
1117                           $devicetype = 'block';
1118                       } elsif ($info->{type} == 1) { # tape
1119                           $devicetype = 'generic';
1120                       }
1121                   }
1122               }
1123          }
1124
1125         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1126             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1127         } else {
1128             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         }
1130
1131     } elsif ($drive->{interface} eq 'ide'){
1132         $maxdev = 2;
1133         my $controller = int($drive->{index} / $maxdev);
1134         my $unit = $drive->{index} % $maxdev;
1135         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1136
1137         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1138         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1139             $device .= ",model=$model";
1140         }
1141     } elsif ($drive->{interface} eq 'sata'){
1142         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1143         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1144         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1145     } elsif ($drive->{interface} eq 'usb') {
1146         die "implement me";
1147         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1148     } else {
1149         die "unsupported interface type";
1150     }
1151
1152     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1153
1154     return $device;
1155 }
1156
1157 sub get_initiator_name {
1158     my $initiator;
1159
1160     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1161     while (defined(my $line = <$fh>)) {
1162         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1163         $initiator = $1;
1164         last;
1165     }
1166     $fh->close();
1167
1168     return $initiator;
1169 }
1170
1171 sub print_drive_full {
1172     my ($storecfg, $vmid, $drive) = @_;
1173
1174     my $path;
1175     my $volid = $drive->{file};
1176     my $format;
1177
1178     if (drive_is_cdrom($drive)) {
1179         $path = get_iso_path($storecfg, $vmid, $volid);
1180     } else {
1181         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1182         if ($storeid) {
1183             $path = PVE::Storage::path($storecfg, $volid);
1184             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1185             $format = qemu_img_format($scfg, $volname);
1186         } else {
1187             $path = $volid;
1188         }
1189    }
1190
1191     my $opts = '';
1192     foreach my $o (@qemu_drive_options) {
1193         next if $o eq 'bootindex';
1194         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1195     }
1196
1197     $opts .= ",format=$format" if $format && !$drive->{format};
1198
1199     foreach my $o (qw(bps bps_rd bps_wr)) {
1200         my $v = $drive->{"m$o"};
1201         $opts .= ",$o=" . int($v*1024*1024) if $v;
1202     }
1203
1204     my $cache_direct = 0;
1205
1206     if (my $cache = $drive->{cache}) {
1207         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1208     } elsif (!drive_is_cdrom($drive)) {
1209         $opts .= ",cache=none";
1210         $cache_direct = 1;
1211     }
1212
1213     # aio native works only with O_DIRECT
1214     if (!$drive->{aio}) {
1215         if($cache_direct) {
1216             $opts .= ",aio=native";
1217         } else {
1218             $opts .= ",aio=threads";
1219         }
1220     }
1221
1222     if (!drive_is_cdrom($drive)) {
1223         my $detectzeroes;
1224         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1225             $detectzeroes = 'off';
1226         } elsif ($drive->{discard}) {
1227             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1228         } else {
1229             # This used to be our default with discard not being specified:
1230             $detectzeroes = 'on';
1231         }
1232         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1233     }
1234
1235     my $pathinfo = $path ? "file=$path," : '';
1236
1237     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1238 }
1239
1240 sub print_netdevice_full {
1241     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1242
1243     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1244
1245     my $device = $net->{model};
1246     if ($net->{model} eq 'virtio') {
1247          $device = 'virtio-net-pci';
1248      };
1249
1250     my $pciaddr = print_pci_addr("$netid", $bridges);
1251     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1252     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1253         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1254         my $vectors = $net->{queues} * 2 + 2;
1255         $tmpstr .= ",vectors=$vectors,mq=on";
1256     }
1257     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1258
1259     if ($use_old_bios_files) {
1260         my $romfile;
1261         if ($device eq 'virtio-net-pci') {
1262             $romfile = 'pxe-virtio.rom';
1263         } elsif ($device eq 'e1000') {
1264             $romfile = 'pxe-e1000.rom';
1265         } elsif ($device eq 'ne2k') {
1266             $romfile = 'pxe-ne2k_pci.rom';
1267         } elsif ($device eq 'pcnet') {
1268             $romfile = 'pxe-pcnet.rom';
1269         } elsif ($device eq 'rtl8139') {
1270             $romfile = 'pxe-rtl8139.rom';
1271         }
1272         $tmpstr .= ",romfile=$romfile" if $romfile;
1273     }
1274
1275     return $tmpstr;
1276 }
1277
1278 sub print_netdev_full {
1279     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1280
1281     my $i = '';
1282     if ($netid =~ m/^net(\d+)$/) {
1283         $i = int($1);
1284     }
1285
1286     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1287
1288     my $ifname = "tap${vmid}i$i";
1289
1290     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1291     die "interface name '$ifname' is too long (max 15 character)\n"
1292         if length($ifname) >= 16;
1293
1294     my $vhostparam = '';
1295     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1296
1297     my $vmname = $conf->{name} || "vm$vmid";
1298
1299     my $netdev = "";
1300     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1301
1302     if ($net->{bridge}) {
1303         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1304     } else {
1305         $netdev = "type=user,id=$netid,hostname=$vmname";
1306     }
1307
1308     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1309
1310     return $netdev;
1311 }
1312
1313 sub drive_is_cdrom {
1314     my ($drive) = @_;
1315
1316     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1317
1318 }
1319
1320 sub parse_numa {
1321     my ($data) = @_;
1322
1323     my $res = {};
1324
1325     foreach my $kvp (split(/,/, $data)) {
1326
1327         if ($kvp =~ m/^memory=(\S+)$/) {
1328             $res->{memory} = $1;
1329         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1330             $res->{policy} = $1;
1331         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1332             $res->{cpus}->{start} = $1;
1333             $res->{cpus}->{end} = $3;
1334         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1335             $res->{hostnodes}->{start} = $1;
1336             $res->{hostnodes}->{end} = $3;
1337         } else {
1338             return undef;
1339         }
1340     }
1341
1342     return $res;
1343 }
1344
1345 sub parse_hostpci {
1346     my ($value) = @_;
1347
1348     return undef if !$value;
1349
1350
1351     my @list = split(/,/, $value);
1352     my $found;
1353
1354     my $res = {};
1355     foreach my $kv (@list) {
1356
1357         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1358             $found = 1;
1359             if(defined($4)){
1360                 push @{$res->{pciid}}, { id => $2 , function => $4};
1361
1362             }else{
1363                 my $pcidevices = lspci($2);
1364                 $res->{pciid} = $pcidevices->{$2};
1365             }
1366         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1367             $res->{rombar} = $1;
1368         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1369             $res->{'x-vga'} = $1;
1370         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1371             $res->{pcie} = 1 if $1 == 1;
1372         } else {
1373             warn "unknown hostpci setting '$kv'\n";
1374         }
1375     }
1376
1377     return undef if !$found;
1378
1379     return $res;
1380 }
1381
1382 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1383 sub parse_net {
1384     my ($data) = @_;
1385
1386     my $res = {};
1387
1388     foreach my $kvp (split(/,/, $data)) {
1389
1390         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1391             my $model = lc($1);
1392             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1393             $res->{model} = $model;
1394             $res->{macaddr} = $mac;
1395         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1396             $res->{bridge} = $1;
1397         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1398             $res->{queues} = $1;
1399         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1400             $res->{rate} = $1;
1401         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1402             $res->{tag} = $1;
1403         } elsif ($kvp =~ m/^firewall=([01])$/) {
1404             $res->{firewall} = $1;
1405         } elsif ($kvp =~ m/^link_down=([01])$/) {
1406             $res->{link_down} = $1;
1407         } else {
1408             return undef;
1409         }
1410
1411     }
1412
1413     return undef if !$res->{model};
1414
1415     return $res;
1416 }
1417
1418 sub print_net {
1419     my $net = shift;
1420
1421     my $res = "$net->{model}";
1422     $res .= "=$net->{macaddr}" if $net->{macaddr};
1423     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1424     $res .= ",rate=$net->{rate}" if $net->{rate};
1425     $res .= ",tag=$net->{tag}" if $net->{tag};
1426     $res .= ",firewall=1" if $net->{firewall};
1427     $res .= ",link_down=1" if $net->{link_down};
1428     $res .= ",queues=$net->{queues}" if $net->{queues};
1429
1430     return $res;
1431 }
1432
1433 sub add_random_macs {
1434     my ($settings) = @_;
1435
1436     foreach my $opt (keys %$settings) {
1437         next if $opt !~ m/^net(\d+)$/;
1438         my $net = parse_net($settings->{$opt});
1439         next if !$net;
1440         $settings->{$opt} = print_net($net);
1441     }
1442 }
1443
1444 sub add_unused_volume {
1445     my ($config, $volid) = @_;
1446
1447     my $key;
1448     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1449         my $test = "unused$ind";
1450         if (my $vid = $config->{$test}) {
1451             return if $vid eq $volid; # do not add duplicates
1452         } else {
1453             $key = $test;
1454         }
1455     }
1456
1457     die "To many unused volume - please delete them first.\n" if !$key;
1458
1459     $config->{$key} = $volid;
1460
1461     return $key;
1462 }
1463
1464 sub vm_is_volid_owner {
1465     my ($storecfg, $vmid, $volid) = @_;
1466
1467     if ($volid !~  m|^/|) {
1468         my ($path, $owner);
1469         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1470         if ($owner && ($owner == $vmid)) {
1471             return 1;
1472         }
1473     }
1474
1475     return undef;
1476 }
1477
1478 sub split_flagged_list {
1479     my $text = shift || '';
1480     $text =~ s/[,;]/ /g;
1481     $text =~ s/^\s+//;
1482     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1483 }
1484
1485 sub join_flagged_list {
1486     my ($how, $lst) = @_;
1487     join $how, map { $lst->{$_} . $_ } keys %$lst;
1488 }
1489
1490 sub vmconfig_delete_pending_option {
1491     my ($conf, $key, $force) = @_;
1492
1493     delete $conf->{pending}->{$key};
1494     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1495     $pending_delete_hash->{$key} = $force ? '!' : '';
1496     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1497 }
1498
1499 sub vmconfig_undelete_pending_option {
1500     my ($conf, $key) = @_;
1501
1502     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1503     delete $pending_delete_hash->{$key};
1504
1505     if (%$pending_delete_hash) {
1506         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1507     } else {
1508         delete $conf->{pending}->{delete};
1509     }
1510 }
1511
1512 sub vmconfig_register_unused_drive {
1513     my ($storecfg, $vmid, $conf, $drive) = @_;
1514
1515     if (!drive_is_cdrom($drive)) {
1516         my $volid = $drive->{file};
1517         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1518             add_unused_volume($conf, $volid, $vmid);
1519         }
1520     }
1521 }
1522
1523 sub vmconfig_cleanup_pending {
1524     my ($conf) = @_;
1525
1526     # remove pending changes when nothing changed
1527     my $changes;
1528     foreach my $opt (keys %{$conf->{pending}}) {
1529         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1530             $changes = 1;
1531             delete $conf->{pending}->{$opt};
1532         }
1533     }
1534
1535     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1536     my $pending_delete_hash = {};
1537     while (my ($opt, $force) = each %$current_delete_hash) {
1538         if (defined($conf->{$opt})) {
1539             $pending_delete_hash->{$opt} = $force;
1540         } else {
1541             $changes = 1;
1542         }
1543     }
1544
1545     if (%$pending_delete_hash) {
1546         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1547     } else {
1548         delete $conf->{pending}->{delete};
1549     }
1550
1551     return $changes;
1552 }
1553
1554 my $valid_smbios1_options = {
1555     manufacturer => '\S+',
1556     product => '\S+',
1557     version => '\S+',
1558     serial => '\S+',
1559     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1560     sku => '\S+',
1561     family => '\S+',
1562 };
1563
1564 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1565 sub parse_smbios1 {
1566     my ($data) = @_;
1567
1568     my $res = {};
1569
1570     foreach my $kvp (split(/,/, $data)) {
1571         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1572         my ($k, $v) = split(/=/, $kvp);
1573         return undef if !defined($k) || !defined($v);
1574         return undef if !$valid_smbios1_options->{$k};
1575         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1576         $res->{$k} = $v;
1577     }
1578
1579     return $res;
1580 }
1581
1582 sub print_smbios1 {
1583     my ($smbios1) = @_;
1584
1585     my $data = '';
1586     foreach my $k (keys %$smbios1) {
1587         next if !defined($smbios1->{$k});
1588         next if !$valid_smbios1_options->{$k};
1589         $data .= ',' if $data;
1590         $data .= "$k=$smbios1->{$k}";
1591     }
1592     return $data;
1593 }
1594
1595 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1596 sub verify_smbios1 {
1597     my ($value, $noerr) = @_;
1598
1599     return $value if parse_smbios1($value);
1600
1601     return undef if $noerr;
1602
1603     die "unable to parse smbios (type 1) options\n";
1604 }
1605
1606 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1607 sub verify_bootdisk {
1608     my ($value, $noerr) = @_;
1609
1610     return $value if valid_drivename($value);
1611
1612     return undef if $noerr;
1613
1614     die "invalid boot disk '$value'\n";
1615 }
1616
1617 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1618 sub verify_numa {
1619     my ($value, $noerr) = @_;
1620
1621     return $value if parse_numa($value);
1622
1623     return undef if $noerr;
1624
1625     die "unable to parse numa options\n";
1626 }
1627
1628 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1629 sub verify_net {
1630     my ($value, $noerr) = @_;
1631
1632     return $value if parse_net($value);
1633
1634     return undef if $noerr;
1635
1636     die "unable to parse network options\n";
1637 }
1638
1639 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1640 sub verify_drive {
1641     my ($value, $noerr) = @_;
1642
1643     return $value if parse_drive(undef, $value);
1644
1645     return undef if $noerr;
1646
1647     die "unable to parse drive options\n";
1648 }
1649
1650 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1651 sub verify_hostpci {
1652     my ($value, $noerr) = @_;
1653
1654     return $value if parse_hostpci($value);
1655
1656     return undef if $noerr;
1657
1658     die "unable to parse pci id\n";
1659 }
1660
1661 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1662 sub verify_watchdog {
1663     my ($value, $noerr) = @_;
1664
1665     return $value if parse_watchdog($value);
1666
1667     return undef if $noerr;
1668
1669     die "unable to parse watchdog options\n";
1670 }
1671
1672 sub parse_watchdog {
1673     my ($value) = @_;
1674
1675     return undef if !$value;
1676
1677     my $res = {};
1678
1679     foreach my $p (split(/,/, $value)) {
1680         next if $p =~ m/^\s*$/;
1681
1682         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1683             $res->{model} = $2;
1684         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1685             $res->{action} = $2;
1686         } else {
1687             return undef;
1688         }
1689     }
1690
1691     return $res;
1692 }
1693
1694 sub parse_usb_device {
1695     my ($value) = @_;
1696
1697     return undef if !$value;
1698
1699     my @dl = split(/,/, $value);
1700     my $found;
1701
1702     my $res = {};
1703     foreach my $v (@dl) {
1704         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1705             $found = 1;
1706             $res->{vendorid} = $2;
1707             $res->{productid} = $4;
1708         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1709             $found = 1;
1710             $res->{hostbus} = $1;
1711             $res->{hostport} = $2;
1712         } elsif ($v =~ m/^spice$/) {
1713             $found = 1;
1714             $res->{spice} = 1;
1715         } else {
1716             return undef;
1717         }
1718     }
1719     return undef if !$found;
1720
1721     return $res;
1722 }
1723
1724 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1725 sub verify_usb_device {
1726     my ($value, $noerr) = @_;
1727
1728     return $value if parse_usb_device($value);
1729
1730     return undef if $noerr;
1731
1732     die "unable to parse usb device\n";
1733 }
1734
1735 # add JSON properties for create and set function
1736 sub json_config_properties {
1737     my $prop = shift;
1738
1739     foreach my $opt (keys %$confdesc) {
1740         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1741         $prop->{$opt} = $confdesc->{$opt};
1742     }
1743
1744     return $prop;
1745 }
1746
1747 sub check_type {
1748     my ($key, $value) = @_;
1749
1750     die "unknown setting '$key'\n" if !$confdesc->{$key};
1751
1752     my $type = $confdesc->{$key}->{type};
1753
1754     if (!defined($value)) {
1755         die "got undefined value\n";
1756     }
1757
1758     if ($value =~ m/[\n\r]/) {
1759         die "property contains a line feed\n";
1760     }
1761
1762     if ($type eq 'boolean') {
1763         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1764         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1765         die "type check ('boolean') failed - got '$value'\n";
1766     } elsif ($type eq 'integer') {
1767         return int($1) if $value =~ m/^(\d+)$/;
1768         die "type check ('integer') failed - got '$value'\n";
1769     } elsif ($type eq 'number') {
1770         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1771         die "type check ('number') failed - got '$value'\n";
1772     } elsif ($type eq 'string') {
1773         if (my $fmt = $confdesc->{$key}->{format}) {
1774             if ($fmt eq 'pve-qm-drive') {
1775                 # special case - we need to pass $key to parse_drive()
1776                 my $drive = parse_drive($key, $value);
1777                 return $value if $drive;
1778                 die "unable to parse drive options\n";
1779             }
1780             PVE::JSONSchema::check_format($fmt, $value);
1781             return $value;
1782         }
1783         $value =~ s/^\"(.*)\"$/$1/;
1784         return $value;
1785     } else {
1786         die "internal error"
1787     }
1788 }
1789
1790 sub lock_config_full {
1791     my ($vmid, $timeout, $code, @param) = @_;
1792
1793     my $filename = config_file_lock($vmid);
1794
1795     my $res = lock_file($filename, $timeout, $code, @param);
1796
1797     die $@ if $@;
1798
1799     return $res;
1800 }
1801
1802 sub lock_config_mode {
1803     my ($vmid, $timeout, $shared, $code, @param) = @_;
1804
1805     my $filename = config_file_lock($vmid);
1806
1807     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1808
1809     die $@ if $@;
1810
1811     return $res;
1812 }
1813
1814 sub lock_config {
1815     my ($vmid, $code, @param) = @_;
1816
1817     return lock_config_full($vmid, 10, $code, @param);
1818 }
1819
1820 sub cfs_config_path {
1821     my ($vmid, $node) = @_;
1822
1823     $node = $nodename if !$node;
1824     return "nodes/$node/qemu-server/$vmid.conf";
1825 }
1826
1827 sub check_iommu_support{
1828     #fixme : need to check IOMMU support
1829     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1830
1831     my $iommu=1;
1832     return $iommu;
1833
1834 }
1835
1836 sub config_file {
1837     my ($vmid, $node) = @_;
1838
1839     my $cfspath = cfs_config_path($vmid, $node);
1840     return "/etc/pve/$cfspath";
1841 }
1842
1843 sub config_file_lock {
1844     my ($vmid) = @_;
1845
1846     return "$lock_dir/lock-$vmid.conf";
1847 }
1848
1849 sub touch_config {
1850     my ($vmid) = @_;
1851
1852     my $conf = config_file($vmid);
1853     utime undef, undef, $conf;
1854 }
1855
1856 sub destroy_vm {
1857     my ($storecfg, $vmid, $keep_empty_config) = @_;
1858
1859     my $conffile = config_file($vmid);
1860
1861     my $conf = load_config($vmid);
1862
1863     check_lock($conf);
1864
1865     # only remove disks owned by this VM
1866     foreach_drive($conf, sub {
1867         my ($ds, $drive) = @_;
1868
1869         return if drive_is_cdrom($drive);
1870
1871         my $volid = $drive->{file};
1872
1873         return if !$volid || $volid =~ m|^/|;
1874
1875         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1876         return if !$path || !$owner || ($owner != $vmid);
1877
1878         PVE::Storage::vdisk_free($storecfg, $volid);
1879     });
1880
1881     if ($keep_empty_config) {
1882         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1883     } else {
1884         unlink $conffile;
1885     }
1886
1887     # also remove unused disk
1888     eval {
1889         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1890
1891         eval {
1892             PVE::Storage::foreach_volid($dl, sub {
1893                 my ($volid, $sid, $volname, $d) = @_;
1894                 PVE::Storage::vdisk_free($storecfg, $volid);
1895             });
1896         };
1897         warn $@ if $@;
1898
1899     };
1900     warn $@ if $@;
1901 }
1902
1903 sub load_config {
1904     my ($vmid, $node) = @_;
1905
1906     my $cfspath = cfs_config_path($vmid, $node);
1907
1908     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1909
1910     die "no such VM ('$vmid')\n" if !defined($conf);
1911
1912     return $conf;
1913 }
1914
1915 sub parse_vm_config {
1916     my ($filename, $raw) = @_;
1917
1918     return undef if !defined($raw);
1919
1920     my $res = {
1921         digest => Digest::SHA::sha1_hex($raw),
1922         snapshots => {},
1923         pending => {},
1924     };
1925
1926     $filename =~ m|/qemu-server/(\d+)\.conf$|
1927         || die "got strange filename '$filename'";
1928
1929     my $vmid = $1;
1930
1931     my $conf = $res;
1932     my $descr;
1933     my $section = '';
1934
1935     my @lines = split(/\n/, $raw);
1936     foreach my $line (@lines) {
1937         next if $line =~ m/^\s*$/;
1938
1939         if ($line =~ m/^\[PENDING\]\s*$/i) {
1940             $section = 'pending';
1941             if (defined($descr)) {
1942                 $descr =~ s/\s+$//;
1943                 $conf->{description} = $descr;
1944             }
1945             $descr = undef;
1946             $conf = $res->{$section} = {};
1947             next;
1948
1949         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1950             $section = $1;
1951             if (defined($descr)) {
1952                 $descr =~ s/\s+$//;
1953                 $conf->{description} = $descr;
1954             }
1955             $descr = undef;
1956             $conf = $res->{snapshots}->{$section} = {};
1957             next;
1958         }
1959
1960         if ($line =~ m/^\#(.*)\s*$/) {
1961             $descr = '' if !defined($descr);
1962             $descr .= PVE::Tools::decode_text($1) . "\n";
1963             next;
1964         }
1965
1966         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1967             $descr = '' if !defined($descr);
1968             $descr .= PVE::Tools::decode_text($2);
1969         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1970             $conf->{snapstate} = $1;
1971         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1972             my $key = $1;
1973             my $value = $2;
1974             $conf->{$key} = $value;
1975         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1976             my $value = $1;
1977             if ($section eq 'pending') {
1978                 $conf->{delete} = $value; # we parse this later
1979             } else {
1980                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1981             }
1982         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1983             my $key = $1;
1984             my $value = $2;
1985             eval { $value = check_type($key, $value); };
1986             if ($@) {
1987                 warn "vm $vmid - unable to parse value of '$key' - $@";
1988             } else {
1989                 my $fmt = $confdesc->{$key}->{format};
1990                 if ($fmt && $fmt eq 'pve-qm-drive') {
1991                     my $v = parse_drive($key, $value);
1992                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1993                         $v->{file} = $volid;
1994                         $value = print_drive($vmid, $v);
1995                     } else {
1996                         warn "vm $vmid - unable to parse value of '$key'\n";
1997                         next;
1998                     }
1999                 }
2000
2001                 if ($key eq 'cdrom') {
2002                     $conf->{ide2} = $value;
2003                 } else {
2004                     $conf->{$key} = $value;
2005                 }
2006             }
2007         }
2008     }
2009
2010     if (defined($descr)) {
2011         $descr =~ s/\s+$//;
2012         $conf->{description} = $descr;
2013     }
2014     delete $res->{snapstate}; # just to be sure
2015
2016     return $res;
2017 }
2018
2019 sub write_vm_config {
2020     my ($filename, $conf) = @_;
2021
2022     delete $conf->{snapstate}; # just to be sure
2023
2024     if ($conf->{cdrom}) {
2025         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2026         $conf->{ide2} = $conf->{cdrom};
2027         delete $conf->{cdrom};
2028     }
2029
2030     # we do not use 'smp' any longer
2031     if ($conf->{sockets}) {
2032         delete $conf->{smp};
2033     } elsif ($conf->{smp}) {
2034         $conf->{sockets} = $conf->{smp};
2035         delete $conf->{cores};
2036         delete $conf->{smp};
2037     }
2038
2039     my $used_volids = {};
2040
2041     my $cleanup_config = sub {
2042         my ($cref, $pending, $snapname) = @_;
2043
2044         foreach my $key (keys %$cref) {
2045             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2046                 $key eq 'snapstate' || $key eq 'pending';
2047             my $value = $cref->{$key};
2048             if ($key eq 'delete') {
2049                 die "propertry 'delete' is only allowed in [PENDING]\n"
2050                     if !$pending;
2051                 # fixme: check syntax?
2052                 next;
2053             }
2054             eval { $value = check_type($key, $value); };
2055             die "unable to parse value of '$key' - $@" if $@;
2056
2057             $cref->{$key} = $value;
2058
2059             if (!$snapname && valid_drivename($key)) {
2060                 my $drive = parse_drive($key, $value);
2061                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2062             }
2063         }
2064     };
2065
2066     &$cleanup_config($conf);
2067
2068     &$cleanup_config($conf->{pending}, 1);
2069
2070     foreach my $snapname (keys %{$conf->{snapshots}}) {
2071         die "internal error" if $snapname eq 'pending';
2072         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2073     }
2074
2075     # remove 'unusedX' settings if we re-add a volume
2076     foreach my $key (keys %$conf) {
2077         my $value = $conf->{$key};
2078         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2079             delete $conf->{$key};
2080         }
2081     }
2082
2083     my $generate_raw_config = sub {
2084         my ($conf, $pending) = @_;
2085
2086         my $raw = '';
2087
2088         # add description as comment to top of file
2089         if (defined(my $descr = $conf->{description})) {
2090             if ($descr) {
2091                 foreach my $cl (split(/\n/, $descr)) {
2092                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2093                 }
2094             } else {
2095                 $raw .= "#\n" if $pending;
2096             }
2097         }
2098
2099         foreach my $key (sort keys %$conf) {
2100             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2101             $raw .= "$key: $conf->{$key}\n";
2102         }
2103         return $raw;
2104     };
2105
2106     my $raw = &$generate_raw_config($conf);
2107
2108     if (scalar(keys %{$conf->{pending}})){
2109         $raw .= "\n[PENDING]\n";
2110         $raw .= &$generate_raw_config($conf->{pending}, 1);
2111     }
2112
2113     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2114         $raw .= "\n[$snapname]\n";
2115         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2116     }
2117
2118     return $raw;
2119 }
2120
2121 sub update_config_nolock {
2122     my ($vmid, $conf, $skiplock) = @_;
2123
2124     check_lock($conf) if !$skiplock;
2125
2126     my $cfspath = cfs_config_path($vmid);
2127
2128     PVE::Cluster::cfs_write_file($cfspath, $conf);
2129 }
2130
2131 sub update_config {
2132     my ($vmid, $conf, $skiplock) = @_;
2133
2134     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2135 }
2136
2137 sub load_defaults {
2138
2139     my $res = {};
2140
2141     # we use static defaults from our JSON schema configuration
2142     foreach my $key (keys %$confdesc) {
2143         if (defined(my $default = $confdesc->{$key}->{default})) {
2144             $res->{$key} = $default;
2145         }
2146     }
2147
2148     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2149     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2150
2151     return $res;
2152 }
2153
2154 sub config_list {
2155     my $vmlist = PVE::Cluster::get_vmlist();
2156     my $res = {};
2157     return $res if !$vmlist || !$vmlist->{ids};
2158     my $ids = $vmlist->{ids};
2159
2160     foreach my $vmid (keys %$ids) {
2161         my $d = $ids->{$vmid};
2162         next if !$d->{node} || $d->{node} ne $nodename;
2163         next if !$d->{type} || $d->{type} ne 'qemu';
2164         $res->{$vmid}->{exists} = 1;
2165     }
2166     return $res;
2167 }
2168
2169 # test if VM uses local resources (to prevent migration)
2170 sub check_local_resources {
2171     my ($conf, $noerr) = @_;
2172
2173     my $loc_res = 0;
2174
2175     $loc_res = 1 if $conf->{hostusb}; # old syntax
2176     $loc_res = 1 if $conf->{hostpci}; # old syntax
2177
2178     foreach my $k (keys %$conf) {
2179         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2180         # sockets are safe: they will recreated be on the target side post-migrate
2181         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2182         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2183     }
2184
2185     die "VM uses local resources\n" if $loc_res && !$noerr;
2186
2187     return $loc_res;
2188 }
2189
2190 # check if used storages are available on all nodes (use by migrate)
2191 sub check_storage_availability {
2192     my ($storecfg, $conf, $node) = @_;
2193
2194     foreach_drive($conf, sub {
2195         my ($ds, $drive) = @_;
2196
2197         my $volid = $drive->{file};
2198         return if !$volid;
2199
2200         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2201         return if !$sid;
2202
2203         # check if storage is available on both nodes
2204         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2205         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2206    });
2207 }
2208
2209 # list nodes where all VM images are available (used by has_feature API)
2210 sub shared_nodes {
2211     my ($conf, $storecfg) = @_;
2212
2213     my $nodelist = PVE::Cluster::get_nodelist();
2214     my $nodehash = { map { $_ => 1 } @$nodelist };
2215     my $nodename = PVE::INotify::nodename();
2216
2217     foreach_drive($conf, sub {
2218         my ($ds, $drive) = @_;
2219
2220         my $volid = $drive->{file};
2221         return if !$volid;
2222
2223         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2224         if ($storeid) {
2225             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2226             if ($scfg->{disable}) {
2227                 $nodehash = {};
2228             } elsif (my $avail = $scfg->{nodes}) {
2229                 foreach my $node (keys %$nodehash) {
2230                     delete $nodehash->{$node} if !$avail->{$node};
2231                 }
2232             } elsif (!$scfg->{shared}) {
2233                 foreach my $node (keys %$nodehash) {
2234                     delete $nodehash->{$node} if $node ne $nodename
2235                 }
2236             }
2237         }
2238     });
2239
2240     return $nodehash
2241 }
2242
2243 sub check_lock {
2244     my ($conf) = @_;
2245
2246     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2247 }
2248
2249 sub check_cmdline {
2250     my ($pidfile, $pid) = @_;
2251
2252     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2253     if (defined($fh)) {
2254         my $line = <$fh>;
2255         $fh->close;
2256         return undef if !$line;
2257         my @param = split(/\0/, $line);
2258
2259         my $cmd = $param[0];
2260         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2261
2262         for (my $i = 0; $i < scalar (@param); $i++) {
2263             my $p = $param[$i];
2264             next if !$p;
2265             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2266                 my $p = $param[$i+1];
2267                 return 1 if $p && ($p eq $pidfile);
2268                 return undef;
2269             }
2270         }
2271     }
2272     return undef;
2273 }
2274
2275 sub check_running {
2276     my ($vmid, $nocheck, $node) = @_;
2277
2278     my $filename = config_file($vmid, $node);
2279
2280     die "unable to find configuration file for VM $vmid - no such machine\n"
2281         if !$nocheck && ! -f $filename;
2282
2283     my $pidfile = pidfile_name($vmid);
2284
2285     if (my $fd = IO::File->new("<$pidfile")) {
2286         my $st = stat($fd);
2287         my $line = <$fd>;
2288         close($fd);
2289
2290         my $mtime = $st->mtime;
2291         if ($mtime > time()) {
2292             warn "file '$filename' modified in future\n";
2293         }
2294
2295         if ($line =~ m/^(\d+)$/) {
2296             my $pid = $1;
2297             if (check_cmdline($pidfile, $pid)) {
2298                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2299                     return $pid;
2300                 }
2301             }
2302         }
2303     }
2304
2305     return undef;
2306 }
2307
2308 sub vzlist {
2309
2310     my $vzlist = config_list();
2311
2312     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2313
2314     while (defined(my $de = $fd->read)) {
2315         next if $de !~ m/^(\d+)\.pid$/;
2316         my $vmid = $1;
2317         next if !defined($vzlist->{$vmid});
2318         if (my $pid = check_running($vmid)) {
2319             $vzlist->{$vmid}->{pid} = $pid;
2320         }
2321     }
2322
2323     return $vzlist;
2324 }
2325
2326 sub disksize {
2327     my ($storecfg, $conf) = @_;
2328
2329     my $bootdisk = $conf->{bootdisk};
2330     return undef if !$bootdisk;
2331     return undef if !valid_drivename($bootdisk);
2332
2333     return undef if !$conf->{$bootdisk};
2334
2335     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2336     return undef if !defined($drive);
2337
2338     return undef if drive_is_cdrom($drive);
2339
2340     my $volid = $drive->{file};
2341     return undef if !$volid;
2342
2343     return $drive->{size};
2344 }
2345
2346 my $last_proc_pid_stat;
2347
2348 # get VM status information
2349 # This must be fast and should not block ($full == false)
2350 # We only query KVM using QMP if $full == true (this can be slow)
2351 sub vmstatus {
2352     my ($opt_vmid, $full) = @_;
2353
2354     my $res = {};
2355
2356     my $storecfg = PVE::Storage::config();
2357
2358     my $list = vzlist();
2359     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2360
2361     my $cpucount = $cpuinfo->{cpus} || 1;
2362
2363     foreach my $vmid (keys %$list) {
2364         next if $opt_vmid && ($vmid ne $opt_vmid);
2365
2366         my $cfspath = cfs_config_path($vmid);
2367         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2368
2369         my $d = {};
2370         $d->{pid} = $list->{$vmid}->{pid};
2371
2372         # fixme: better status?
2373         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2374
2375         my $size = disksize($storecfg, $conf);
2376         if (defined($size)) {
2377             $d->{disk} = 0; # no info available
2378             $d->{maxdisk} = $size;
2379         } else {
2380             $d->{disk} = 0;
2381             $d->{maxdisk} = 0;
2382         }
2383
2384         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2385         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2386         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2387
2388         $d->{name} = $conf->{name} || "VM $vmid";
2389         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2390
2391         if ($conf->{balloon}) {
2392             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2393             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2394         }
2395
2396         $d->{uptime} = 0;
2397         $d->{cpu} = 0;
2398         $d->{mem} = 0;
2399
2400         $d->{netout} = 0;
2401         $d->{netin} = 0;
2402
2403         $d->{diskread} = 0;
2404         $d->{diskwrite} = 0;
2405
2406         $d->{template} = is_template($conf);
2407
2408         $res->{$vmid} = $d;
2409     }
2410
2411     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2412     foreach my $dev (keys %$netdev) {
2413         next if $dev !~ m/^tap([1-9]\d*)i/;
2414         my $vmid = $1;
2415         my $d = $res->{$vmid};
2416         next if !$d;
2417
2418         $d->{netout} += $netdev->{$dev}->{receive};
2419         $d->{netin} += $netdev->{$dev}->{transmit};
2420
2421         if ($full) {
2422             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2423             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2424         }
2425
2426     }
2427
2428     my $ctime = gettimeofday;
2429
2430     foreach my $vmid (keys %$list) {
2431
2432         my $d = $res->{$vmid};
2433         my $pid = $d->{pid};
2434         next if !$pid;
2435
2436         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2437         next if !$pstat; # not running
2438
2439         my $used = $pstat->{utime} + $pstat->{stime};
2440
2441         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2442
2443         if ($pstat->{vsize}) {
2444             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2445         }
2446
2447         my $old = $last_proc_pid_stat->{$pid};
2448         if (!$old) {
2449             $last_proc_pid_stat->{$pid} = {
2450                 time => $ctime,
2451                 used => $used,
2452                 cpu => 0,
2453             };
2454             next;
2455         }
2456
2457         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2458
2459         if ($dtime > 1000) {
2460             my $dutime = $used -  $old->{used};
2461
2462             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2463             $last_proc_pid_stat->{$pid} = {
2464                 time => $ctime,
2465                 used => $used,
2466                 cpu => $d->{cpu},
2467             };
2468         } else {
2469             $d->{cpu} = $old->{cpu};
2470         }
2471     }
2472
2473     return $res if !$full;
2474
2475     my $qmpclient = PVE::QMPClient->new();
2476
2477     my $ballooncb = sub {
2478         my ($vmid, $resp) = @_;
2479
2480         my $info = $resp->{'return'};
2481         return if !$info->{max_mem};
2482
2483         my $d = $res->{$vmid};
2484
2485         # use memory assigned to VM
2486         $d->{maxmem} = $info->{max_mem};
2487         $d->{balloon} = $info->{actual};
2488
2489         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2490             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2491             $d->{freemem} = $info->{free_mem};
2492         }
2493
2494         $d->{ballooninfo} = $info;
2495     };
2496
2497     my $blockstatscb = sub {
2498         my ($vmid, $resp) = @_;
2499         my $data = $resp->{'return'} || [];
2500         my $totalrdbytes = 0;
2501         my $totalwrbytes = 0;
2502
2503         for my $blockstat (@$data) {
2504             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2505             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2506
2507             $blockstat->{device} =~ s/drive-//;
2508             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2509         }
2510         $res->{$vmid}->{diskread} = $totalrdbytes;
2511         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2512     };
2513
2514     my $statuscb = sub {
2515         my ($vmid, $resp) = @_;
2516
2517         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2518         # this fails if ballon driver is not loaded, so this must be
2519         # the last commnand (following command are aborted if this fails).
2520         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2521
2522         my $status = 'unknown';
2523         if (!defined($status = $resp->{'return'}->{status})) {
2524             warn "unable to get VM status\n";
2525             return;
2526         }
2527
2528         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2529     };
2530
2531     foreach my $vmid (keys %$list) {
2532         next if $opt_vmid && ($vmid ne $opt_vmid);
2533         next if !$res->{$vmid}->{pid}; # not running
2534         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2535     }
2536
2537     $qmpclient->queue_execute(undef, 1);
2538
2539     foreach my $vmid (keys %$list) {
2540         next if $opt_vmid && ($vmid ne $opt_vmid);
2541         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub foreach_dimm {
2548     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2549
2550     my $dimm_id = 0;
2551     my $current_size = 1024;
2552     my $dimm_size = 512;
2553     return if $current_size == $memory;
2554
2555     for (my $j = 0; $j < 8; $j++) {
2556         for (my $i = 0; $i < 32; $i++) {
2557             my $name = "dimm${dimm_id}";
2558             $dimm_id++;
2559             my $numanode = $i % $sockets;
2560             $current_size += $dimm_size;
2561             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2562             return  $current_size if $current_size >= $memory;
2563         }
2564         $dimm_size *= 2;
2565     }
2566 }
2567
2568 sub foreach_reverse_dimm {
2569     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2570
2571     my $dimm_id = 253;
2572     my $current_size = 4177920;
2573     my $dimm_size = 65536;
2574     return if $current_size == $memory;
2575
2576     for (my $j = 0; $j < 8; $j++) {
2577         for (my $i = 0; $i < 32; $i++) {
2578             my $name = "dimm${dimm_id}";
2579             $dimm_id--;
2580             my $numanode = $i % $sockets;
2581             $current_size -= $dimm_size;
2582             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2583             return  $current_size if $current_size <= $memory;
2584         }
2585         $dimm_size /= 2;
2586     }
2587 }
2588
2589 sub foreach_drive {
2590     my ($conf, $func) = @_;
2591
2592     foreach my $ds (keys %$conf) {
2593         next if !valid_drivename($ds);
2594
2595         my $drive = parse_drive($ds, $conf->{$ds});
2596         next if !$drive;
2597
2598         &$func($ds, $drive);
2599     }
2600 }
2601
2602 sub foreach_volid {
2603     my ($conf, $func) = @_;
2604
2605     my $volhash = {};
2606
2607     my $test_volid = sub {
2608         my ($volid, $is_cdrom) = @_;
2609
2610         return if !$volid;
2611
2612         $volhash->{$volid} = $is_cdrom || 0;
2613     };
2614
2615     foreach_drive($conf, sub {
2616         my ($ds, $drive) = @_;
2617         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2618     });
2619
2620     foreach my $snapname (keys %{$conf->{snapshots}}) {
2621         my $snap = $conf->{snapshots}->{$snapname};
2622         &$test_volid($snap->{vmstate}, 0);
2623         foreach_drive($snap, sub {
2624             my ($ds, $drive) = @_;
2625             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2626         });
2627     }
2628
2629     foreach my $volid (keys %$volhash) {
2630         &$func($volid, $volhash->{$volid});
2631     }
2632 }
2633
2634 sub vga_conf_has_spice {
2635     my ($vga) = @_;
2636
2637     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2638
2639     return $1 || 1;
2640 }
2641
2642 sub config_to_command {
2643     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2644
2645     my $cmd = [];
2646     my $globalFlags = [];
2647     my $machineFlags = [];
2648     my $rtcFlags = [];
2649     my $cpuFlags = [];
2650     my $devices = [];
2651     my $pciaddr = '';
2652     my $bridges = {};
2653     my $kvmver = kvm_user_version();
2654     my $vernum = 0; # unknown
2655     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2656         $vernum = $1*1000000+$2*1000;
2657     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2658         $vernum = $1*1000000+$2*1000+$3;
2659     }
2660
2661     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2662
2663     my $have_ovz = -f '/proc/vz/vestat';
2664
2665     my $q35 = machine_type_is_q35($conf);
2666     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2667     my $machine_type = $forcemachine || $conf->{machine};
2668     my $use_old_bios_files = undef;
2669     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2670
2671     my $cpuunits = defined($conf->{cpuunits}) ?
2672             $conf->{cpuunits} : $defaults->{cpuunits};
2673
2674     push @$cmd, '/usr/bin/systemd-run';
2675     push @$cmd, '--scope';
2676     push @$cmd, '--slice', "qemu";
2677     push @$cmd, '--unit', $vmid;
2678     # set KillMode=none, so that systemd don't kill those scopes
2679     # at shutdown (pve-manager service should stop the VMs instead)
2680     push @$cmd, '-p', "KillMode=none";
2681     push @$cmd, '-p', "CPUShares=$cpuunits";
2682     if ($conf->{cpulimit}) {
2683         my $cpulimit = int($conf->{cpulimit} * 100);
2684         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2685     }
2686
2687     push @$cmd, '/usr/bin/kvm';
2688
2689     push @$cmd, '-id', $vmid;
2690
2691     my $use_virtio = 0;
2692
2693     my $qmpsocket = qmp_socket($vmid);
2694     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2695     push @$cmd, '-mon', "chardev=qmp,mode=control";
2696
2697     my $socket = vnc_socket($vmid);
2698     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2699
2700     push @$cmd, '-pidfile' , pidfile_name($vmid);
2701
2702     push @$cmd, '-daemonize';
2703
2704     if ($conf->{smbios1}) {
2705         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2706     }
2707
2708     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2709         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2710         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2711         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2712         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2713         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2714         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2715     }
2716
2717     if ($q35) {
2718         # the q35 chipset support native usb2, so we enable usb controller
2719         # by default for this machine type
2720         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2721     } else {
2722         $pciaddr = print_pci_addr("piix3", $bridges);
2723         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2724
2725         my $use_usb2 = 0;
2726         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2727             next if !$conf->{"usb$i"};
2728             $use_usb2 = 1;
2729         }
2730         # include usb device config
2731         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2732     }
2733
2734     my $vga = $conf->{vga};
2735
2736     my $qxlnum = vga_conf_has_spice($vga);
2737     $vga = 'qxl' if $qxlnum;
2738
2739     if (!$vga) {
2740         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2741                                 $conf->{ostype} eq 'win7' ||
2742                                 $conf->{ostype} eq 'w2k8')) {
2743             $vga = 'std';
2744         } else {
2745             $vga = 'cirrus';
2746         }
2747     }
2748
2749     # enable absolute mouse coordinates (needed by vnc)
2750     my $tablet;
2751     if (defined($conf->{tablet})) {
2752         $tablet = $conf->{tablet};
2753     } else {
2754         $tablet = $defaults->{tablet};
2755         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2756         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2757     }
2758
2759     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2760
2761     my $nohyperv;
2762     # host pci devices
2763     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2764         my $d = parse_hostpci($conf->{"hostpci$i"});
2765         next if !$d;
2766
2767         my $pcie = $d->{pcie};
2768         if($pcie){
2769             die "q35 machine model is not enabled" if !$q35;
2770             $pciaddr = print_pcie_addr("hostpci$i");
2771         }else{
2772             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2773         }
2774
2775         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2776         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2777         if ($xvga && $xvga ne '') {
2778             push @$cpuFlags, 'kvm=off';
2779             $vga = 'none';
2780             $nohyperv = 1;
2781             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2782                 $xvga = "";
2783             }
2784         }
2785         my $pcidevices = $d->{pciid};
2786         my $multifunction = 1 if @$pcidevices > 1;
2787
2788         my $j=0;
2789         foreach my $pcidevice (@$pcidevices) {
2790
2791             my $id = "hostpci$i";
2792             $id .= ".$j" if $multifunction;
2793             my $addr = $pciaddr;
2794             $addr .= ".$j" if $multifunction;
2795             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2796
2797             if($j == 0){
2798                 $devicestr .= "$rombar$xvga";
2799                 $devicestr .= ",multifunction=on" if $multifunction;
2800             }
2801
2802             push @$devices, '-device', $devicestr;
2803             $j++;
2804         }
2805     }
2806
2807     # usb devices
2808     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2809         my $d = parse_usb_device($conf->{"usb$i"});
2810         next if !$d;
2811         if ($d->{vendorid} && $d->{productid}) {
2812             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2813         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2814             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2815         } elsif ($d->{spice}) {
2816             # usb redir support for spice
2817             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2818             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2819         }
2820     }
2821
2822     # serial devices
2823     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2824         if (my $path = $conf->{"serial$i"}) {
2825             if ($path eq 'socket') {
2826                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2827                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2828                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2829             } else {
2830                 die "no such serial device\n" if ! -c $path;
2831                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2832                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2833             }
2834         }
2835     }
2836
2837     # parallel devices
2838     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2839         if (my $path = $conf->{"parallel$i"}) {
2840             die "no such parallel device\n" if ! -c $path;
2841             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2842             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2843             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2844         }
2845     }
2846
2847     my $vmname = $conf->{name} || "vm$vmid";
2848
2849     push @$cmd, '-name', $vmname;
2850
2851     my $sockets = 1;
2852     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2853     $sockets = $conf->{sockets} if  $conf->{sockets};
2854
2855     my $cores = $conf->{cores} || 1;
2856
2857     my $maxcpus = $sockets * $cores;
2858
2859     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2860
2861     my $allowed_vcpus = $cpuinfo->{cpus};
2862
2863     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2864         if ($allowed_vcpus < $maxcpus);
2865
2866     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2867
2868     push @$cmd, '-nodefaults';
2869
2870     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2871
2872     my $bootindex_hash = {};
2873     my $i = 1;
2874     foreach my $o (split(//, $bootorder)) {
2875         $bootindex_hash->{$o} = $i*100;
2876         $i++;
2877     }
2878
2879     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2880
2881     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2882
2883     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2884
2885     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2886
2887     # time drift fix
2888     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2889
2890     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2891     my $useLocaltime = $conf->{localtime};
2892
2893     if (my $ost = $conf->{ostype}) {
2894         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2895
2896         if ($ost =~ m/^w/) { # windows
2897             $useLocaltime = 1 if !defined($conf->{localtime});
2898
2899             # use time drift fix when acpi is enabled
2900             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2901                 $tdf = 1 if !defined($conf->{tdf});
2902             }
2903         }
2904
2905         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2906             $ost eq 'wvista') {
2907             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2908             push @$cmd, '-no-hpet';
2909             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2910                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
2911                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
2912                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
2913
2914             } else {
2915                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
2916             }
2917         }
2918
2919         if ($ost eq 'win7' || $ost eq 'win8') {
2920             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
2921         }
2922     }
2923
2924     push @$rtcFlags, 'driftfix=slew' if $tdf;
2925
2926     if ($nokvm) {
2927         push @$machineFlags, 'accel=tcg';
2928     } else {
2929         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2930     }
2931
2932     if ($machine_type) {
2933         push @$machineFlags, "type=${machine_type}";
2934     }
2935
2936     if ($conf->{startdate}) {
2937         push @$rtcFlags, "base=$conf->{startdate}";
2938     } elsif ($useLocaltime) {
2939         push @$rtcFlags, 'base=localtime';
2940     }
2941
2942     my $cpu = $nokvm ? "qemu64" : "kvm64";
2943     $cpu = $conf->{cpu} if $conf->{cpu};
2944
2945     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2946
2947     push @$cpuFlags , '-x2apic'
2948         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2949
2950     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2951
2952     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2953
2954     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2955
2956         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2957         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2958     }
2959
2960     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2961
2962     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2963
2964     push @$cmd, '-cpu', $cpu;
2965
2966     my $memory = $conf->{memory} || $defaults->{memory};
2967     my $static_memory = 0;
2968     my $dimm_memory = 0;
2969
2970     if ($hotplug_features->{memory}) {
2971         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2972         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2973         $static_memory = $STATICMEM;
2974         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2975         $dimm_memory = $memory - $static_memory;
2976         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2977
2978     } else {
2979
2980         $static_memory = $memory;
2981         push @$cmd, '-m', $static_memory;
2982     }
2983
2984     if ($conf->{numa}) {
2985
2986         my $numa_totalmemory = undef;
2987         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2988             next if !$conf->{"numa$i"};
2989             my $numa = parse_numa($conf->{"numa$i"});
2990             next if !$numa;
2991             # memory
2992             die "missing numa node$i memory value\n" if !$numa->{memory};
2993             my $numa_memory = $numa->{memory};
2994             $numa_totalmemory += $numa_memory;
2995             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2996
2997             # cpus
2998             my $cpus_start = $numa->{cpus}->{start};
2999             die "missing numa node$i cpus\n" if !defined($cpus_start);
3000             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3001             my $cpus = $cpus_start;
3002             if (defined($cpus_end)) {
3003                 $cpus .= "-$cpus_end";
3004                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3005             }
3006
3007             # hostnodes
3008             my $hostnodes_start = $numa->{hostnodes}->{start};
3009             if (defined($hostnodes_start)) {
3010                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3011                 my $hostnodes = $hostnodes_start;
3012                 if (defined($hostnodes_end)) {
3013                     $hostnodes .= "-$hostnodes_end";
3014                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3015                 }
3016
3017                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3018                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3019                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3020                 }
3021
3022                 # policy
3023                 my $policy = $numa->{policy};
3024                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3025                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3026             }
3027
3028             push @$cmd, '-object', $numa_object;
3029             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3030         }
3031
3032         die "total memory for NUMA nodes must be equal to vm static memory\n"
3033             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3034
3035         #if no custom tology, we split memory and cores across numa nodes
3036         if(!$numa_totalmemory) {
3037
3038             my $numa_memory = ($static_memory / $sockets) . "M";
3039
3040             for (my $i = 0; $i < $sockets; $i++)  {
3041
3042                 my $cpustart = ($cores * $i);
3043                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3044                 my $cpus = $cpustart;
3045                 $cpus .= "-$cpuend" if $cpuend;
3046
3047                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3048                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3049             }
3050         }
3051     }
3052
3053     if ($hotplug_features->{memory}) {
3054         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3055             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3056             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3057             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3058
3059             #if dimm_memory is not aligned to dimm map
3060             if($current_size > $memory) {
3061                  $conf->{memory} = $current_size;
3062                  update_config_nolock($vmid, $conf, 1);
3063             }
3064         });
3065     }
3066
3067     push @$cmd, '-S' if $conf->{freeze};
3068
3069     # set keyboard layout
3070     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3071     push @$cmd, '-k', $kb if $kb;
3072
3073     # enable sound
3074     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3075     #push @$cmd, '-soundhw', 'es1370';
3076     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3077
3078     if($conf->{agent}) {
3079         my $qgasocket = qmp_socket($vmid, 1);
3080         my $pciaddr = print_pci_addr("qga0", $bridges);
3081         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3082         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3083         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3084     }
3085
3086     my $spice_port;
3087
3088     if ($qxlnum) {
3089         if ($qxlnum > 1) {
3090             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3091                 for(my $i = 1; $i < $qxlnum; $i++){
3092                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3093                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3094                 }
3095             } else {
3096                 # assume other OS works like Linux
3097                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3098                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3099             }
3100         }
3101
3102         my $pciaddr = print_pci_addr("spice", $bridges);
3103
3104         my $nodename = PVE::INotify::nodename();
3105         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3106         $spice_port = PVE::Tools::next_spice_port($pfamily);
3107
3108         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3109
3110         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3111         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3112         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3113     }
3114
3115     # enable balloon by default, unless explicitly disabled
3116     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3117         $pciaddr = print_pci_addr("balloon0", $bridges);
3118         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3119     }
3120
3121     if ($conf->{watchdog}) {
3122         my $wdopts = parse_watchdog($conf->{watchdog});
3123         $pciaddr = print_pci_addr("watchdog", $bridges);
3124         my $watchdog = $wdopts->{model} || 'i6300esb';
3125         push @$devices, '-device', "$watchdog$pciaddr";
3126         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3127     }
3128
3129     my $vollist = [];
3130     my $scsicontroller = {};
3131     my $ahcicontroller = {};
3132     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3133
3134     # Add iscsi initiator name if available
3135     if (my $initiator = get_initiator_name()) {
3136         push @$devices, '-iscsi', "initiator-name=$initiator";
3137     }
3138
3139     foreach_drive($conf, sub {
3140         my ($ds, $drive) = @_;
3141
3142         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3143             push @$vollist, $drive->{file};
3144         }
3145
3146         $use_virtio = 1 if $ds =~ m/^virtio/;
3147
3148         if (drive_is_cdrom ($drive)) {
3149             if ($bootindex_hash->{d}) {
3150                 $drive->{bootindex} = $bootindex_hash->{d};
3151                 $bootindex_hash->{d} += 1;
3152             }
3153         } else {
3154             if ($bootindex_hash->{c}) {
3155                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3156                 $bootindex_hash->{c} += 1;
3157             }
3158         }
3159
3160         if($drive->{interface} eq 'virtio'){
3161            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3162         }
3163
3164         if ($drive->{interface} eq 'scsi') {
3165
3166             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3167
3168             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3169             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3170
3171             my $iothread = '';
3172             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3173                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3174                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3175             }
3176
3177             my $queues = '';
3178             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3179                 $queues = ",num_queues=$drive->{queues}";
3180             }
3181
3182             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3183             $scsicontroller->{$controller}=1;
3184         }
3185
3186         if ($drive->{interface} eq 'sata') {
3187            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3188            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3189            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3190            $ahcicontroller->{$controller}=1;
3191         }
3192
3193         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3194         push @$devices, '-drive',$drive_cmd;
3195         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3196     });
3197
3198     for (my $i = 0; $i < $MAX_NETS; $i++) {
3199          next if !$conf->{"net$i"};
3200          my $d = parse_net($conf->{"net$i"});
3201          next if !$d;
3202
3203          $use_virtio = 1 if $d->{model} eq 'virtio';
3204
3205          if ($bootindex_hash->{n}) {
3206             $d->{bootindex} = $bootindex_hash->{n};
3207             $bootindex_hash->{n} += 1;
3208          }
3209
3210          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3211          push @$devices, '-netdev', $netdevfull;
3212
3213          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3214          push @$devices, '-device', $netdevicefull;
3215     }
3216
3217     if (!$q35) {
3218         # add pci bridges
3219         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3220            $bridges->{1} = 1;
3221            $bridges->{2} = 1;
3222         }
3223
3224         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3225
3226         while (my ($k, $v) = each %$bridges) {
3227             $pciaddr = print_pci_addr("pci.$k");
3228             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3229         }
3230     }
3231
3232     # add custom args
3233     if ($conf->{args}) {
3234         my $aa = PVE::Tools::split_args($conf->{args});
3235         push @$cmd, @$aa;
3236     }
3237
3238     push @$cmd, @$devices;
3239     push @$cmd, '-rtc', join(',', @$rtcFlags)
3240         if scalar(@$rtcFlags);
3241     push @$cmd, '-machine', join(',', @$machineFlags)
3242         if scalar(@$machineFlags);
3243     push @$cmd, '-global', join(',', @$globalFlags)
3244         if scalar(@$globalFlags);
3245
3246     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3247 }
3248
3249 sub vnc_socket {
3250     my ($vmid) = @_;
3251     return "${var_run_tmpdir}/$vmid.vnc";
3252 }
3253
3254 sub spice_port {
3255     my ($vmid) = @_;
3256
3257     my $res = vm_mon_cmd($vmid, 'query-spice');
3258
3259     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3260 }
3261
3262 sub qmp_socket {
3263     my ($vmid, $qga) = @_;
3264     my $sockettype = $qga ? 'qga' : 'qmp';
3265     return "${var_run_tmpdir}/$vmid.$sockettype";
3266 }
3267
3268 sub pidfile_name {
3269     my ($vmid) = @_;
3270     return "${var_run_tmpdir}/$vmid.pid";
3271 }
3272
3273 sub vm_devices_list {
3274     my ($vmid) = @_;
3275
3276     my $res = vm_mon_cmd($vmid, 'query-pci');
3277     my $devices = {};
3278     foreach my $pcibus (@$res) {
3279         foreach my $device (@{$pcibus->{devices}}) {
3280             next if !$device->{'qdev_id'};
3281             if ($device->{'pci_bridge'}) {
3282                 $devices->{$device->{'qdev_id'}} = 1;
3283                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3284                     next if !$bridge_device->{'qdev_id'};
3285                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3286                     $devices->{$device->{'qdev_id'}}++;
3287                 }
3288             } else {
3289                 $devices->{$device->{'qdev_id'}} = 1;
3290             }
3291         }
3292     }
3293
3294     my $resblock = vm_mon_cmd($vmid, 'query-block');
3295     foreach my $block (@$resblock) {
3296         if($block->{device} =~ m/^drive-(\S+)/){
3297                 $devices->{$1} = 1;
3298         }
3299     }
3300
3301     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3302     foreach my $mice (@$resmice) {
3303         if ($mice->{name} eq 'QEMU HID Tablet') {
3304             $devices->{tablet} = 1;
3305             last;
3306         }
3307     }
3308
3309     return $devices;
3310 }
3311
3312 sub vm_deviceplug {
3313     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3314
3315     my $q35 = machine_type_is_q35($conf);
3316
3317     my $devices_list = vm_devices_list($vmid);
3318     return 1 if defined($devices_list->{$deviceid});
3319
3320     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3321
3322     if ($deviceid eq 'tablet') {
3323
3324         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3325
3326     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3327
3328         qemu_iothread_add($vmid, $deviceid, $device);
3329
3330         qemu_driveadd($storecfg, $vmid, $device);
3331         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3332
3333         qemu_deviceadd($vmid, $devicefull);
3334         eval { qemu_deviceaddverify($vmid, $deviceid); };
3335         if (my $err = $@) {
3336             eval { qemu_drivedel($vmid, $deviceid); };
3337             warn $@ if $@;
3338             die $err;
3339         }
3340
3341     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3342
3343
3344         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3345         my $pciaddr = print_pci_addr($deviceid);
3346         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3347
3348         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3349
3350         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3351             qemu_iothread_add($vmid, $deviceid, $device);
3352             $devicefull .= ",iothread=iothread-$deviceid";
3353         }
3354
3355         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3356             $devicefull .= ",num_queues=$device->{queues}";
3357         }
3358
3359         qemu_deviceadd($vmid, $devicefull);
3360         qemu_deviceaddverify($vmid, $deviceid);
3361
3362     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3363
3364         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3365         qemu_driveadd($storecfg, $vmid, $device);
3366
3367         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3368         eval { qemu_deviceadd($vmid, $devicefull); };
3369         if (my $err = $@) {
3370             eval { qemu_drivedel($vmid, $deviceid); };
3371             warn $@ if $@;
3372             die $err;
3373         }
3374
3375     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3376
3377         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3378
3379         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3380         my $use_old_bios_files = undef;
3381         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3382
3383         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3384         qemu_deviceadd($vmid, $netdevicefull);
3385         eval { qemu_deviceaddverify($vmid, $deviceid); };
3386         if (my $err = $@) {
3387             eval { qemu_netdevdel($vmid, $deviceid); };
3388             warn $@ if $@;
3389             die $err;
3390         }
3391
3392     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3393
3394         my $bridgeid = $2;
3395         my $pciaddr = print_pci_addr($deviceid);
3396         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3397
3398         qemu_deviceadd($vmid, $devicefull);
3399         qemu_deviceaddverify($vmid, $deviceid);
3400
3401     } else {
3402         die "can't hotplug device '$deviceid'\n";
3403     }
3404
3405     return 1;
3406 }
3407
3408 # fixme: this should raise exceptions on error!
3409 sub vm_deviceunplug {
3410     my ($vmid, $conf, $deviceid) = @_;
3411
3412     my $devices_list = vm_devices_list($vmid);
3413     return 1 if !defined($devices_list->{$deviceid});
3414
3415     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3416
3417     if ($deviceid eq 'tablet') {
3418
3419         qemu_devicedel($vmid, $deviceid);
3420
3421     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3422
3423         qemu_devicedel($vmid, $deviceid);
3424         qemu_devicedelverify($vmid, $deviceid);
3425         qemu_drivedel($vmid, $deviceid);
3426         qemu_iothread_del($conf, $vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3429
3430         qemu_devicedel($vmid, $deviceid);
3431         qemu_devicedelverify($vmid, $deviceid);
3432         qemu_iothread_del($conf, $vmid, $deviceid);
3433
3434     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3435
3436         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3437         my $device = parse_drive($deviceid, $conf->{$deviceid});
3438         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3439
3440         qemu_devicedel($vmid, $deviceid);
3441         qemu_drivedel($vmid, $deviceid);
3442         qemu_deletescsihw($conf, $vmid, $deviceid);
3443
3444     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3445
3446         qemu_devicedel($vmid, $deviceid);
3447         qemu_devicedelverify($vmid, $deviceid);
3448         qemu_netdevdel($vmid, $deviceid);
3449
3450     } else {
3451         die "can't unplug device '$deviceid'\n";
3452     }
3453
3454     return 1;
3455 }
3456
3457 sub qemu_deviceadd {
3458     my ($vmid, $devicefull) = @_;
3459
3460     $devicefull = "driver=".$devicefull;
3461     my %options =  split(/[=,]/, $devicefull);
3462
3463     vm_mon_cmd($vmid, "device_add" , %options);
3464 }
3465
3466 sub qemu_devicedel {
3467     my ($vmid, $deviceid) = @_;
3468
3469     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3470 }
3471
3472 sub qemu_iothread_add {
3473     my($vmid, $deviceid, $device) = @_;
3474
3475     if ($device->{iothread}) {
3476         my $iothreads = vm_iothreads_list($vmid);
3477         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3478     }
3479 }
3480
3481 sub qemu_iothread_del {
3482     my($conf, $vmid, $deviceid) = @_;
3483
3484     my $device = parse_drive($deviceid, $conf->{$deviceid});
3485     if ($device->{iothread}) {
3486         my $iothreads = vm_iothreads_list($vmid);
3487         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3488     }
3489 }
3490
3491 sub qemu_objectadd {
3492     my($vmid, $objectid, $qomtype) = @_;
3493
3494     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3495
3496     return 1;
3497 }
3498
3499 sub qemu_objectdel {
3500     my($vmid, $objectid) = @_;
3501
3502     vm_mon_cmd($vmid, "object-del", id => $objectid);
3503
3504     return 1;
3505 }
3506
3507 sub qemu_driveadd {
3508     my ($storecfg, $vmid, $device) = @_;
3509
3510     my $drive = print_drive_full($storecfg, $vmid, $device);
3511     $drive =~ s/\\/\\\\/g;
3512     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3513
3514     # If the command succeeds qemu prints: "OK"
3515     return 1 if $ret =~ m/OK/s;
3516
3517     die "adding drive failed: $ret\n";
3518 }
3519
3520 sub qemu_drivedel {
3521     my($vmid, $deviceid) = @_;
3522
3523     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3524     $ret =~ s/^\s+//;
3525
3526     return 1 if $ret eq "";
3527
3528     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3529     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3530
3531     die "deleting drive $deviceid failed : $ret\n";
3532 }
3533
3534 sub qemu_deviceaddverify {
3535     my ($vmid, $deviceid) = @_;
3536
3537     for (my $i = 0; $i <= 5; $i++) {
3538          my $devices_list = vm_devices_list($vmid);
3539          return 1 if defined($devices_list->{$deviceid});
3540          sleep 1;
3541     }
3542
3543     die "error on hotplug device '$deviceid'\n";
3544 }
3545
3546
3547 sub qemu_devicedelverify {
3548     my ($vmid, $deviceid) = @_;
3549
3550     # need to verify that the device is correctly removed as device_del
3551     # is async and empty return is not reliable
3552
3553     for (my $i = 0; $i <= 5; $i++) {
3554          my $devices_list = vm_devices_list($vmid);
3555          return 1 if !defined($devices_list->{$deviceid});
3556          sleep 1;
3557     }
3558
3559     die "error on hot-unplugging device '$deviceid'\n";
3560 }
3561
3562 sub qemu_findorcreatescsihw {
3563     my ($storecfg, $conf, $vmid, $device) = @_;
3564
3565     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3566
3567     my $scsihwid="$controller_prefix$controller";
3568     my $devices_list = vm_devices_list($vmid);
3569
3570     if(!defined($devices_list->{$scsihwid})) {
3571         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3572     }
3573
3574     return 1;
3575 }
3576
3577 sub qemu_deletescsihw {
3578     my ($conf, $vmid, $opt) = @_;
3579
3580     my $device = parse_drive($opt, $conf->{$opt});
3581
3582     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3583         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3584         return 1;
3585     }
3586
3587     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3588
3589     my $devices_list = vm_devices_list($vmid);
3590     foreach my $opt (keys %{$devices_list}) {
3591         if (PVE::QemuServer::valid_drivename($opt)) {
3592             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3593             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3594                 return 1;
3595             }
3596         }
3597     }
3598
3599     my $scsihwid="scsihw$controller";
3600
3601     vm_deviceunplug($vmid, $conf, $scsihwid);
3602
3603     return 1;
3604 }
3605
3606 sub qemu_add_pci_bridge {
3607     my ($storecfg, $conf, $vmid, $device) = @_;
3608
3609     my $bridges = {};
3610
3611     my $bridgeid;
3612
3613     print_pci_addr($device, $bridges);
3614
3615     while (my ($k, $v) = each %$bridges) {
3616         $bridgeid = $k;
3617     }
3618     return 1 if !defined($bridgeid) || $bridgeid < 1;
3619
3620     my $bridge = "pci.$bridgeid";
3621     my $devices_list = vm_devices_list($vmid);
3622
3623     if (!defined($devices_list->{$bridge})) {
3624         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3625     }
3626
3627     return 1;
3628 }
3629
3630 sub qemu_set_link_status {
3631     my ($vmid, $device, $up) = @_;
3632
3633     vm_mon_cmd($vmid, "set_link", name => $device,
3634                up => $up ? JSON::true : JSON::false);
3635 }
3636
3637 sub qemu_netdevadd {
3638     my ($vmid, $conf, $device, $deviceid) = @_;
3639
3640     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3641     my %options =  split(/[=,]/, $netdev);
3642
3643     vm_mon_cmd($vmid, "netdev_add",  %options);
3644     return 1;
3645 }
3646
3647 sub qemu_netdevdel {
3648     my ($vmid, $deviceid) = @_;
3649
3650     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3651 }
3652
3653 sub qemu_cpu_hotplug {
3654     my ($vmid, $conf, $vcpus) = @_;
3655
3656     my $sockets = 1;
3657     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3658     $sockets = $conf->{sockets} if  $conf->{sockets};
3659     my $cores = $conf->{cores} || 1;
3660     my $maxcpus = $sockets * $cores;
3661
3662     $vcpus = $maxcpus if !$vcpus;
3663
3664     die "you can't add more vcpus than maxcpus\n"
3665         if $vcpus > $maxcpus;
3666
3667     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3668     die "online cpu unplug is not yet possible\n"
3669         if $vcpus < $currentvcpus;
3670
3671     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3672     die "vcpus in running vm is different than configuration\n"
3673         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3674
3675     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3676         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3677     }
3678 }
3679
3680 sub qemu_memory_hotplug {
3681     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3682
3683     return $value if !check_running($vmid);
3684
3685     my $memory = $conf->{memory} || $defaults->{memory};
3686     $value = $defaults->{memory} if !$value;
3687     return $value if $value == $memory;
3688
3689     my $static_memory = $STATICMEM;
3690     my $dimm_memory = $memory - $static_memory;
3691
3692     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3693     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3694
3695
3696     my $sockets = 1;
3697     $sockets = $conf->{sockets} if $conf->{sockets};
3698
3699     if($value > $memory) {
3700
3701         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3702             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3703
3704                 return if $current_size <= $conf->{memory};
3705
3706                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3707                 if (my $err = $@) {
3708                     eval { qemu_objectdel($vmid, "mem-$name"); };
3709                     die $err;
3710                 }
3711
3712                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3713                 if (my $err = $@) {
3714                     eval { qemu_objectdel($vmid, "mem-$name"); };
3715                     die $err;
3716                 }
3717                 #update conf after each succesful module hotplug
3718                 $conf->{memory} = $current_size;
3719                 update_config_nolock($vmid, $conf, 1);
3720         });
3721
3722     } else {
3723
3724         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3725             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3726
3727                 return if $current_size >= $conf->{memory};
3728                 print "try to unplug memory dimm $name\n";
3729
3730                 my $retry = 0;
3731                 while (1) {
3732                     eval { qemu_devicedel($vmid, $name) };
3733                     sleep 3;
3734                     my $dimm_list = qemu_dimm_list($vmid);
3735                     last if !$dimm_list->{$name};
3736                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3737                     $retry++;
3738                 }
3739
3740                 #update conf after each succesful module unplug
3741                 $conf->{memory} = $current_size;
3742
3743                 eval { qemu_objectdel($vmid, "mem-$name"); };
3744                 update_config_nolock($vmid, $conf, 1);
3745         });
3746     }
3747 }
3748
3749 sub qemu_dimm_list {
3750     my ($vmid) = @_;
3751
3752     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3753     my $dimms = {};
3754
3755     foreach my $dimm (@$dimmarray) {
3756
3757         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3758         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3759         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3760         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3761         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3762     }
3763     return $dimms;
3764 }
3765
3766 sub qemu_block_set_io_throttle {
3767     my ($vmid, $deviceid,
3768         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3769         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3770
3771     return if !check_running($vmid) ;
3772
3773     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3774         bps => int($bps),
3775         bps_rd => int($bps_rd),
3776         bps_wr => int($bps_wr),
3777         iops => int($iops),
3778         iops_rd => int($iops_rd),
3779         iops_wr => int($iops_wr),
3780         bps_max => int($bps_max),
3781         bps_rd_max => int($bps_rd_max),
3782         bps_wr_max => int($bps_wr_max),
3783         iops_max => int($iops_max),
3784         iops_rd_max => int($iops_rd_max),
3785         iops_wr_max => int($iops_wr_max)
3786     );
3787
3788 }
3789
3790 # old code, only used to shutdown old VM after update
3791 sub __read_avail {
3792     my ($fh, $timeout) = @_;
3793
3794     my $sel = new IO::Select;
3795     $sel->add($fh);
3796
3797     my $res = '';
3798     my $buf;
3799
3800     my @ready;
3801     while (scalar (@ready = $sel->can_read($timeout))) {
3802         my $count;
3803         if ($count = $fh->sysread($buf, 8192)) {
3804             if ($buf =~ /^(.*)\(qemu\) $/s) {
3805                 $res .= $1;
3806                 last;
3807             } else {
3808                 $res .= $buf;
3809             }
3810         } else {
3811             if (!defined($count)) {
3812                 die "$!\n";
3813             }
3814             last;
3815         }
3816     }
3817
3818     die "monitor read timeout\n" if !scalar(@ready);
3819
3820     return $res;
3821 }
3822
3823 # old code, only used to shutdown old VM after update
3824 sub vm_monitor_command {
3825     my ($vmid, $cmdstr, $nocheck) = @_;
3826
3827     my $res;
3828
3829     eval {
3830         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3831
3832         my $sname = "${var_run_tmpdir}/$vmid.mon";
3833
3834         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3835             die "unable to connect to VM $vmid socket - $!\n";
3836
3837         my $timeout = 3;
3838
3839         # hack: migrate sometime blocks the monitor (when migrate_downtime
3840         # is set)
3841         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3842             $timeout = 60*60; # 1 hour
3843         }
3844
3845         # read banner;
3846         my $data = __read_avail($sock, $timeout);
3847
3848         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3849             die "got unexpected qemu monitor banner\n";
3850         }
3851
3852         my $sel = new IO::Select;
3853         $sel->add($sock);
3854
3855         if (!scalar(my @ready = $sel->can_write($timeout))) {
3856             die "monitor write error - timeout";
3857         }
3858
3859         my $fullcmd = "$cmdstr\r";
3860
3861         # syslog('info', "VM $vmid monitor command: $cmdstr");
3862
3863         my $b;
3864         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3865             die "monitor write error - $!";
3866         }
3867
3868         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3869
3870         $timeout = 20;
3871
3872         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3873             $timeout = 60*60; # 1 hour
3874         } elsif ($cmdstr =~ m/^(eject|change)/) {
3875             $timeout = 60; # note: cdrom mount command is slow
3876         }
3877         if ($res = __read_avail($sock, $timeout)) {
3878
3879             my @lines = split("\r?\n", $res);
3880
3881             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3882
3883             $res = join("\n", @lines);
3884             $res .= "\n";
3885         }
3886     };
3887
3888     my $err = $@;
3889
3890     if ($err) {
3891         syslog("err", "VM $vmid monitor command failed - $err");
3892         die $err;
3893     }
3894
3895     return $res;
3896 }
3897
3898 sub qemu_block_resize {
3899     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3900
3901     my $running = check_running($vmid);
3902
3903     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3904
3905     return if !$running;
3906
3907     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3908
3909 }
3910
3911 sub qemu_volume_snapshot {
3912     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3913
3914     my $running = check_running($vmid);
3915
3916     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3917         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3918     } else {
3919         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3920     }
3921 }
3922
3923 sub qemu_volume_snapshot_delete {
3924     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3925
3926     my $running = check_running($vmid);
3927
3928     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3929
3930     return if !$running;
3931
3932     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3933 }
3934
3935 sub set_migration_caps {
3936     my ($vmid) = @_;
3937
3938     my $cap_ref = [];
3939
3940     my $enabled_cap = {
3941         "auto-converge" => 1,
3942         "xbzrle" => 1,
3943         "x-rdma-pin-all" => 0,
3944         "zero-blocks" => 0,
3945         "compress" => 0
3946     };
3947
3948     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3949
3950     for my $supported_capability (@$supported_capabilities) {
3951         push @$cap_ref, {
3952             capability => $supported_capability->{capability},
3953             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3954         };
3955     }
3956
3957     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3958 }
3959
3960 my $fast_plug_option = {
3961     'lock' => 1,
3962     'name' => 1,
3963     'onboot' => 1,
3964     'shares' => 1,
3965     'startup' => 1,
3966     'description' => 1,
3967 };
3968
3969 # hotplug changes in [PENDING]
3970 # $selection hash can be used to only apply specified options, for
3971 # example: { cores => 1 } (only apply changed 'cores')
3972 # $errors ref is used to return error messages
3973 sub vmconfig_hotplug_pending {
3974     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3975
3976     my $defaults = load_defaults();
3977
3978     # commit values which do not have any impact on running VM first
3979     # Note: those option cannot raise errors, we we do not care about
3980     # $selection and always apply them.
3981
3982     my $add_error = sub {
3983         my ($opt, $msg) = @_;
3984         $errors->{$opt} = "hotplug problem - $msg";
3985     };
3986
3987     my $changes = 0;
3988     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3989         if ($fast_plug_option->{$opt}) {
3990             $conf->{$opt} = $conf->{pending}->{$opt};
3991             delete $conf->{pending}->{$opt};
3992             $changes = 1;
3993         }
3994     }
3995
3996     if ($changes) {
3997         update_config_nolock($vmid, $conf, 1);
3998         $conf = load_config($vmid); # update/reload
3999     }
4000
4001     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4002
4003     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4004     while (my ($opt, $force) = each %$pending_delete_hash) {
4005         next if $selection && !$selection->{$opt};
4006         eval {
4007             if ($opt eq 'hotplug') {
4008                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4009             } elsif ($opt eq 'tablet') {
4010                 die "skip\n" if !$hotplug_features->{usb};
4011                 if ($defaults->{tablet}) {
4012                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4013                 } else {
4014                     vm_deviceunplug($vmid, $conf, $opt);
4015                 }
4016             } elsif ($opt eq 'vcpus') {
4017                 die "skip\n" if !$hotplug_features->{cpu};
4018                 qemu_cpu_hotplug($vmid, $conf, undef);
4019             } elsif ($opt eq 'balloon') {
4020                 # enable balloon device is not hotpluggable
4021                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4022             } elsif ($fast_plug_option->{$opt}) {
4023                 # do nothing
4024             } elsif ($opt =~ m/^net(\d+)$/) {
4025                 die "skip\n" if !$hotplug_features->{network};
4026                 vm_deviceunplug($vmid, $conf, $opt);
4027             } elsif (valid_drivename($opt)) {
4028                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4029                 vm_deviceunplug($vmid, $conf, $opt);
4030                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4031             } elsif ($opt =~ m/^memory$/) {
4032                 die "skip\n" if !$hotplug_features->{memory};
4033                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4034             } elsif ($opt eq 'cpuunits') {
4035                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4036             } elsif ($opt eq 'cpulimit') {
4037                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4038             } else {
4039                 die "skip\n";
4040             }
4041         };
4042         if (my $err = $@) {
4043             &$add_error($opt, $err) if $err ne "skip\n";
4044         } else {
4045             # save new config if hotplug was successful
4046             delete $conf->{$opt};
4047             vmconfig_undelete_pending_option($conf, $opt);
4048             update_config_nolock($vmid, $conf, 1);
4049             $conf = load_config($vmid); # update/reload
4050         }
4051     }
4052
4053     foreach my $opt (keys %{$conf->{pending}}) {
4054         next if $selection && !$selection->{$opt};
4055         my $value = $conf->{pending}->{$opt};
4056         eval {
4057             if ($opt eq 'hotplug') {
4058                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4059             } elsif ($opt eq 'tablet') {
4060                 die "skip\n" if !$hotplug_features->{usb};
4061                 if ($value == 1) {
4062                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4063                 } elsif ($value == 0) {
4064                     vm_deviceunplug($vmid, $conf, $opt);
4065                 }
4066             } elsif ($opt eq 'vcpus') {
4067                 die "skip\n" if !$hotplug_features->{cpu};
4068                 qemu_cpu_hotplug($vmid, $conf, $value);
4069             } elsif ($opt eq 'balloon') {
4070                 # enable/disable balloning device is not hotpluggable
4071                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4072                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4073                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4074
4075                 # allow manual ballooning if shares is set to zero
4076                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4077                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4078                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4079                 }
4080             } elsif ($opt =~ m/^net(\d+)$/) {
4081                 # some changes can be done without hotplug
4082                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4083                                     $vmid, $opt, $value);
4084             } elsif (valid_drivename($opt)) {
4085                 # some changes can be done without hotplug
4086                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4087                                      $vmid, $opt, $value, 1);
4088             } elsif ($opt =~ m/^memory$/) { #dimms
4089                 die "skip\n" if !$hotplug_features->{memory};
4090                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4091             } elsif ($opt eq 'cpuunits') {
4092                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4093             } elsif ($opt eq 'cpulimit') {
4094                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4095                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4096             } else {
4097                 die "skip\n";  # skip non-hot-pluggable options
4098             }
4099         };
4100         if (my $err = $@) {
4101             &$add_error($opt, $err) if $err ne "skip\n";
4102         } else {
4103             # save new config if hotplug was successful
4104             $conf->{$opt} = $value;
4105             delete $conf->{pending}->{$opt};
4106             update_config_nolock($vmid, $conf, 1);
4107             $conf = load_config($vmid); # update/reload
4108         }
4109     }
4110 }
4111
4112 sub try_deallocate_drive {
4113     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4114
4115     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4116         my $volid = $drive->{file};
4117         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4118             my $sid = PVE::Storage::parse_volume_id($volid);
4119             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4120
4121             # check if the disk is really unused
4122             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4123                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4124             PVE::Storage::vdisk_free($storecfg, $volid);
4125             return 1;
4126         } else {
4127             # If vm is not owner of this disk remove from config
4128             return 1;
4129         }
4130     }
4131
4132     return undef;
4133 }
4134
4135 sub vmconfig_delete_or_detach_drive {
4136     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4137
4138     my $drive = parse_drive($opt, $conf->{$opt});
4139
4140     my $rpcenv = PVE::RPCEnvironment::get();
4141     my $authuser = $rpcenv->get_user();
4142
4143     if ($force) {
4144         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4145         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4146     } else {
4147         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4148     }
4149 }
4150
4151 sub vmconfig_apply_pending {
4152     my ($vmid, $conf, $storecfg) = @_;
4153
4154     # cold plug
4155
4156     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4157     while (my ($opt, $force) = each %$pending_delete_hash) {
4158         die "internal error" if $opt =~ m/^unused/;
4159         $conf = load_config($vmid); # update/reload
4160         if (!defined($conf->{$opt})) {
4161             vmconfig_undelete_pending_option($conf, $opt);
4162             update_config_nolock($vmid, $conf, 1);
4163         } elsif (valid_drivename($opt)) {
4164             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4165             vmconfig_undelete_pending_option($conf, $opt);
4166             delete $conf->{$opt};
4167             update_config_nolock($vmid, $conf, 1);
4168         } else {
4169             vmconfig_undelete_pending_option($conf, $opt);
4170             delete $conf->{$opt};
4171             update_config_nolock($vmid, $conf, 1);
4172         }
4173     }
4174
4175     $conf = load_config($vmid); # update/reload
4176
4177     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4178         $conf = load_config($vmid); # update/reload
4179
4180         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4181             # skip if nothing changed
4182         } elsif (valid_drivename($opt)) {
4183             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4184                 if defined($conf->{$opt});
4185             $conf->{$opt} = $conf->{pending}->{$opt};
4186         } else {
4187             $conf->{$opt} = $conf->{pending}->{$opt};
4188         }
4189
4190         delete $conf->{pending}->{$opt};
4191         update_config_nolock($vmid, $conf, 1);
4192     }
4193 }
4194
4195 my $safe_num_ne = sub {
4196     my ($a, $b) = @_;
4197
4198     return 0 if !defined($a) && !defined($b);
4199     return 1 if !defined($a);
4200     return 1 if !defined($b);
4201
4202     return $a != $b;
4203 };
4204
4205 my $safe_string_ne = sub {
4206     my ($a, $b) = @_;
4207
4208     return 0 if !defined($a) && !defined($b);
4209     return 1 if !defined($a);
4210     return 1 if !defined($b);
4211
4212     return $a ne $b;
4213 };
4214
4215 sub vmconfig_update_net {
4216     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4217
4218     my $newnet = parse_net($value);
4219
4220     if ($conf->{$opt}) {
4221         my $oldnet = parse_net($conf->{$opt});
4222
4223         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4224             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4225             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4226             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4227
4228             # for non online change, we try to hot-unplug
4229             die "skip\n" if !$hotplug;
4230             vm_deviceunplug($vmid, $conf, $opt);
4231         } else {
4232
4233             die "internal error" if $opt !~ m/net(\d+)/;
4234             my $iface = "tap${vmid}i$1";
4235
4236             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4237                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4238             }
4239
4240             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4241                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4242                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4243                 PVE::Network::tap_unplug($iface);
4244                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4245             }
4246
4247             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4248                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4249             }
4250
4251             return 1;
4252         }
4253     }
4254
4255     if ($hotplug) {
4256         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4257     } else {
4258         die "skip\n";
4259     }
4260 }
4261
4262 sub vmconfig_update_disk {
4263     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4264
4265     # fixme: do we need force?
4266
4267     my $drive = parse_drive($opt, $value);
4268
4269     if ($conf->{$opt}) {
4270
4271         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4272
4273             my $media = $drive->{media} || 'disk';
4274             my $oldmedia = $old_drive->{media} || 'disk';
4275             die "unable to change media type\n" if $media ne $oldmedia;
4276
4277             if (!drive_is_cdrom($old_drive)) {
4278
4279                 if ($drive->{file} ne $old_drive->{file}) {
4280
4281                     die "skip\n" if !$hotplug;
4282
4283                     # unplug and register as unused
4284                     vm_deviceunplug($vmid, $conf, $opt);
4285                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4286
4287                 } else {
4288                     # update existing disk
4289
4290                     # skip non hotpluggable value
4291                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4292                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4293                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4294                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4295                         die "skip\n";
4296                     }
4297
4298                     # apply throttle
4299                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4300                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4301                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4302                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4303                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4304                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4305                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4306                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4307                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4308                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4309                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4310                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4311
4312                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4313                                                    ($drive->{mbps} || 0)*1024*1024,
4314                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4315                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4316                                                    $drive->{iops} || 0,
4317                                                    $drive->{iops_rd} || 0,
4318                                                    $drive->{iops_wr} || 0,
4319                                                    ($drive->{mbps_max} || 0)*1024*1024,
4320                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4321                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4322                                                    $drive->{iops_max} || 0,
4323                                                    $drive->{iops_rd_max} || 0,
4324                                                    $drive->{iops_wr_max} || 0);
4325
4326                     }
4327
4328                     return 1;
4329                 }
4330
4331             } else { # cdrom
4332
4333                 if ($drive->{file} eq 'none') {
4334                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4335                 } else {
4336                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4337                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4338                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4339                 }
4340
4341                 return 1;
4342             }
4343         }
4344     }
4345
4346     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4347     # hotplug new disks
4348     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4349     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4350 }
4351
4352 sub vm_start {
4353     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4354         $forcemachine, $spice_ticket) = @_;
4355
4356     lock_config($vmid, sub {
4357         my $conf = load_config($vmid, $migratedfrom);
4358
4359         die "you can't start a vm if it's a template\n" if is_template($conf);
4360
4361         check_lock($conf) if !$skiplock;
4362
4363         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4364
4365         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4366             vmconfig_apply_pending($vmid, $conf, $storecfg);
4367             $conf = load_config($vmid); # update/reload
4368         }
4369
4370         my $defaults = load_defaults();
4371
4372         # set environment variable useful inside network script
4373         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4374
4375         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4376
4377         my $migrate_port = 0;
4378         my $migrate_uri;
4379         if ($statefile) {
4380             if ($statefile eq 'tcp') {
4381                 my $localip = "localhost";
4382                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4383                 my $nodename = PVE::INotify::nodename();
4384                 if ($datacenterconf->{migration_unsecure}) {
4385                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4386                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4387                 }
4388                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4389                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4390                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4391                 push @$cmd, '-incoming', $migrate_uri;
4392                 push @$cmd, '-S';
4393             } else {
4394                 push @$cmd, '-loadstate', $statefile;
4395             }
4396         } elsif ($paused) {
4397             push @$cmd, '-S';
4398         }
4399
4400         # host pci devices
4401         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4402           my $d = parse_hostpci($conf->{"hostpci$i"});
4403           next if !$d;
4404           my $pcidevices = $d->{pciid};
4405           foreach my $pcidevice (@$pcidevices) {
4406                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4407
4408                 my $info = pci_device_info("0000:$pciid");
4409                 die "IOMMU not present\n" if !check_iommu_support();
4410                 die "no pci device info for device '$pciid'\n" if !$info;
4411                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4412                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4413           }
4414         }
4415
4416         PVE::Storage::activate_volumes($storecfg, $vollist);
4417
4418         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4419                     umask => 0077); };
4420         my $err = $@;
4421         die "start failed: $err" if $err;
4422
4423         print "migration listens on $migrate_uri\n" if $migrate_uri;
4424
4425         if ($statefile && $statefile ne 'tcp')  {
4426             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4427             warn $@ if $@;
4428         }
4429
4430         if ($migratedfrom) {
4431
4432             eval {
4433                 set_migration_caps($vmid);
4434             };
4435             warn $@ if $@;
4436
4437             if ($spice_port) {
4438                 print "spice listens on port $spice_port\n";
4439                 if ($spice_ticket) {
4440                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4441                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4442                 }
4443             }
4444
4445         } else {
4446
4447             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4448                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4449                     if $conf->{balloon};
4450             }
4451
4452             foreach my $opt (keys %$conf) {
4453                 next if $opt !~  m/^net\d+$/;
4454                 my $nicconf = parse_net($conf->{$opt});
4455                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4456             }
4457         }
4458
4459         vm_mon_cmd_nocheck($vmid, 'qom-set',
4460                     path => "machine/peripheral/balloon0",
4461                     property => "guest-stats-polling-interval",
4462                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4463
4464     });
4465 }
4466
4467 sub vm_mon_cmd {
4468     my ($vmid, $execute, %params) = @_;
4469
4470     my $cmd = { execute => $execute, arguments => \%params };
4471     vm_qmp_command($vmid, $cmd);
4472 }
4473
4474 sub vm_mon_cmd_nocheck {
4475     my ($vmid, $execute, %params) = @_;
4476
4477     my $cmd = { execute => $execute, arguments => \%params };
4478     vm_qmp_command($vmid, $cmd, 1);
4479 }
4480
4481 sub vm_qmp_command {
4482     my ($vmid, $cmd, $nocheck) = @_;
4483
4484     my $res;
4485
4486     my $timeout;
4487     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4488         $timeout = $cmd->{arguments}->{timeout};
4489         delete $cmd->{arguments}->{timeout};
4490     }
4491
4492     eval {
4493         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4494         my $sname = qmp_socket($vmid);
4495         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4496             my $qmpclient = PVE::QMPClient->new();
4497
4498             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4499         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4500             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4501                 if scalar(%{$cmd->{arguments}});
4502             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4503         } else {
4504             die "unable to open monitor socket\n";
4505         }
4506     };
4507     if (my $err = $@) {
4508         syslog("err", "VM $vmid qmp command failed - $err");
4509         die $err;
4510     }
4511
4512     return $res;
4513 }
4514
4515 sub vm_human_monitor_command {
4516     my ($vmid, $cmdline) = @_;
4517
4518     my $res;
4519
4520     my $cmd = {
4521         execute => 'human-monitor-command',
4522         arguments => { 'command-line' => $cmdline},
4523     };
4524
4525     return vm_qmp_command($vmid, $cmd);
4526 }
4527
4528 sub vm_commandline {
4529     my ($storecfg, $vmid) = @_;
4530
4531     my $conf = load_config($vmid);
4532
4533     my $defaults = load_defaults();
4534
4535     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4536
4537     return join(' ', @$cmd);
4538 }
4539
4540 sub vm_reset {
4541     my ($vmid, $skiplock) = @_;
4542
4543     lock_config($vmid, sub {
4544
4545         my $conf = load_config($vmid);
4546
4547         check_lock($conf) if !$skiplock;
4548
4549         vm_mon_cmd($vmid, "system_reset");
4550     });
4551 }
4552
4553 sub get_vm_volumes {
4554     my ($conf) = @_;
4555
4556     my $vollist = [];
4557     foreach_volid($conf, sub {
4558         my ($volid, $is_cdrom) = @_;
4559
4560         return if $volid =~ m|^/|;
4561
4562         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4563         return if !$sid;
4564
4565         push @$vollist, $volid;
4566     });
4567
4568     return $vollist;
4569 }
4570
4571 sub vm_stop_cleanup {
4572     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4573
4574     eval {
4575
4576         if (!$keepActive) {
4577             my $vollist = get_vm_volumes($conf);
4578             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4579         }
4580
4581         foreach my $ext (qw(mon qmp pid vnc qga)) {
4582             unlink "/var/run/qemu-server/${vmid}.$ext";
4583         }
4584
4585         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4586     };
4587     warn $@ if $@; # avoid errors - just warn
4588 }
4589
4590 # Note: use $nockeck to skip tests if VM configuration file exists.
4591 # We need that when migration VMs to other nodes (files already moved)
4592 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4593 sub vm_stop {
4594     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4595
4596     $force = 1 if !defined($force) && !$shutdown;
4597
4598     if ($migratedfrom){
4599         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4600         kill 15, $pid if $pid;
4601         my $conf = load_config($vmid, $migratedfrom);
4602         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4603         return;
4604     }
4605
4606     lock_config($vmid, sub {
4607
4608         my $pid = check_running($vmid, $nocheck);
4609         return if !$pid;
4610
4611         my $conf;
4612         if (!$nocheck) {
4613             $conf = load_config($vmid);
4614             check_lock($conf) if !$skiplock;
4615             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4616                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4617                 $timeout = $opts->{down} if $opts->{down};
4618             }
4619         }
4620
4621         $timeout = 60 if !defined($timeout);
4622
4623         eval {
4624             if ($shutdown) {
4625                 if (defined($conf) && $conf->{agent}) {
4626                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4627                 } else {
4628                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4629                 }
4630             } else {
4631                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4632             }
4633         };
4634         my $err = $@;
4635
4636         if (!$err) {
4637             my $count = 0;
4638             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4639                 $count++;
4640                 sleep 1;
4641             }
4642
4643             if ($count >= $timeout) {
4644                 if ($force) {
4645                     warn "VM still running - terminating now with SIGTERM\n";
4646                     kill 15, $pid;
4647                 } else {
4648                     die "VM quit/powerdown failed - got timeout\n";
4649                 }
4650             } else {
4651                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4652                 return;
4653             }
4654         } else {
4655             if ($force) {
4656                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4657                 kill 15, $pid;
4658             } else {
4659                 die "VM quit/powerdown failed\n";
4660             }
4661         }
4662
4663         # wait again
4664         $timeout = 10;
4665
4666         my $count = 0;
4667         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4668             $count++;
4669             sleep 1;
4670         }
4671
4672         if ($count >= $timeout) {
4673             warn "VM still running - terminating now with SIGKILL\n";
4674             kill 9, $pid;
4675             sleep 1;
4676         }
4677
4678         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4679    });
4680 }
4681
4682 sub vm_suspend {
4683     my ($vmid, $skiplock) = @_;
4684
4685     lock_config($vmid, sub {
4686
4687         my $conf = load_config($vmid);
4688
4689         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4690
4691         vm_mon_cmd($vmid, "stop");
4692     });
4693 }
4694
4695 sub vm_resume {
4696     my ($vmid, $skiplock, $nocheck) = @_;
4697
4698     lock_config($vmid, sub {
4699
4700         if (!$nocheck) {
4701
4702             my $conf = load_config($vmid);
4703
4704             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4705
4706             vm_mon_cmd($vmid, "cont");
4707
4708         } else {
4709             vm_mon_cmd_nocheck($vmid, "cont");
4710         }
4711     });
4712 }
4713
4714 sub vm_sendkey {
4715     my ($vmid, $skiplock, $key) = @_;
4716
4717     lock_config($vmid, sub {
4718
4719         my $conf = load_config($vmid);
4720
4721         # there is no qmp command, so we use the human monitor command
4722         vm_human_monitor_command($vmid, "sendkey $key");
4723     });
4724 }
4725
4726 sub vm_destroy {
4727     my ($storecfg, $vmid, $skiplock) = @_;
4728
4729     lock_config($vmid, sub {
4730
4731         my $conf = load_config($vmid);
4732
4733         check_lock($conf) if !$skiplock;
4734
4735         if (!check_running($vmid)) {
4736             destroy_vm($storecfg, $vmid);
4737         } else {
4738             die "VM $vmid is running - destroy failed\n";
4739         }
4740     });
4741 }
4742
4743 # pci helpers
4744
4745 sub file_write {
4746     my ($filename, $buf) = @_;
4747
4748     my $fh = IO::File->new($filename, "w");
4749     return undef if !$fh;
4750
4751     my $res = print $fh $buf;
4752
4753     $fh->close();
4754
4755     return $res;
4756 }
4757
4758 sub pci_device_info {
4759     my ($name) = @_;
4760
4761     my $res;
4762
4763     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4764     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4765
4766     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4767     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4768
4769     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4770     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4771
4772     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4773     return undef if !defined($product) || $product !~ s/^0x//;
4774
4775     $res = {
4776         name => $name,
4777         vendor => $vendor,
4778         product => $product,
4779         domain => $domain,
4780         bus => $bus,
4781         slot => $slot,
4782         func => $func,
4783         irq => $irq,
4784         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4785     };
4786
4787     return $res;
4788 }
4789
4790 sub pci_dev_reset {
4791     my ($dev) = @_;
4792
4793     my $name = $dev->{name};
4794
4795     my $fn = "$pcisysfs/devices/$name/reset";
4796
4797     return file_write($fn, "1");
4798 }
4799
4800 sub pci_dev_bind_to_vfio {
4801     my ($dev) = @_;
4802
4803     my $name = $dev->{name};
4804
4805     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4806
4807     if (!-d $vfio_basedir) {
4808         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4809     }
4810     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4811
4812     my $testdir = "$vfio_basedir/$name";
4813     return 1 if -d $testdir;
4814
4815     my $data = "$dev->{vendor} $dev->{product}";
4816     return undef if !file_write("$vfio_basedir/new_id", $data);
4817
4818     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4819     if (!file_write($fn, $name)) {
4820         return undef if -f $fn;
4821     }
4822
4823     $fn = "$vfio_basedir/bind";
4824     if (! -d $testdir) {
4825         return undef if !file_write($fn, $name);
4826     }
4827
4828     return -d $testdir;
4829 }
4830
4831 sub pci_dev_group_bind_to_vfio {
4832     my ($pciid) = @_;
4833
4834     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4835
4836     if (!-d $vfio_basedir) {
4837         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4838     }
4839     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4840
4841     # get IOMMU group devices
4842     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4843       my @devs = grep /^0000:/, readdir($D);
4844     closedir($D);
4845
4846     foreach my $pciid (@devs) {
4847         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4848
4849         # pci bridges, switches or root ports are not supported
4850         # they have a pci_bus subdirectory so skip them
4851         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4852
4853         my $info = pci_device_info($1);
4854         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4855     }
4856
4857     return 1;
4858 }
4859
4860 sub print_pci_addr {
4861     my ($id, $bridges) = @_;
4862
4863     my $res = '';
4864     my $devices = {
4865         piix3 => { bus => 0, addr => 1 },
4866         #addr2 : first videocard
4867         balloon0 => { bus => 0, addr => 3 },
4868         watchdog => { bus => 0, addr => 4 },
4869         scsihw0 => { bus => 0, addr => 5 },
4870         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4871         scsihw1 => { bus => 0, addr => 6 },
4872         ahci0 => { bus => 0, addr => 7 },
4873         qga0 => { bus => 0, addr => 8 },
4874         spice => { bus => 0, addr => 9 },
4875         virtio0 => { bus => 0, addr => 10 },
4876         virtio1 => { bus => 0, addr => 11 },
4877         virtio2 => { bus => 0, addr => 12 },
4878         virtio3 => { bus => 0, addr => 13 },
4879         virtio4 => { bus => 0, addr => 14 },
4880         virtio5 => { bus => 0, addr => 15 },
4881         hostpci0 => { bus => 0, addr => 16 },
4882         hostpci1 => { bus => 0, addr => 17 },
4883         net0 => { bus => 0, addr => 18 },
4884         net1 => { bus => 0, addr => 19 },
4885         net2 => { bus => 0, addr => 20 },
4886         net3 => { bus => 0, addr => 21 },
4887         net4 => { bus => 0, addr => 22 },
4888         net5 => { bus => 0, addr => 23 },
4889         vga1 => { bus => 0, addr => 24 },
4890         vga2 => { bus => 0, addr => 25 },
4891         vga3 => { bus => 0, addr => 26 },
4892         hostpci2 => { bus => 0, addr => 27 },
4893         hostpci3 => { bus => 0, addr => 28 },
4894         #addr29 : usb-host (pve-usb.cfg)
4895         'pci.1' => { bus => 0, addr => 30 },
4896         'pci.2' => { bus => 0, addr => 31 },
4897         'net6' => { bus => 1, addr => 1 },
4898         'net7' => { bus => 1, addr => 2 },
4899         'net8' => { bus => 1, addr => 3 },
4900         'net9' => { bus => 1, addr => 4 },
4901         'net10' => { bus => 1, addr => 5 },
4902         'net11' => { bus => 1, addr => 6 },
4903         'net12' => { bus => 1, addr => 7 },
4904         'net13' => { bus => 1, addr => 8 },
4905         'net14' => { bus => 1, addr => 9 },
4906         'net15' => { bus => 1, addr => 10 },
4907         'net16' => { bus => 1, addr => 11 },
4908         'net17' => { bus => 1, addr => 12 },
4909         'net18' => { bus => 1, addr => 13 },
4910         'net19' => { bus => 1, addr => 14 },
4911         'net20' => { bus => 1, addr => 15 },
4912         'net21' => { bus => 1, addr => 16 },
4913         'net22' => { bus => 1, addr => 17 },
4914         'net23' => { bus => 1, addr => 18 },
4915         'net24' => { bus => 1, addr => 19 },
4916         'net25' => { bus => 1, addr => 20 },
4917         'net26' => { bus => 1, addr => 21 },
4918         'net27' => { bus => 1, addr => 22 },
4919         'net28' => { bus => 1, addr => 23 },
4920         'net29' => { bus => 1, addr => 24 },
4921         'net30' => { bus => 1, addr => 25 },
4922         'net31' => { bus => 1, addr => 26 },
4923         'virtio6' => { bus => 2, addr => 1 },
4924         'virtio7' => { bus => 2, addr => 2 },
4925         'virtio8' => { bus => 2, addr => 3 },
4926         'virtio9' => { bus => 2, addr => 4 },
4927         'virtio10' => { bus => 2, addr => 5 },
4928         'virtio11' => { bus => 2, addr => 6 },
4929         'virtio12' => { bus => 2, addr => 7 },
4930         'virtio13' => { bus => 2, addr => 8 },
4931         'virtio14' => { bus => 2, addr => 9 },
4932         'virtio15' => { bus => 2, addr => 10 },
4933         'virtioscsi0' => { bus => 3, addr => 1 },
4934         'virtioscsi1' => { bus => 3, addr => 2 },
4935         'virtioscsi2' => { bus => 3, addr => 3 },
4936         'virtioscsi3' => { bus => 3, addr => 4 },
4937         'virtioscsi4' => { bus => 3, addr => 5 },
4938         'virtioscsi5' => { bus => 3, addr => 6 },
4939         'virtioscsi6' => { bus => 3, addr => 7 },
4940         'virtioscsi7' => { bus => 3, addr => 8 },
4941         'virtioscsi8' => { bus => 3, addr => 9 },
4942         'virtioscsi9' => { bus => 3, addr => 10 },
4943         'virtioscsi10' => { bus => 3, addr => 11 },
4944         'virtioscsi11' => { bus => 3, addr => 12 },
4945         'virtioscsi12' => { bus => 3, addr => 13 },
4946         'virtioscsi13' => { bus => 3, addr => 14 },
4947         'virtioscsi14' => { bus => 3, addr => 15 },
4948         'virtioscsi15' => { bus => 3, addr => 16 },
4949         'virtioscsi16' => { bus => 3, addr => 17 },
4950         'virtioscsi17' => { bus => 3, addr => 18 },
4951         'virtioscsi18' => { bus => 3, addr => 19 },
4952         'virtioscsi19' => { bus => 3, addr => 20 },
4953         'virtioscsi20' => { bus => 3, addr => 21 },
4954         'virtioscsi21' => { bus => 3, addr => 22 },
4955         'virtioscsi22' => { bus => 3, addr => 23 },
4956         'virtioscsi23' => { bus => 3, addr => 24 },
4957         'virtioscsi24' => { bus => 3, addr => 25 },
4958         'virtioscsi25' => { bus => 3, addr => 26 },
4959         'virtioscsi26' => { bus => 3, addr => 27 },
4960         'virtioscsi27' => { bus => 3, addr => 28 },
4961         'virtioscsi28' => { bus => 3, addr => 29 },
4962         'virtioscsi29' => { bus => 3, addr => 30 },
4963         'virtioscsi30' => { bus => 3, addr => 31 },
4964
4965     };
4966
4967     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4968            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4969            my $bus = $devices->{$id}->{bus};
4970            $res = ",bus=pci.$bus,addr=$addr";
4971            $bridges->{$bus} = 1 if $bridges;
4972     }
4973     return $res;
4974
4975 }
4976
4977 sub print_pcie_addr {
4978     my ($id) = @_;
4979
4980     my $res = '';
4981     my $devices = {
4982         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4983         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4984         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4985         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4986     };
4987
4988     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4989            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4990            my $bus = $devices->{$id}->{bus};
4991            $res = ",bus=$bus,addr=$addr";
4992     }
4993     return $res;
4994
4995 }
4996
4997 # vzdump restore implementaion
4998
4999 sub tar_archive_read_firstfile {
5000     my $archive = shift;
5001
5002     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5003
5004     # try to detect archive type first
5005     my $pid = open (TMP, "tar tf '$archive'|") ||
5006         die "unable to open file '$archive'\n";
5007     my $firstfile = <TMP>;
5008     kill 15, $pid;
5009     close TMP;
5010
5011     die "ERROR: archive contaions no data\n" if !$firstfile;
5012     chomp $firstfile;
5013
5014     return $firstfile;
5015 }
5016
5017 sub tar_restore_cleanup {
5018     my ($storecfg, $statfile) = @_;
5019
5020     print STDERR "starting cleanup\n";
5021
5022     if (my $fd = IO::File->new($statfile, "r")) {
5023         while (defined(my $line = <$fd>)) {
5024             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5025                 my $volid = $2;
5026                 eval {
5027                     if ($volid =~ m|^/|) {
5028                         unlink $volid || die 'unlink failed\n';
5029                     } else {
5030                         PVE::Storage::vdisk_free($storecfg, $volid);
5031                     }
5032                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5033                 };
5034                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5035             } else {
5036                 print STDERR "unable to parse line in statfile - $line";
5037             }
5038         }
5039         $fd->close();
5040     }
5041 }
5042
5043 sub restore_archive {
5044     my ($archive, $vmid, $user, $opts) = @_;
5045
5046     my $format = $opts->{format};
5047     my $comp;
5048
5049     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5050         $format = 'tar' if !$format;
5051         $comp = 'gzip';
5052     } elsif ($archive =~ m/\.tar$/) {
5053         $format = 'tar' if !$format;
5054     } elsif ($archive =~ m/.tar.lzo$/) {
5055         $format = 'tar' if !$format;
5056         $comp = 'lzop';
5057     } elsif ($archive =~ m/\.vma$/) {
5058         $format = 'vma' if !$format;
5059     } elsif ($archive =~ m/\.vma\.gz$/) {
5060         $format = 'vma' if !$format;
5061         $comp = 'gzip';
5062     } elsif ($archive =~ m/\.vma\.lzo$/) {
5063         $format = 'vma' if !$format;
5064         $comp = 'lzop';
5065     } else {
5066         $format = 'vma' if !$format; # default
5067     }
5068
5069     # try to detect archive format
5070     if ($format eq 'tar') {
5071         return restore_tar_archive($archive, $vmid, $user, $opts);
5072     } else {
5073         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5074     }
5075 }
5076
5077 sub restore_update_config_line {
5078     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5079
5080     return if $line =~ m/^\#qmdump\#/;
5081     return if $line =~ m/^\#vzdump\#/;
5082     return if $line =~ m/^lock:/;
5083     return if $line =~ m/^unused\d+:/;
5084     return if $line =~ m/^parent:/;
5085     return if $line =~ m/^template:/; # restored VM is never a template
5086
5087     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5088         # try to convert old 1.X settings
5089         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5090         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5091             my ($model, $macaddr) = split(/\=/, $devconfig);
5092             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5093             my $net = {
5094                 model => $model,
5095                 bridge => "vmbr$ind",
5096                 macaddr => $macaddr,
5097             };
5098             my $netstr = print_net($net);
5099
5100             print $outfd "net$cookie->{netcount}: $netstr\n";
5101             $cookie->{netcount}++;
5102         }
5103     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5104         my ($id, $netstr) = ($1, $2);
5105         my $net = parse_net($netstr);
5106         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5107         $netstr = print_net($net);
5108         print $outfd "$id: $netstr\n";
5109     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5110         my $virtdev = $1;
5111         my $value = $3;
5112         if ($line =~ m/backup=no/) {
5113             print $outfd "#$line";
5114         } elsif ($virtdev && $map->{$virtdev}) {
5115             my $di = parse_drive($virtdev, $value);
5116             delete $di->{format}; # format can change on restore
5117             $di->{file} = $map->{$virtdev};
5118             $value = print_drive($vmid, $di);
5119             print $outfd "$virtdev: $value\n";
5120         } else {
5121             print $outfd $line;
5122         }
5123     } else {
5124         print $outfd $line;
5125     }
5126 }
5127
5128 sub scan_volids {
5129     my ($cfg, $vmid) = @_;
5130
5131     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5132
5133     my $volid_hash = {};
5134     foreach my $storeid (keys %$info) {
5135         foreach my $item (@{$info->{$storeid}}) {
5136             next if !($item->{volid} && $item->{size});
5137             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5138             $volid_hash->{$item->{volid}} = $item;
5139         }
5140     }
5141
5142     return $volid_hash;
5143 }
5144
5145 sub is_volume_in_use {
5146     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5147
5148     my $path = PVE::Storage::path($storecfg, $volid);
5149
5150     my $scan_config = sub {
5151         my ($cref, $snapname) = @_;
5152
5153         foreach my $key (keys %$cref) {
5154             my $value = $cref->{$key};
5155             if (valid_drivename($key)) {
5156                 next if $skip_drive && $key eq $skip_drive;
5157                 my $drive = parse_drive($key, $value);
5158                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5159                 return 1 if $volid eq $drive->{file};
5160                 if ($drive->{file} =~ m!^/!) {
5161                     return 1 if $drive->{file} eq $path;
5162                 } else {
5163                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5164                     next if !$storeid;
5165                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5166                     next if !$scfg;
5167                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5168                 }
5169             }
5170         }
5171
5172         return 0;
5173     };
5174
5175     return 1 if &$scan_config($conf);
5176
5177     undef $skip_drive;
5178
5179     foreach my $snapname (keys %{$conf->{snapshots}}) {
5180         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5181     }
5182
5183     return 0;
5184 }
5185
5186 sub update_disksize {
5187     my ($vmid, $conf, $volid_hash) = @_;
5188
5189     my $changes;
5190
5191     my $used = {};
5192
5193     # Note: it is allowed to define multiple storages with same path (alias), so
5194     # we need to check both 'volid' and real 'path' (two different volid can point
5195     # to the same path).
5196
5197     my $usedpath = {};
5198
5199     # update size info
5200     foreach my $opt (keys %$conf) {
5201         if (valid_drivename($opt)) {
5202             my $drive = parse_drive($opt, $conf->{$opt});
5203             my $volid = $drive->{file};
5204             next if !$volid;
5205
5206             $used->{$volid} = 1;
5207             if ($volid_hash->{$volid} &&
5208                 (my $path = $volid_hash->{$volid}->{path})) {
5209                 $usedpath->{$path} = 1;
5210             }
5211
5212             next if drive_is_cdrom($drive);
5213             next if !$volid_hash->{$volid};
5214
5215             $drive->{size} = $volid_hash->{$volid}->{size};
5216             my $new = print_drive($vmid, $drive);
5217             if ($new ne $conf->{$opt}) {
5218                 $changes = 1;
5219                 $conf->{$opt} = $new;
5220             }
5221         }
5222     }
5223
5224     # remove 'unusedX' entry if volume is used
5225     foreach my $opt (keys %$conf) {
5226         next if $opt !~ m/^unused\d+$/;
5227         my $volid = $conf->{$opt};
5228         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5229         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5230             $changes = 1;
5231             delete $conf->{$opt};
5232         }
5233     }
5234
5235     foreach my $volid (sort keys %$volid_hash) {
5236         next if $volid =~ m/vm-$vmid-state-/;
5237         next if $used->{$volid};
5238         my $path = $volid_hash->{$volid}->{path};
5239         next if !$path; # just to be sure
5240         next if $usedpath->{$path};
5241         $changes = 1;
5242         add_unused_volume($conf, $volid);
5243         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5244     }
5245
5246     return $changes;
5247 }
5248
5249 sub rescan {
5250     my ($vmid, $nolock) = @_;
5251
5252     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5253
5254     my $volid_hash = scan_volids($cfg, $vmid);
5255
5256     my $updatefn =  sub {
5257         my ($vmid) = @_;
5258
5259         my $conf = load_config($vmid);
5260
5261         check_lock($conf);
5262
5263         my $vm_volids = {};
5264         foreach my $volid (keys %$volid_hash) {
5265             my $info = $volid_hash->{$volid};
5266             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5267         }
5268
5269         my $changes = update_disksize($vmid, $conf, $vm_volids);
5270
5271         update_config_nolock($vmid, $conf, 1) if $changes;
5272     };
5273
5274     if (defined($vmid)) {
5275         if ($nolock) {
5276             &$updatefn($vmid);
5277         } else {
5278             lock_config($vmid, $updatefn, $vmid);
5279         }
5280     } else {
5281         my $vmlist = config_list();
5282         foreach my $vmid (keys %$vmlist) {
5283             if ($nolock) {
5284                 &$updatefn($vmid);
5285             } else {
5286                 lock_config($vmid, $updatefn, $vmid);
5287             }
5288         }
5289     }
5290 }
5291
5292 sub restore_vma_archive {
5293     my ($archive, $vmid, $user, $opts, $comp) = @_;
5294
5295     my $input = $archive eq '-' ? "<&STDIN" : undef;
5296     my $readfrom = $archive;
5297
5298     my $uncomp = '';
5299     if ($comp) {
5300         $readfrom = '-';
5301         my $qarchive = PVE::Tools::shellquote($archive);
5302         if ($comp eq 'gzip') {
5303             $uncomp = "zcat $qarchive|";
5304         } elsif ($comp eq 'lzop') {
5305             $uncomp = "lzop -d -c $qarchive|";
5306         } else {
5307             die "unknown compression method '$comp'\n";
5308         }
5309
5310     }
5311
5312     my $tmpdir = "/var/tmp/vzdumptmp$$";
5313     rmtree $tmpdir;
5314
5315     # disable interrupts (always do cleanups)
5316     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5317         warn "got interrupt - ignored\n";
5318     };
5319
5320     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5321     POSIX::mkfifo($mapfifo, 0600);
5322     my $fifofh;
5323
5324     my $openfifo = sub {
5325         open($fifofh, '>', $mapfifo) || die $!;
5326     };
5327
5328     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5329
5330     my $oldtimeout;
5331     my $timeout = 5;
5332
5333     my $devinfo = {};
5334
5335     my $rpcenv = PVE::RPCEnvironment::get();
5336
5337     my $conffile = config_file($vmid);
5338     my $tmpfn = "$conffile.$$.tmp";
5339
5340     # Note: $oldconf is undef if VM does not exists
5341     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5342
5343     my $print_devmap = sub {
5344         my $virtdev_hash = {};
5345
5346         my $cfgfn = "$tmpdir/qemu-server.conf";
5347
5348         # we can read the config - that is already extracted
5349         my $fh = IO::File->new($cfgfn, "r") ||
5350             "unable to read qemu-server.conf - $!\n";
5351
5352         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5353         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5354             if -f $fwcfgfn;
5355
5356         while (defined(my $line = <$fh>)) {
5357             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5358                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5359                 die "archive does not contain data for drive '$virtdev'\n"
5360                     if !$devinfo->{$devname};
5361                 if (defined($opts->{storage})) {
5362                     $storeid = $opts->{storage} || 'local';
5363                 } elsif (!$storeid) {
5364                     $storeid = 'local';
5365                 }
5366                 $format = 'raw' if !$format;
5367                 $devinfo->{$devname}->{devname} = $devname;
5368                 $devinfo->{$devname}->{virtdev} = $virtdev;
5369                 $devinfo->{$devname}->{format} = $format;
5370                 $devinfo->{$devname}->{storeid} = $storeid;
5371
5372                 # check permission on storage
5373                 my $pool = $opts->{pool}; # todo: do we need that?
5374                 if ($user ne 'root@pam') {
5375                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5376                 }
5377
5378                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5379             }
5380         }
5381
5382         foreach my $devname (keys %$devinfo) {
5383             die "found no device mapping information for device '$devname'\n"
5384                 if !$devinfo->{$devname}->{virtdev};
5385         }
5386
5387         my $cfg = cfs_read_file('storage.cfg');
5388
5389         # create empty/temp config
5390         if ($oldconf) {
5391             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5392             foreach_drive($oldconf, sub {
5393                 my ($ds, $drive) = @_;
5394
5395                 return if drive_is_cdrom($drive);
5396
5397                 my $volid = $drive->{file};
5398
5399                 return if !$volid || $volid =~ m|^/|;
5400
5401                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5402                 return if !$path || !$owner || ($owner != $vmid);
5403
5404                 # Note: only delete disk we want to restore
5405                 # other volumes will become unused
5406                 if ($virtdev_hash->{$ds}) {
5407                     PVE::Storage::vdisk_free($cfg, $volid);
5408                 }
5409             });
5410         }
5411
5412         my $map = {};
5413         foreach my $virtdev (sort keys %$virtdev_hash) {
5414             my $d = $virtdev_hash->{$virtdev};
5415             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5416             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5417
5418             # test if requested format is supported
5419             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5420             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5421             $d->{format} = $defFormat if !$supported;
5422
5423             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5424                                                   $d->{format}, undef, $alloc_size);
5425             print STDERR "new volume ID is '$volid'\n";
5426             $d->{volid} = $volid;
5427             my $path = PVE::Storage::path($cfg, $volid);
5428
5429             PVE::Storage::activate_volumes($cfg,[$volid]);
5430
5431             my $write_zeros = 1;
5432             # fixme: what other storages types initialize volumes with zero?
5433             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5434                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5435                 $write_zeros = 0;
5436             }
5437
5438             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5439
5440             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5441             $map->{$virtdev} = $volid;
5442         }
5443
5444         $fh->seek(0, 0) || die "seek failed - $!\n";
5445
5446         my $outfd = new IO::File ($tmpfn, "w") ||
5447             die "unable to write config for VM $vmid\n";
5448
5449         my $cookie = { netcount => 0 };
5450         while (defined(my $line = <$fh>)) {
5451             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5452         }
5453
5454         $fh->close();
5455         $outfd->close();
5456     };
5457
5458     eval {
5459         # enable interrupts
5460         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5461             die "interrupted by signal\n";
5462         };
5463         local $SIG{ALRM} = sub { die "got timeout\n"; };
5464
5465         $oldtimeout = alarm($timeout);
5466
5467         my $parser = sub {
5468             my $line = shift;
5469
5470             print "$line\n";
5471
5472             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5473                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5474                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5475             } elsif ($line =~ m/^CTIME: /) {
5476                 # we correctly received the vma config, so we can disable
5477                 # the timeout now for disk allocation (set to 10 minutes, so
5478                 # that we always timeout if something goes wrong)
5479                 alarm(600);
5480                 &$print_devmap();
5481                 print $fifofh "done\n";
5482                 my $tmp = $oldtimeout || 0;
5483                 $oldtimeout = undef;
5484                 alarm($tmp);
5485                 close($fifofh);
5486             }
5487         };
5488
5489         print "restore vma archive: $cmd\n";
5490         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5491     };
5492     my $err = $@;
5493
5494     alarm($oldtimeout) if $oldtimeout;
5495
5496     my $vollist = [];
5497     foreach my $devname (keys %$devinfo) {
5498         my $volid = $devinfo->{$devname}->{volid};
5499         push @$vollist, $volid if $volid;
5500     }
5501
5502     my $cfg = cfs_read_file('storage.cfg');
5503     PVE::Storage::deactivate_volumes($cfg, $vollist);
5504
5505     unlink $mapfifo;
5506
5507     if ($err) {
5508         rmtree $tmpdir;
5509         unlink $tmpfn;
5510
5511         foreach my $devname (keys %$devinfo) {
5512             my $volid = $devinfo->{$devname}->{volid};
5513             next if !$volid;
5514             eval {
5515                 if ($volid =~ m|^/|) {
5516                     unlink $volid || die 'unlink failed\n';
5517                 } else {
5518                     PVE::Storage::vdisk_free($cfg, $volid);
5519                 }
5520                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5521             };
5522             print STDERR "unable to cleanup '$volid' - $@" if $@;
5523         }
5524         die $err;
5525     }
5526
5527     rmtree $tmpdir;
5528
5529     rename($tmpfn, $conffile) ||
5530         die "unable to commit configuration file '$conffile'\n";
5531
5532     PVE::Cluster::cfs_update(); # make sure we read new file
5533
5534     eval { rescan($vmid, 1); };
5535     warn $@ if $@;
5536 }
5537
5538 sub restore_tar_archive {
5539     my ($archive, $vmid, $user, $opts) = @_;
5540
5541     if ($archive ne '-') {
5542         my $firstfile = tar_archive_read_firstfile($archive);
5543         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5544             if $firstfile ne 'qemu-server.conf';
5545     }
5546
5547     my $storecfg = cfs_read_file('storage.cfg');
5548
5549     # destroy existing data - keep empty config
5550     my $vmcfgfn = config_file($vmid);
5551     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5552
5553     my $tocmd = "/usr/lib/qemu-server/qmextract";
5554
5555     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5556     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5557     $tocmd .= ' --prealloc' if $opts->{prealloc};
5558     $tocmd .= ' --info' if $opts->{info};
5559
5560     # tar option "xf" does not autodetect compression when read from STDIN,
5561     # so we pipe to zcat
5562     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5563         PVE::Tools::shellquote("--to-command=$tocmd");
5564
5565     my $tmpdir = "/var/tmp/vzdumptmp$$";
5566     mkpath $tmpdir;
5567
5568     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5569     local $ENV{VZDUMP_VMID} = $vmid;
5570     local $ENV{VZDUMP_USER} = $user;
5571
5572     my $conffile = config_file($vmid);
5573     my $tmpfn = "$conffile.$$.tmp";
5574
5575     # disable interrupts (always do cleanups)
5576     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5577         print STDERR "got interrupt - ignored\n";
5578     };
5579
5580     eval {
5581         # enable interrupts
5582         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5583             die "interrupted by signal\n";
5584         };
5585
5586         if ($archive eq '-') {
5587             print "extracting archive from STDIN\n";
5588             run_command($cmd, input => "<&STDIN");
5589         } else {
5590             print "extracting archive '$archive'\n";
5591             run_command($cmd);
5592         }
5593
5594         return if $opts->{info};
5595
5596         # read new mapping
5597         my $map = {};
5598         my $statfile = "$tmpdir/qmrestore.stat";
5599         if (my $fd = IO::File->new($statfile, "r")) {
5600             while (defined (my $line = <$fd>)) {
5601                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5602                     $map->{$1} = $2 if $1;
5603                 } else {
5604                     print STDERR "unable to parse line in statfile - $line\n";
5605                 }
5606             }
5607             $fd->close();
5608         }
5609
5610         my $confsrc = "$tmpdir/qemu-server.conf";
5611
5612         my $srcfd = new IO::File($confsrc, "r") ||
5613             die "unable to open file '$confsrc'\n";
5614
5615         my $outfd = new IO::File ($tmpfn, "w") ||
5616             die "unable to write config for VM $vmid\n";
5617
5618         my $cookie = { netcount => 0 };
5619         while (defined (my $line = <$srcfd>)) {
5620             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5621         }
5622
5623         $srcfd->close();
5624         $outfd->close();
5625     };
5626     my $err = $@;
5627
5628     if ($err) {
5629
5630         unlink $tmpfn;
5631
5632         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5633
5634         die $err;
5635     }
5636
5637     rmtree $tmpdir;
5638
5639     rename $tmpfn, $conffile ||
5640         die "unable to commit configuration file '$conffile'\n";
5641
5642     PVE::Cluster::cfs_update(); # make sure we read new file
5643
5644     eval { rescan($vmid, 1); };
5645     warn $@ if $@;
5646 };
5647
5648
5649 # Internal snapshots
5650
5651 # NOTE: Snapshot create/delete involves several non-atomic
5652 # action, and can take a long time.
5653 # So we try to avoid locking the file and use 'lock' variable
5654 # inside the config file instead.
5655
5656 my $snapshot_copy_config = sub {
5657     my ($source, $dest) = @_;
5658
5659     foreach my $k (keys %$source) {
5660         next if $k eq 'snapshots';
5661         next if $k eq 'snapstate';
5662         next if $k eq 'snaptime';
5663         next if $k eq 'vmstate';
5664         next if $k eq 'lock';
5665         next if $k eq 'digest';
5666         next if $k eq 'description';
5667         next if $k =~ m/^unused\d+$/;
5668
5669         $dest->{$k} = $source->{$k};
5670     }
5671 };
5672
5673 my $snapshot_apply_config = sub {
5674     my ($conf, $snap) = @_;
5675
5676     # copy snapshot list
5677     my $newconf = {
5678         snapshots => $conf->{snapshots},
5679     };
5680
5681     # keep description and list of unused disks
5682     foreach my $k (keys %$conf) {
5683         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5684         $newconf->{$k} = $conf->{$k};
5685     }
5686
5687     &$snapshot_copy_config($snap, $newconf);
5688
5689     return $newconf;
5690 };
5691
5692 sub foreach_writable_storage {
5693     my ($conf, $func) = @_;
5694
5695     my $sidhash = {};
5696
5697     foreach my $ds (keys %$conf) {
5698         next if !valid_drivename($ds);
5699
5700         my $drive = parse_drive($ds, $conf->{$ds});
5701         next if !$drive;
5702         next if drive_is_cdrom($drive);
5703
5704         my $volid = $drive->{file};
5705
5706         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5707         $sidhash->{$sid} = $sid if $sid;
5708     }
5709
5710     foreach my $sid (sort keys %$sidhash) {
5711         &$func($sid);
5712     }
5713 }
5714
5715 my $alloc_vmstate_volid = sub {
5716     my ($storecfg, $vmid, $conf, $snapname) = @_;
5717
5718     # Note: we try to be smart when selecting a $target storage
5719
5720     my $target;
5721
5722     # search shared storage first
5723     foreach_writable_storage($conf, sub {
5724         my ($sid) = @_;
5725         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5726         return if !$scfg->{shared};
5727
5728         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5729     });
5730
5731     if (!$target) {
5732         # now search local storage
5733         foreach_writable_storage($conf, sub {
5734             my ($sid) = @_;
5735             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5736             return if $scfg->{shared};
5737
5738             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5739         });
5740     }
5741
5742     $target = 'local' if !$target;
5743
5744     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5745     # we abort live save after $conf->{memory}, so we need at max twice that space
5746     my $size = $conf->{memory}*2 + $driver_state_size;
5747
5748     my $name = "vm-$vmid-state-$snapname";
5749     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5750     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5751     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5752
5753     return $volid;
5754 };
5755
5756 my $snapshot_prepare = sub {
5757     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5758
5759     my $snap;
5760
5761     my $updatefn =  sub {
5762
5763         my $conf = load_config($vmid);
5764
5765         die "you can't take a snapshot if it's a template\n"
5766             if is_template($conf);
5767
5768         check_lock($conf);
5769
5770         $conf->{lock} = 'snapshot';
5771
5772         die "snapshot name '$snapname' already used\n"
5773             if defined($conf->{snapshots}->{$snapname});
5774
5775         my $storecfg = PVE::Storage::config();
5776         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5777
5778         $snap = $conf->{snapshots}->{$snapname} = {};
5779
5780         if ($save_vmstate && check_running($vmid)) {
5781             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5782         }
5783
5784         &$snapshot_copy_config($conf, $snap);
5785
5786         $snap->{snapstate} = "prepare";
5787         $snap->{snaptime} = time();
5788         $snap->{description} = $comment if $comment;
5789
5790         # always overwrite machine if we save vmstate. This makes sure we
5791         # can restore it later using correct machine type
5792         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5793
5794         update_config_nolock($vmid, $conf, 1);
5795     };
5796
5797     lock_config($vmid, $updatefn);
5798
5799     return $snap;
5800 };
5801
5802 my $snapshot_commit = sub {
5803     my ($vmid, $snapname) = @_;
5804
5805     my $updatefn = sub {
5806
5807         my $conf = load_config($vmid);
5808
5809         die "missing snapshot lock\n"
5810             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5811
5812         my $has_machine_config = defined($conf->{machine});
5813
5814         my $snap = $conf->{snapshots}->{$snapname};
5815
5816         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5817
5818         die "wrong snapshot state\n"
5819             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5820
5821         delete $snap->{snapstate};
5822         delete $conf->{lock};
5823
5824         my $newconf = &$snapshot_apply_config($conf, $snap);
5825
5826         delete $newconf->{machine} if !$has_machine_config;
5827
5828         $newconf->{parent} = $snapname;
5829
5830         update_config_nolock($vmid, $newconf, 1);
5831     };
5832
5833     lock_config($vmid, $updatefn);
5834 };
5835
5836 sub snapshot_rollback {
5837     my ($vmid, $snapname) = @_;
5838
5839     my $prepare = 1;
5840
5841     my $storecfg = PVE::Storage::config();
5842
5843     my $conf = load_config($vmid);
5844
5845     my $get_snapshot_config = sub {
5846
5847         die "you can't rollback if vm is a template\n" if is_template($conf);
5848
5849         my $res = $conf->{snapshots}->{$snapname};
5850
5851         die "snapshot '$snapname' does not exist\n" if !defined($res);
5852
5853         return $res;
5854     };
5855
5856     my $snap = &$get_snapshot_config();
5857
5858     foreach_drive($snap, sub {
5859         my ($ds, $drive) = @_;
5860
5861         return if drive_is_cdrom($drive);
5862
5863         my $volid = $drive->{file};
5864
5865         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5866     });
5867
5868     my $updatefn = sub {
5869
5870         $conf = load_config($vmid);
5871
5872         $snap = &$get_snapshot_config();
5873
5874         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5875             if $snap->{snapstate};
5876
5877         if ($prepare) {
5878             check_lock($conf);
5879             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5880         }
5881
5882         die "unable to rollback vm $vmid: vm is running\n"
5883             if check_running($vmid);
5884
5885         if ($prepare) {
5886             $conf->{lock} = 'rollback';
5887         } else {
5888             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5889             delete $conf->{lock};
5890         }
5891
5892         my $forcemachine;
5893
5894         if (!$prepare) {
5895             my $has_machine_config = defined($conf->{machine});
5896
5897             # copy snapshot config to current config
5898             $conf = &$snapshot_apply_config($conf, $snap);
5899             $conf->{parent} = $snapname;
5900
5901             # Note: old code did not store 'machine', so we try to be smart
5902             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5903             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5904             # we remove the 'machine' configuration if not explicitly specified
5905             # in the original config.
5906             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5907         }
5908
5909         update_config_nolock($vmid, $conf, 1);
5910
5911         if (!$prepare && $snap->{vmstate}) {
5912             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5913             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5914         }
5915     };
5916
5917     lock_config($vmid, $updatefn);
5918
5919     foreach_drive($snap, sub {
5920         my ($ds, $drive) = @_;
5921
5922         return if drive_is_cdrom($drive);
5923
5924         my $volid = $drive->{file};
5925         my $device = "drive-$ds";
5926
5927         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5928     });
5929
5930     $prepare = 0;
5931     lock_config($vmid, $updatefn);
5932 }
5933
5934 my $savevm_wait = sub {
5935     my ($vmid) = @_;
5936
5937     for(;;) {
5938         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5939         if (!$stat->{status}) {
5940             die "savevm not active\n";
5941         } elsif ($stat->{status} eq 'active') {
5942             sleep(1);
5943             next;
5944         } elsif ($stat->{status} eq 'completed') {
5945             last;
5946         } else {
5947             die "query-savevm returned status '$stat->{status}'\n";
5948         }
5949     }
5950 };
5951
5952 sub do_snapshots_with_qemu {
5953     my ($storecfg, $volid) = @_;
5954
5955     my $storage_name = PVE::Storage::parse_volume_id($volid);
5956
5957     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5958         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5959         return 1;
5960     }
5961
5962     if ($volid =~ m/\.(qcow2|qed)$/){
5963         return 1;
5964     }
5965
5966     return undef;
5967 }
5968
5969 sub snapshot_create {
5970     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5971
5972     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5973
5974     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5975
5976     my $config = load_config($vmid);
5977
5978     my $running = check_running($vmid);
5979
5980     my $freezefs = $running && $config->{agent};
5981     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5982
5983     my $drivehash = {};
5984
5985     if ($freezefs) {
5986         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5987         warn "guest-fsfreeze-freeze problems - $@" if $@;
5988     }
5989
5990     eval {
5991         # create internal snapshots of all drives
5992
5993         my $storecfg = PVE::Storage::config();
5994
5995         if ($running) {
5996             if ($snap->{vmstate}) {
5997                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5998                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5999                 &$savevm_wait($vmid);
6000             } else {
6001                 vm_mon_cmd($vmid, "savevm-start");
6002             }
6003         };
6004
6005         foreach_drive($snap, sub {
6006             my ($ds, $drive) = @_;
6007
6008             return if drive_is_cdrom($drive);
6009
6010             my $volid = $drive->{file};
6011             my $device = "drive-$ds";
6012
6013             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
6014             $drivehash->{$ds} = 1;
6015        });
6016     };
6017     my $err = $@;
6018
6019     if ($running) {
6020         eval { vm_mon_cmd($vmid, "savevm-end")  };
6021         warn $@ if $@;
6022
6023         if ($freezefs) {
6024             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6025             warn "guest-fsfreeze-thaw problems - $@" if $@;
6026         }
6027
6028         # savevm-end is async, we need to wait
6029         for (;;) {
6030             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6031             if (!$stat->{bytes}) {
6032                 last;
6033             } else {
6034                 print "savevm not yet finished\n";
6035                 sleep(1);
6036                 next;
6037             }
6038         }
6039     }
6040
6041     if ($err) {
6042         warn "snapshot create failed: starting cleanup\n";
6043         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6044         warn $@ if $@;
6045         die $err;
6046     }
6047
6048     &$snapshot_commit($vmid, $snapname);
6049 }
6050
6051 # Note: $drivehash is only set when called from snapshot_create.
6052 sub snapshot_delete {
6053     my ($vmid, $snapname, $force, $drivehash) = @_;
6054
6055     my $prepare = 1;
6056
6057     my $snap;
6058     my $unused = [];
6059
6060     my $unlink_parent = sub {
6061         my ($confref, $new_parent) = @_;
6062
6063         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6064             if ($new_parent) {
6065                 $confref->{parent} = $new_parent;
6066             } else {
6067                 delete $confref->{parent};
6068             }
6069         }
6070     };
6071
6072     my $updatefn =  sub {
6073         my ($remove_drive) = @_;
6074
6075         my $conf = load_config($vmid);
6076
6077         if (!$drivehash) {
6078             check_lock($conf);
6079             die "you can't delete a snapshot if vm is a template\n"
6080                 if is_template($conf);
6081         }
6082
6083         $snap = $conf->{snapshots}->{$snapname};
6084
6085         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6086
6087         # remove parent refs
6088         if (!$prepare) {
6089             &$unlink_parent($conf, $snap->{parent});
6090             foreach my $sn (keys %{$conf->{snapshots}}) {
6091                 next if $sn eq $snapname;
6092                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6093             }
6094         }
6095
6096         if ($remove_drive) {
6097             if ($remove_drive eq 'vmstate') {
6098                 delete $snap->{$remove_drive};
6099             } else {
6100                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6101                 my $volid = $drive->{file};
6102                 delete $snap->{$remove_drive};
6103                 add_unused_volume($conf, $volid);
6104             }
6105         }
6106
6107         if ($prepare) {
6108             $snap->{snapstate} = 'delete';
6109         } else {
6110             delete $conf->{snapshots}->{$snapname};
6111             delete $conf->{lock} if $drivehash;
6112             foreach my $volid (@$unused) {
6113                 add_unused_volume($conf, $volid);
6114             }
6115         }
6116
6117         update_config_nolock($vmid, $conf, 1);
6118     };
6119
6120     lock_config($vmid, $updatefn);
6121
6122     # now remove vmstate file
6123
6124     my $storecfg = PVE::Storage::config();
6125
6126     if ($snap->{vmstate}) {
6127         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6128         if (my $err = $@) {
6129             die $err if !$force;
6130             warn $err;
6131         }
6132         # save changes (remove vmstate from snapshot)
6133         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6134     };
6135
6136     # now remove all internal snapshots
6137     foreach_drive($snap, sub {
6138         my ($ds, $drive) = @_;
6139
6140         return if drive_is_cdrom($drive);
6141
6142         my $volid = $drive->{file};
6143         my $device = "drive-$ds";
6144
6145         if (!$drivehash || $drivehash->{$ds}) {
6146             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6147             if (my $err = $@) {
6148                 die $err if !$force;
6149                 warn $err;
6150             }
6151         }
6152
6153         # save changes (remove drive fron snapshot)
6154         lock_config($vmid, $updatefn, $ds) if !$force;
6155         push @$unused, $volid;
6156     });
6157
6158     # now cleanup config
6159     $prepare = 0;
6160     lock_config($vmid, $updatefn);
6161 }
6162
6163 sub has_feature {
6164     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6165
6166     my $err;
6167     foreach_drive($conf, sub {
6168         my ($ds, $drive) = @_;
6169
6170         return if drive_is_cdrom($drive);
6171         my $volid = $drive->{file};
6172         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6173     });
6174
6175     return $err ? 0 : 1;
6176 }
6177
6178 sub template_create {
6179     my ($vmid, $conf, $disk) = @_;
6180
6181     my $storecfg = PVE::Storage::config();
6182
6183     foreach_drive($conf, sub {
6184         my ($ds, $drive) = @_;
6185
6186         return if drive_is_cdrom($drive);
6187         return if $disk && $ds ne $disk;
6188
6189         my $volid = $drive->{file};
6190         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6191
6192         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6193         $drive->{file} = $voliddst;
6194         $conf->{$ds} = print_drive($vmid, $drive);
6195         update_config_nolock($vmid, $conf, 1);
6196     });
6197 }
6198
6199 sub is_template {
6200     my ($conf) = @_;
6201
6202     return 1 if defined $conf->{template} && $conf->{template} == 1;
6203 }
6204
6205 sub qemu_img_convert {
6206     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6207
6208     my $storecfg = PVE::Storage::config();
6209     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6210     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6211
6212     if ($src_storeid && $dst_storeid) {
6213
6214         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6215
6216         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6217         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6218
6219         my $src_format = qemu_img_format($src_scfg, $src_volname);
6220         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6221
6222         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6223         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6224
6225         my $cmd = [];
6226         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6227         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6228         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6229
6230         my $parser = sub {
6231             my $line = shift;
6232             if($line =~ m/\((\S+)\/100\%\)/){
6233                 my $percent = $1;
6234                 my $transferred = int($size * $percent / 100);
6235                 my $remaining = $size - $transferred;
6236
6237                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6238             }
6239
6240         };
6241
6242         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6243         my $err = $@;
6244         die "copy failed: $err" if $err;
6245     }
6246 }
6247
6248 sub qemu_img_format {
6249     my ($scfg, $volname) = @_;
6250
6251     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6252         return $1;
6253     } else {
6254         return "raw";
6255     }
6256 }
6257
6258 sub qemu_drive_mirror {
6259     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6260
6261     my $storecfg = PVE::Storage::config();
6262     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6263
6264     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6265
6266     my $format = qemu_img_format($dst_scfg, $dst_volname);
6267
6268     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6269
6270     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6271     $opts->{format} = $format if $format;
6272
6273     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6274
6275     eval {
6276     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6277         while (1) {
6278             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6279             my $stat = @$stats[0];
6280             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6281             die "error job is not mirroring" if $stat->{type} ne "mirror";
6282
6283             my $busy = $stat->{busy};
6284             my $ready = $stat->{ready};
6285
6286             if (my $total = $stat->{len}) {
6287                 my $transferred = $stat->{offset} || 0;
6288                 my $remaining = $total - $transferred;
6289                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6290
6291                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6292             }
6293
6294
6295             if ($stat->{ready} eq 'true') {
6296
6297                 last if $vmiddst != $vmid;
6298
6299                 # try to switch the disk if source and destination are on the same guest
6300                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6301                 last if !$@;
6302                 die $@ if $@ !~ m/cannot be completed/;
6303             }
6304             sleep 1;
6305         }
6306
6307
6308     };
6309     my $err = $@;
6310
6311     my $cancel_job = sub {
6312         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6313         while (1) {
6314             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6315             my $stat = @$stats[0];
6316             last if !$stat;
6317             sleep 1;
6318         }
6319     };
6320
6321     if ($err) {
6322         eval { &$cancel_job(); };
6323         die "mirroring error: $err";
6324     }
6325
6326     if ($vmiddst != $vmid) {
6327         # if we clone a disk for a new target vm, we don't switch the disk
6328         &$cancel_job(); # so we call block-job-cancel
6329     }
6330 }
6331
6332 sub clone_disk {
6333     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6334         $newvmid, $storage, $format, $full, $newvollist) = @_;
6335
6336     my $newvolid;
6337
6338     if (!$full) {
6339         print "create linked clone of drive $drivename ($drive->{file})\n";
6340         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6341         push @$newvollist, $newvolid;
6342     } else {
6343         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6344         $storeid = $storage if $storage;
6345
6346         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6347         if (!$format) {
6348             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6349             $format = qemu_img_format($scfg, $volname);
6350         }
6351
6352         # test if requested format is supported - else use default
6353         my $supported = grep { $_ eq $format } @$validFormats;
6354         $format = $defFormat if !$supported;
6355
6356         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6357
6358         print "create full clone of drive $drivename ($drive->{file})\n";
6359         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6360         push @$newvollist, $newvolid;
6361
6362         PVE::Storage::activate_volumes($storecfg, $newvollist);
6363
6364         if (!$running || $snapname) {
6365             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6366         } else {
6367             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6368         }
6369     }
6370
6371     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6372
6373     my $disk = $drive;
6374     $disk->{format} = undef;
6375     $disk->{file} = $newvolid;
6376     $disk->{size} = $size;
6377
6378     return $disk;
6379 }
6380
6381 # this only works if VM is running
6382 sub get_current_qemu_machine {
6383     my ($vmid) = @_;
6384
6385     my $cmd = { execute => 'query-machines', arguments => {} };
6386     my $res = vm_qmp_command($vmid, $cmd);
6387
6388     my ($current, $default);
6389     foreach my $e (@$res) {
6390         $default = $e->{name} if $e->{'is-default'};
6391         $current = $e->{name} if $e->{'is-current'};
6392     }
6393
6394     # fallback to the default machine if current is not supported by qemu
6395     return $current || $default || 'pc';
6396 }
6397
6398 sub qemu_machine_feature_enabled {
6399     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6400
6401     my $current_major;
6402     my $current_minor;
6403
6404     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6405
6406         $current_major = $3;
6407         $current_minor = $4;
6408
6409     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6410
6411         $current_major = $1;
6412         $current_minor = $2;
6413     }
6414
6415     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6416
6417
6418 }
6419
6420 sub qemu_machine_pxe {
6421     my ($vmid, $conf, $machine) = @_;
6422
6423     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6424
6425     foreach my $opt (keys %$conf) {
6426         next if $opt !~ m/^net(\d+)$/;
6427         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6428         next if !$net;
6429         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6430         return $machine.".pxe" if $romfile =~ m/pxe/;
6431         last;
6432     }
6433
6434 }
6435
6436 sub qemu_use_old_bios_files {
6437     my ($machine_type) = @_;
6438
6439     return if !$machine_type;
6440
6441     my $use_old_bios_files = undef;
6442
6443     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6444         $machine_type = $1;
6445         $use_old_bios_files = 1;
6446     } else {
6447         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6448         # load new efi bios files on migration. So this hack is required to allow
6449         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6450         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6451         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6452     }
6453
6454     return ($use_old_bios_files, $machine_type);
6455 }
6456
6457 sub lspci {
6458
6459     my $devices = {};
6460
6461     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6462             my (undef, $id, $function) = @_;
6463             my $res = { id => $id, function => $function};
6464             push @{$devices->{$id}}, $res;
6465     });
6466
6467     return $devices;
6468 }
6469
6470 sub vm_iothreads_list {
6471     my ($vmid) = @_;
6472
6473     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6474
6475     my $iothreads = {};
6476     foreach my $iothread (@$res) {
6477         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6478     }
6479
6480     return $iothreads;
6481 }
6482
6483 sub scsihw_infos {
6484     my ($conf, $drive) = @_;
6485
6486     my $maxdev = 0;
6487
6488     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6489         $maxdev = 7;
6490     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6491         $maxdev = 1;
6492     } else {
6493         $maxdev = 256;
6494     }
6495
6496     my $controller = int($drive->{index} / $maxdev);
6497     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6498
6499     return ($maxdev, $controller, $controller_prefix);
6500 }
6501
6502 # bash completion helper
6503
6504 sub complete_backup_archives {
6505     my ($cmdname, $pname, $cvalue) = @_;
6506
6507     my $cfg = PVE::Storage::config();
6508
6509     my $storeid;
6510
6511     if ($cvalue =~ m/^([^:]+):/) {
6512         $storeid = $1;
6513     }
6514
6515     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6516
6517     my $res = [];
6518     foreach my $id (keys %$data) {
6519         foreach my $item (@{$data->{$id}}) {
6520             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6521             push @$res, $item->{volid} if defined($item->{volid});
6522         }
6523     }
6524
6525     return $res;
6526 }
6527
6528 my $complete_vmid_full = sub {
6529     my ($running) = @_;
6530
6531     my $idlist = vmstatus();
6532
6533     my $res = [];
6534
6535     foreach my $id (keys %$idlist) {
6536         my $d = $idlist->{$id};
6537         if (defined($running)) {
6538             next if $d->{template};
6539             next if $running && $d->{status} ne 'running';
6540             next if !$running && $d->{status} eq 'running';
6541         }
6542         push @$res, $id;
6543
6544     }
6545     return $res;
6546 };
6547
6548 sub complete_vmid {
6549     return &$complete_vmid_full();
6550 }
6551
6552 sub complete_vmid_stopped {
6553     return &$complete_vmid_full(0);
6554 }
6555
6556 sub complete_vmid_running {
6557     return &$complete_vmid_full(1);
6558 }
6559
6560 sub complete_storage {
6561
6562     my $cfg = PVE::Storage::config();
6563     my $ids = $cfg->{ids};
6564
6565     my $res = [];
6566     foreach my $sid (keys %$ids) {
6567         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6568         next if !$ids->{$sid}->{content}->{images};
6569         push @$res, $sid;
6570     }
6571
6572     return $res;
6573 }
6574
6575 1;