PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 385         maxLength => 256,
 386         optional => 1,
 387     },
 388     protection => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 392         default => 0,
 393     },
 394     bios => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [ qw(seabios ovmf) ],
 398         description => "Select BIOS implementation.",
 399         default => 'seabios',
 400     },
 401 };
 402
 403 # what about other qemu settings ?
 404 #cpu => 'string',
 405 #machine => 'string',
 406 #fda => 'file',
 407 #fdb => 'file',
 408 #mtdblock => 'file',
 409 #sd => 'file',
 410 #pflash => 'file',
 411 #snapshot => 'bool',
 412 #bootp => 'file',
 413 ##tftp => 'dir',
 414 ##smb => 'dir',
 415 #kernel => 'file',
 416 #append => 'string',
 417 #initrd => 'file',
 418 ##soundhw => 'string',
 419
 420 while (my ($k, $v) = each %$confdesc) {
 421     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 422 }
 423
 424 my $MAX_IDE_DISKS = 4;
 425 my $MAX_SCSI_DISKS = 14;
 426 my $MAX_VIRTIO_DISKS = 16;
 427 my $MAX_SATA_DISKS = 6;
 428 my $MAX_USB_DEVICES = 5;
 429 my $MAX_NETS = 32;
 430 my $MAX_UNUSED_DISKS = 8;
 431 my $MAX_HOSTPCI_DEVICES = 4;
 432 my $MAX_SERIAL_PORTS = 4;
 433 my $MAX_PARALLEL_PORTS = 3;
 434 my $MAX_NUMA = 8;
 435 my $MAX_MEM = 4194304;
 436 my $STATICMEM = 1024;
 437
 438 my $numadesc = {
 439     optional => 1,
 440     type => 'string', format => 'pve-qm-numanode',
 441     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 442     description => "numa topology",
 443 };
 444 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 445
 446 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 447     $confdesc->{"numa$i"} = $numadesc;
 448 }
 449
 450 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 451                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 452                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 453 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 454
 455 my $netdesc = {
 456     optional => 1,
 457     type => 'string', format => 'pve-qm-net',
 458     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 459     description => <<EODESCR,
 460 Specify network devices.
 461
 462 MODEL is one of: $nic_model_list_txt
 463
 464 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 465 automatically generated if not specified.
 466
 467 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 468
 469 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 470
 471 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 472
 473 10.0.2.2   Gateway
 474 10.0.2.3   DNS Server
 475 10.0.2.4   SMB Server
 476
 477 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 478
 479 EODESCR
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 482
 483 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 484     $confdesc->{"net$i"} = $netdesc;
 485 }
 486
 487 my $drivename_hash;
 488
 489 my $idedesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial][,model=model]',
 493     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 496
 497 my $scsidesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 501     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 504
 505 my $satadesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial]',
 509     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 512
 513 my $virtiodesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-drive',
 516     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,serial=serial]',
 517     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 518 };
 519 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 520
 521 my $usbdesc = {
 522     optional => 1,
 523     type => 'string', format => 'pve-qm-usb-device',
 524     typetext => 'host=HOSTUSBDEVICE|spice',
 525     description => <<EODESCR,
 526 Configure an USB device (n is 0 to 4). This can be used to
 527 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 528
 529 'bus-port(.port)*' (decimal numbers) or
 530 'vendor_id:product_id' (hexadeciaml numbers)
 531
 532 You can use the 'lsusb -t' command to list existing usb devices.
 533
 534 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 535
 536 The value 'spice' can be used to add a usb redirection devices for spice.
 537
 538 EODESCR
 539 };
 540 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 541
 542 my $hostpcidesc = {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-hostpci',
 545         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 546         description => <<EODESCR,
 547 Map host pci devices. HOSTPCIDEVICE syntax is:
 548
 549 'bus:dev.func' (hexadecimal numbers)
 550
 551 You can us the 'lspci' command to list existing pci devices.
 552
 553 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 561
 562 my $serialdesc = {
 563         optional => 1,
 564         type => 'string',
 565         pattern => '(/dev/.+|socket)',
 566         description =>  <<EODESCR,
 567 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 568
 569 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 570
 571 Experimental: user reported problems with this option.
 572 EODESCR
 573 };
 574
 575 my $paralleldesc= {
 576         optional => 1,
 577         type => 'string',
 578         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 579         description =>  <<EODESCR,
 580 Map host parallel devices (n is 0 to 2).
 581
 582 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 583
 584 Experimental: user reported problems with this option.
 585 EODESCR
 586 };
 587
 588 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 589     $confdesc->{"parallel$i"} = $paralleldesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 593     $confdesc->{"serial$i"} = $serialdesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 597     $confdesc->{"hostpci$i"} = $hostpcidesc;
 598 }
 599
 600 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 601     $drivename_hash->{"ide$i"} = 1;
 602     $confdesc->{"ide$i"} = $idedesc;
 603 }
 604
 605 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 606     $drivename_hash->{"sata$i"} = 1;
 607     $confdesc->{"sata$i"} = $satadesc;
 608 }
 609
 610 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 611     $drivename_hash->{"scsi$i"} = 1;
 612     $confdesc->{"scsi$i"} = $scsidesc ;
 613 }
 614
 615 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 616     $drivename_hash->{"virtio$i"} = 1;
 617     $confdesc->{"virtio$i"} = $virtiodesc;
 618 }
 619
 620 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 621     $confdesc->{"usb$i"} = $usbdesc;
 622 }
 623
 624 my $unuseddesc = {
 625     optional => 1,
 626     type => 'string', format => 'pve-volume-id',
 627     description => "Reference to unused volumes.",
 628 };
 629
 630 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 631     $confdesc->{"unused$i"} = $unuseddesc;
 632 }
 633
 634 my $kvm_api_version = 0;
 635
 636 sub kvm_version {
 637
 638     return $kvm_api_version if $kvm_api_version;
 639
 640     my $fh = IO::File->new("</dev/kvm") ||
 641         return 0;
 642
 643     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 644         $kvm_api_version = $v;
 645     }
 646
 647     $fh->close();
 648
 649     return  $kvm_api_version;
 650 }
 651
 652 my $kvm_user_version;
 653
 654 sub kvm_user_version {
 655
 656     return $kvm_user_version if $kvm_user_version;
 657
 658     $kvm_user_version = 'unknown';
 659
 660     my $tmp = `kvm -help 2>/dev/null`;
 661
 662     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 663         $kvm_user_version = $2;
 664     }
 665
 666     return $kvm_user_version;
 667
 668 }
 669
 670 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 671
 672 sub disknames {
 673     # order is important - used to autoselect boot disk
 674     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 675             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 676             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 677             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 678 }
 679
 680 sub valid_drivename {
 681     my $dev = shift;
 682
 683     return defined($drivename_hash->{$dev});
 684 }
 685
 686 sub option_exists {
 687     my $key = shift;
 688     return defined($confdesc->{$key});
 689 }
 690
 691 sub nic_models {
 692     return $nic_model_list;
 693 }
 694
 695 sub os_list_description {
 696
 697     return {
 698         other => 'Other',
 699         wxp => 'Windows XP',
 700         w2k => 'Windows 2000',
 701         w2k3 =>, 'Windows 2003',
 702         w2k8 => 'Windows 2008',
 703         wvista => 'Windows Vista',
 704         win7 => 'Windows 7',
 705         win8 => 'Windows 8/2012',
 706         l24 => 'Linux 2.4',
 707         l26 => 'Linux 2.6',
 708     };
 709 }
 710
 711 my $cdrom_path;
 712
 713 sub get_cdrom_path {
 714
 715     return  $cdrom_path if $cdrom_path;
 716
 717     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 718     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 719     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 720 }
 721
 722 sub get_iso_path {
 723     my ($storecfg, $vmid, $cdrom) = @_;
 724
 725     if ($cdrom eq 'cdrom') {
 726         return get_cdrom_path();
 727     } elsif ($cdrom eq 'none') {
 728         return '';
 729     } elsif ($cdrom =~ m|^/|) {
 730         return $cdrom;
 731     } else {
 732         return PVE::Storage::path($storecfg, $cdrom);
 733     }
 734 }
 735
 736 # try to convert old style file names to volume IDs
 737 sub filename_to_volume_id {
 738     my ($vmid, $file, $media) = @_;
 739
 740     if (!($file eq 'none' || $file eq 'cdrom' ||
 741           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 742
 743         return undef if $file =~ m|/|;
 744
 745         if ($media && $media eq 'cdrom') {
 746             $file = "local:iso/$file";
 747         } else {
 748             $file = "local:$vmid/$file";
 749         }
 750     }
 751
 752     return $file;
 753 }
 754
 755 sub verify_media_type {
 756     my ($opt, $vtype, $media) = @_;
 757
 758     return if !$media;
 759
 760     my $etype;
 761     if ($media eq 'disk') {
 762         $etype = 'images';
 763     } elsif ($media eq 'cdrom') {
 764         $etype = 'iso';
 765     } else {
 766         die "internal error";
 767     }
 768
 769     return if ($vtype eq $etype);
 770
 771     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 772 }
 773
 774 sub cleanup_drive_path {
 775     my ($opt, $storecfg, $drive) = @_;
 776
 777     # try to convert filesystem paths to volume IDs
 778
 779     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 780         ($drive->{file} !~ m|^/dev/.+|) &&
 781         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 782         ($drive->{file} !~ m/^\d+$/)) {
 783         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 784         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 785         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 786         verify_media_type($opt, $vtype, $drive->{media});
 787         $drive->{file} = $volid;
 788     }
 789
 790     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 791 }
 792
 793 sub create_conf_nolock {
 794     my ($vmid, $settings) = @_;
 795
 796     my $filename = config_file($vmid);
 797
 798     die "configuration file '$filename' already exists\n" if -f $filename;
 799
 800     my $defaults = load_defaults();
 801
 802     $settings->{name} = "vm$vmid" if !$settings->{name};
 803     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 804
 805     my $data = '';
 806     foreach my $opt (keys %$settings) {
 807         next if !$confdesc->{$opt};
 808
 809         my $value = $settings->{$opt};
 810         next if !$value;
 811
 812         $data .= "$opt: $value\n";
 813     }
 814
 815     PVE::Tools::file_set_contents($filename, $data);
 816 }
 817
 818 sub parse_hotplug_features {
 819     my ($data) = @_;
 820
 821     my $res = {};
 822
 823     return $res if $data eq '0';
 824
 825     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 826
 827     foreach my $feature (PVE::Tools::split_list($data)) {
 828         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 829             $res->{$1} = 1;
 830         } else {
 831             warn "ignoring unknown hotplug feature '$feature'\n";
 832         }
 833     }
 834     return $res;
 835 }
 836
 837 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 838 sub pve_verify_hotplug_features {
 839     my ($value, $noerr) = @_;
 840
 841     return $value if parse_hotplug_features($value);
 842
 843     return undef if $noerr;
 844
 845     die "unable to parse hotplug option\n";
 846 }
 847
 848 my $parse_size = sub {
 849     my ($value) = @_;
 850
 851     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 852     my ($size, $unit) = ($1, $3);
 853     if ($unit) {
 854         if ($unit eq 'K') {
 855             $size = $size * 1024;
 856         } elsif ($unit eq 'M') {
 857             $size = $size * 1024 * 1024;
 858         } elsif ($unit eq 'G') {
 859             $size = $size * 1024 * 1024 * 1024;
 860         }
 861     }
 862     return int($size);
 863 };
 864
 865 my $format_size = sub {
 866     my ($size) = @_;
 867
 868     $size = int($size);
 869
 870     my $kb = int($size/1024);
 871     return $size if $kb*1024 != $size;
 872
 873     my $mb = int($kb/1024);
 874     return "${kb}K" if $mb*1024 != $kb;
 875
 876     my $gb = int($mb/1024);
 877     return "${mb}M" if $gb*1024 != $mb;
 878
 879     return "${gb}G";
 880 };
 881
 882 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 883 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 884 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 885 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
 886 #        [,iothread=on][,serial=serial][,model=model]
 887
 888 sub parse_drive {
 889     my ($key, $data) = @_;
 890
 891     my $res = {};
 892
 893     # $key may be undefined - used to verify JSON parameters
 894     if (!defined($key)) {
 895         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 896         $res->{index} = 0;
 897     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 898         $res->{interface} = $1;
 899         $res->{index} = $2;
 900     } else {
 901         return undef;
 902     }
 903
 904     foreach my $p (split (/,/, $data)) {
 905         next if $p =~ m/^\s*$/;
 906
 907         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|detect_zeroes|iothread|queues|serial|model)=(.+)$/) {
 908             my ($k, $v) = ($1, $2);
 909
 910             $k = 'file' if $k eq 'volume';
 911
 912             return undef if defined $res->{$k};
 913
 914             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 915                 return undef if !$v || $v !~ m/^\d+/;
 916                 $k = "m$k";
 917                 $v = sprintf("%.3f", $v / (1024*1024));
 918             }
 919             $res->{$k} = $v;
 920         } else {
 921             if (!$res->{file} && $p !~ m/=/) {
 922                 $res->{file} = $p;
 923             } else {
 924                 return undef;
 925             }
 926         }
 927     }
 928
 929     return undef if !$res->{file};
 930
 931     return undef if $res->{cache} &&
 932         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 933     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 934     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 935     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 936     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 937     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 938     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 939     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 940     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 941     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 942     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 943     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 944     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 945     return undef if $res->{detect_zeroes} && $res->{detect_zeroes} !~ m/^(on|off)$/;
 946     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 947     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 948
 949     return undef if $res->{mbps_rd} && $res->{mbps};
 950     return undef if $res->{mbps_wr} && $res->{mbps};
 951
 952     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 957     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 958
 959     return undef if $res->{iops_rd} && $res->{iops};
 960     return undef if $res->{iops_wr} && $res->{iops};
 961
 962
 963     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 964     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 966     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 968     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 969
 970     if ($res->{size}) {
 971         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 972     }
 973
 974     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 975         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 976         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 977         return undef if $res->{interface} eq 'virtio';
 978     }
 979
 980     # rerror does not work with scsi drives
 981     if ($res->{rerror}) {
 982         return undef if $res->{interface} eq 'scsi';
 983     }
 984
 985     return $res;
 986 }
 987
 988 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 989
 990 sub print_drive {
 991     my ($vmid, $drive) = @_;
 992
 993     my $opts = '';
 994     foreach my $o (@qemu_drive_options, qw(mbps mbps_rd mbps_wr mbps_max mbps_rd_max mbps_wr_max backup iothread queues detect_zeroes)) {
 995         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 996     }
 997
 998     if ($drive->{size}) {
 999         $opts .= ",size=" . &$format_size($drive->{size});
1000     }
1001
1002     if (my $model = $drive->{model}) {
1003         $opts .= ",model=$model";
1004     }
1005
1006     return "$drive->{file}$opts";
1007 }
1008
1009 sub scsi_inquiry {
1010     my($fh, $noerr) = @_;
1011
1012     my $SG_IO = 0x2285;
1013     my $SG_GET_VERSION_NUM = 0x2282;
1014
1015     my $versionbuf = "\x00" x 8;
1016     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1017     if (!$ret) {
1018         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1019         return undef;
1020     }
1021     my $version = unpack("I", $versionbuf);
1022     if ($version < 30000) {
1023         die "scsi generic interface too old\n"  if !$noerr;
1024         return undef;
1025     }
1026
1027     my $buf = "\x00" x 36;
1028     my $sensebuf = "\x00" x 8;
1029     my $cmd = pack("C x3 C x1", 0x12, 36);
1030
1031     # see /usr/include/scsi/sg.h
1032     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1033
1034     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1035                       length($sensebuf), 0, length($buf), $buf,
1036                       $cmd, $sensebuf, 6000);
1037
1038     $ret = ioctl($fh, $SG_IO, $packet);
1039     if (!$ret) {
1040         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1041         return undef;
1042     }
1043
1044     my @res = unpack($sg_io_hdr_t, $packet);
1045     if ($res[17] || $res[18]) {
1046         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1047         return undef;
1048     }
1049
1050     my $res = {};
1051     (my $byte0, my $byte1, $res->{vendor},
1052      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1053
1054     $res->{removable} = $byte1 & 128 ? 1 : 0;
1055     $res->{type} = $byte0 & 31;
1056
1057     return $res;
1058 }
1059
1060 sub path_is_scsi {
1061     my ($path) = @_;
1062
1063     my $fh = IO::File->new("+<$path") || return undef;
1064     my $res = scsi_inquiry($fh, 1);
1065     close($fh);
1066
1067     return $res;
1068 }
1069
1070 sub machine_type_is_q35 {
1071     my ($conf) = @_;
1072
1073     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1074 }
1075
1076 sub print_tabletdevice_full {
1077     my ($conf) = @_;
1078
1079     my $q35 = machine_type_is_q35($conf);
1080
1081     # we use uhci for old VMs because tablet driver was buggy in older qemu
1082     my $usbbus = $q35 ? "ehci" : "uhci";
1083
1084     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1085 }
1086
1087 sub print_drivedevice_full {
1088     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1089
1090     my $device = '';
1091     my $maxdev = 0;
1092
1093     if ($drive->{interface} eq 'virtio') {
1094         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1095         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1096         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1097     } elsif ($drive->{interface} eq 'scsi') {
1098
1099         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1100         my $unit = $drive->{index} % $maxdev;
1101         my $devicetype = 'hd';
1102         my $path = '';
1103         if (drive_is_cdrom($drive)) {
1104               $devicetype = 'cd';
1105         } else {
1106               if ($drive->{file} =~ m|^/|) {
1107                   $path = $drive->{file};
1108               } else {
1109                   $path = PVE::Storage::path($storecfg, $drive->{file});
1110               }
1111
1112               if($path =~ m/^iscsi\:\/\//){
1113                   $devicetype = 'generic';
1114               } else {
1115                   if (my $info = path_is_scsi($path)) {
1116                       if ($info->{type} == 0) {
1117                           $devicetype = 'block';
1118                       } elsif ($info->{type} == 1) { # tape
1119                           $devicetype = 'generic';
1120                       }
1121                   }
1122               }
1123          }
1124
1125         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1126             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1127         } else {
1128             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         }
1130
1131     } elsif ($drive->{interface} eq 'ide'){
1132         $maxdev = 2;
1133         my $controller = int($drive->{index} / $maxdev);
1134         my $unit = $drive->{index} % $maxdev;
1135         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1136
1137         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1138         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1139             $device .= ",model=$model";
1140         }
1141     } elsif ($drive->{interface} eq 'sata'){
1142         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1143         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1144         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1145     } elsif ($drive->{interface} eq 'usb') {
1146         die "implement me";
1147         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1148     } else {
1149         die "unsupported interface type";
1150     }
1151
1152     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1153
1154     return $device;
1155 }
1156
1157 sub get_initiator_name {
1158     my $initiator;
1159
1160     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1161     while (defined(my $line = <$fh>)) {
1162         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1163         $initiator = $1;
1164         last;
1165     }
1166     $fh->close();
1167
1168     return $initiator;
1169 }
1170
1171 sub print_drive_full {
1172     my ($storecfg, $vmid, $drive) = @_;
1173
1174     my $path;
1175     my $volid = $drive->{file};
1176     my $format;
1177
1178     if (drive_is_cdrom($drive)) {
1179         $path = get_iso_path($storecfg, $vmid, $volid);
1180     } else {
1181         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1182         if ($storeid) {
1183             $path = PVE::Storage::path($storecfg, $volid);
1184             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1185             $format = qemu_img_format($scfg, $volname);
1186         } else {
1187             $path = $volid;
1188         }
1189    }
1190
1191     my $opts = '';
1192     foreach my $o (@qemu_drive_options) {
1193         next if $o eq 'bootindex';
1194         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1195     }
1196
1197     $opts .= ",format=$format" if $format && !$drive->{format};
1198
1199     foreach my $o (qw(bps bps_rd bps_wr)) {
1200         my $v = $drive->{"m$o"};
1201         $opts .= ",$o=" . int($v*1024*1024) if $v;
1202     }
1203
1204     my $cache_direct = 0;
1205
1206     if (my $cache = $drive->{cache}) {
1207         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1208     } elsif (!drive_is_cdrom($drive)) {
1209         $opts .= ",cache=none";
1210         $cache_direct = 1;
1211     }
1212
1213     # aio native works only with O_DIRECT
1214     if (!$drive->{aio}) {
1215         if($cache_direct) {
1216             $opts .= ",aio=native";
1217         } else {
1218             $opts .= ",aio=threads";
1219         }
1220     }
1221
1222     if (!drive_is_cdrom($drive)) {
1223         my $detectzeroes;
1224         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1225             $detectzeroes = 'off';
1226         } elsif ($drive->{discard}) {
1227             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1228         } else {
1229             # This used to be our default with discard not being specified:
1230             $detectzeroes = 'on';
1231         }
1232         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1233     }
1234
1235     my $pathinfo = $path ? "file=$path," : '';
1236
1237     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1238 }
1239
1240 sub print_netdevice_full {
1241     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1242
1243     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1244
1245     my $device = $net->{model};
1246     if ($net->{model} eq 'virtio') {
1247          $device = 'virtio-net-pci';
1248      };
1249
1250     my $pciaddr = print_pci_addr("$netid", $bridges);
1251     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1252     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1253         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1254         my $vectors = $net->{queues} * 2 + 2;
1255         $tmpstr .= ",vectors=$vectors,mq=on";
1256     }
1257     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1258
1259     if ($use_old_bios_files) {
1260         my $romfile;
1261         if ($device eq 'virtio-net-pci') {
1262             $romfile = 'pxe-virtio.rom';
1263         } elsif ($device eq 'e1000') {
1264             $romfile = 'pxe-e1000.rom';
1265         } elsif ($device eq 'ne2k') {
1266             $romfile = 'pxe-ne2k_pci.rom';
1267         } elsif ($device eq 'pcnet') {
1268             $romfile = 'pxe-pcnet.rom';
1269         } elsif ($device eq 'rtl8139') {
1270             $romfile = 'pxe-rtl8139.rom';
1271         }
1272         $tmpstr .= ",romfile=$romfile" if $romfile;
1273     }
1274
1275     return $tmpstr;
1276 }
1277
1278 sub print_netdev_full {
1279     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1280
1281     my $i = '';
1282     if ($netid =~ m/^net(\d+)$/) {
1283         $i = int($1);
1284     }
1285
1286     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1287
1288     my $ifname = "tap${vmid}i$i";
1289
1290     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1291     die "interface name '$ifname' is too long (max 15 character)\n"
1292         if length($ifname) >= 16;
1293
1294     my $vhostparam = '';
1295     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1296
1297     my $vmname = $conf->{name} || "vm$vmid";
1298
1299     my $netdev = "";
1300     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1301
1302     if ($net->{bridge}) {
1303         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1304     } else {
1305         $netdev = "type=user,id=$netid,hostname=$vmname";
1306     }
1307
1308     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1309
1310     return $netdev;
1311 }
1312
1313 sub drive_is_cdrom {
1314     my ($drive) = @_;
1315
1316     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1317
1318 }
1319
1320 sub parse_numa {
1321     my ($data) = @_;
1322
1323     my $res = {};
1324
1325     foreach my $kvp (split(/,/, $data)) {
1326
1327         if ($kvp =~ m/^memory=(\S+)$/) {
1328             $res->{memory} = $1;
1329         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1330             $res->{policy} = $1;
1331         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1332             $res->{cpus}->{start} = $1;
1333             $res->{cpus}->{end} = $3;
1334         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1335             $res->{hostnodes}->{start} = $1;
1336             $res->{hostnodes}->{end} = $3;
1337         } else {
1338             return undef;
1339         }
1340     }
1341
1342     return $res;
1343 }
1344
1345 sub parse_hostpci {
1346     my ($value) = @_;
1347
1348     return undef if !$value;
1349
1350
1351     my @list = split(/,/, $value);
1352     my $found;
1353
1354     my $res = {};
1355     foreach my $kv (@list) {
1356
1357         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1358             $found = 1;
1359             if(defined($4)){
1360                 push @{$res->{pciid}}, { id => $2 , function => $4};
1361
1362             }else{
1363                 my $pcidevices = lspci($2);
1364                 $res->{pciid} = $pcidevices->{$2};
1365             }
1366         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1367             $res->{rombar} = $1;
1368         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1369             $res->{'x-vga'} = $1;
1370         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1371             $res->{pcie} = 1 if $1 == 1;
1372         } else {
1373             warn "unknown hostpci setting '$kv'\n";
1374         }
1375     }
1376
1377     return undef if !$found;
1378
1379     return $res;
1380 }
1381
1382 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1383 sub parse_net {
1384     my ($data) = @_;
1385
1386     my $res = {};
1387
1388     foreach my $kvp (split(/,/, $data)) {
1389
1390         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1391             my $model = lc($1);
1392             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1393             $res->{model} = $model;
1394             $res->{macaddr} = $mac;
1395         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1396             $res->{bridge} = $1;
1397         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1398             $res->{queues} = $1;
1399         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1400             $res->{rate} = $1;
1401         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1402             $res->{tag} = $1;
1403         } elsif ($kvp =~ m/^firewall=([01])$/) {
1404             $res->{firewall} = $1;
1405         } elsif ($kvp =~ m/^link_down=([01])$/) {
1406             $res->{link_down} = $1;
1407         } else {
1408             return undef;
1409         }
1410
1411     }
1412
1413     return undef if !$res->{model};
1414
1415     return $res;
1416 }
1417
1418 sub print_net {
1419     my $net = shift;
1420
1421     my $res = "$net->{model}";
1422     $res .= "=$net->{macaddr}" if $net->{macaddr};
1423     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1424     $res .= ",rate=$net->{rate}" if $net->{rate};
1425     $res .= ",tag=$net->{tag}" if $net->{tag};
1426     $res .= ",firewall=1" if $net->{firewall};
1427     $res .= ",link_down=1" if $net->{link_down};
1428     $res .= ",queues=$net->{queues}" if $net->{queues};
1429
1430     return $res;
1431 }
1432
1433 sub add_random_macs {
1434     my ($settings) = @_;
1435
1436     foreach my $opt (keys %$settings) {
1437         next if $opt !~ m/^net(\d+)$/;
1438         my $net = parse_net($settings->{$opt});
1439         next if !$net;
1440         $settings->{$opt} = print_net($net);
1441     }
1442 }
1443
1444 sub add_unused_volume {
1445     my ($config, $volid) = @_;
1446
1447     my $key;
1448     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1449         my $test = "unused$ind";
1450         if (my $vid = $config->{$test}) {
1451             return if $vid eq $volid; # do not add duplicates
1452         } else {
1453             $key = $test;
1454         }
1455     }
1456
1457     die "To many unused volume - please delete them first.\n" if !$key;
1458
1459     $config->{$key} = $volid;
1460
1461     return $key;
1462 }
1463
1464 sub vm_is_volid_owner {
1465     my ($storecfg, $vmid, $volid) = @_;
1466
1467     if ($volid !~  m|^/|) {
1468         my ($path, $owner);
1469         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1470         if ($owner && ($owner == $vmid)) {
1471             return 1;
1472         }
1473     }
1474
1475     return undef;
1476 }
1477
1478 sub split_flagged_list {
1479     my $text = shift || '';
1480     $text =~ s/[,;]/ /g;
1481     $text =~ s/^\s+//;
1482     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1483 }
1484
1485 sub join_flagged_list {
1486     my ($how, $lst) = @_;
1487     join $how, map { $lst->{$_} . $_ } keys %$lst;
1488 }
1489
1490 sub vmconfig_delete_pending_option {
1491     my ($conf, $key, $force) = @_;
1492
1493     delete $conf->{pending}->{$key};
1494     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1495     $pending_delete_hash->{$key} = $force ? '!' : '';
1496     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1497 }
1498
1499 sub vmconfig_undelete_pending_option {
1500     my ($conf, $key) = @_;
1501
1502     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1503     delete $pending_delete_hash->{$key};
1504
1505     if (%$pending_delete_hash) {
1506         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1507     } else {
1508         delete $conf->{pending}->{delete};
1509     }
1510 }
1511
1512 sub vmconfig_register_unused_drive {
1513     my ($storecfg, $vmid, $conf, $drive) = @_;
1514
1515     if (!drive_is_cdrom($drive)) {
1516         my $volid = $drive->{file};
1517         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1518             add_unused_volume($conf, $volid, $vmid);
1519         }
1520     }
1521 }
1522
1523 sub vmconfig_cleanup_pending {
1524     my ($conf) = @_;
1525
1526     # remove pending changes when nothing changed
1527     my $changes;
1528     foreach my $opt (keys %{$conf->{pending}}) {
1529         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1530             $changes = 1;
1531             delete $conf->{pending}->{$opt};
1532         }
1533     }
1534
1535     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1536     my $pending_delete_hash = {};
1537     while (my ($opt, $force) = each %$current_delete_hash) {
1538         if (defined($conf->{$opt})) {
1539             $pending_delete_hash->{$opt} = $force;
1540         } else {
1541             $changes = 1;
1542         }
1543     }
1544
1545     if (%$pending_delete_hash) {
1546         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1547     } else {
1548         delete $conf->{pending}->{delete};
1549     }
1550
1551     return $changes;
1552 }
1553
1554 my $valid_smbios1_options = {
1555     manufacturer => '\S+',
1556     product => '\S+',
1557     version => '\S+',
1558     serial => '\S+',
1559     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1560     sku => '\S+',
1561     family => '\S+',
1562 };
1563
1564 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1565 sub parse_smbios1 {
1566     my ($data) = @_;
1567
1568     my $res = {};
1569
1570     foreach my $kvp (split(/,/, $data)) {
1571         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1572         my ($k, $v) = split(/=/, $kvp);
1573         return undef if !defined($k) || !defined($v);
1574         return undef if !$valid_smbios1_options->{$k};
1575         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1576         $res->{$k} = $v;
1577     }
1578
1579     return $res;
1580 }
1581
1582 sub print_smbios1 {
1583     my ($smbios1) = @_;
1584
1585     my $data = '';
1586     foreach my $k (keys %$smbios1) {
1587         next if !defined($smbios1->{$k});
1588         next if !$valid_smbios1_options->{$k};
1589         $data .= ',' if $data;
1590         $data .= "$k=$smbios1->{$k}";
1591     }
1592     return $data;
1593 }
1594
1595 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1596 sub verify_smbios1 {
1597     my ($value, $noerr) = @_;
1598
1599     return $value if parse_smbios1($value);
1600
1601     return undef if $noerr;
1602
1603     die "unable to parse smbios (type 1) options\n";
1604 }
1605
1606 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1607 sub verify_bootdisk {
1608     my ($value, $noerr) = @_;
1609
1610     return $value if valid_drivename($value);
1611
1612     return undef if $noerr;
1613
1614     die "invalid boot disk '$value'\n";
1615 }
1616
1617 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1618 sub verify_numa {
1619     my ($value, $noerr) = @_;
1620
1621     return $value if parse_numa($value);
1622
1623     return undef if $noerr;
1624
1625     die "unable to parse numa options\n";
1626 }
1627
1628 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1629 sub verify_net {
1630     my ($value, $noerr) = @_;
1631
1632     return $value if parse_net($value);
1633
1634     return undef if $noerr;
1635
1636     die "unable to parse network options\n";
1637 }
1638
1639 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1640 sub verify_drive {
1641     my ($value, $noerr) = @_;
1642
1643     return $value if parse_drive(undef, $value);
1644
1645     return undef if $noerr;
1646
1647     die "unable to parse drive options\n";
1648 }
1649
1650 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1651 sub verify_hostpci {
1652     my ($value, $noerr) = @_;
1653
1654     return $value if parse_hostpci($value);
1655
1656     return undef if $noerr;
1657
1658     die "unable to parse pci id\n";
1659 }
1660
1661 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1662 sub verify_watchdog {
1663     my ($value, $noerr) = @_;
1664
1665     return $value if parse_watchdog($value);
1666
1667     return undef if $noerr;
1668
1669     die "unable to parse watchdog options\n";
1670 }
1671
1672 sub parse_watchdog {
1673     my ($value) = @_;
1674
1675     return undef if !$value;
1676
1677     my $res = {};
1678
1679     foreach my $p (split(/,/, $value)) {
1680         next if $p =~ m/^\s*$/;
1681
1682         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1683             $res->{model} = $2;
1684         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1685             $res->{action} = $2;
1686         } else {
1687             return undef;
1688         }
1689     }
1690
1691     return $res;
1692 }
1693
1694 sub parse_usb_device {
1695     my ($value) = @_;
1696
1697     return undef if !$value;
1698
1699     my @dl = split(/,/, $value);
1700     my $found;
1701
1702     my $res = {};
1703     foreach my $v (@dl) {
1704         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1705             $found = 1;
1706             $res->{vendorid} = $2;
1707             $res->{productid} = $4;
1708         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1709             $found = 1;
1710             $res->{hostbus} = $1;
1711             $res->{hostport} = $2;
1712         } elsif ($v =~ m/^spice$/) {
1713             $found = 1;
1714             $res->{spice} = 1;
1715         } else {
1716             return undef;
1717         }
1718     }
1719     return undef if !$found;
1720
1721     return $res;
1722 }
1723
1724 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1725 sub verify_usb_device {
1726     my ($value, $noerr) = @_;
1727
1728     return $value if parse_usb_device($value);
1729
1730     return undef if $noerr;
1731
1732     die "unable to parse usb device\n";
1733 }
1734
1735 # add JSON properties for create and set function
1736 sub json_config_properties {
1737     my $prop = shift;
1738
1739     foreach my $opt (keys %$confdesc) {
1740         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1741         $prop->{$opt} = $confdesc->{$opt};
1742     }
1743
1744     return $prop;
1745 }
1746
1747 sub check_type {
1748     my ($key, $value) = @_;
1749
1750     die "unknown setting '$key'\n" if !$confdesc->{$key};
1751
1752     my $type = $confdesc->{$key}->{type};
1753
1754     if (!defined($value)) {
1755         die "got undefined value\n";
1756     }
1757
1758     if ($value =~ m/[\n\r]/) {
1759         die "property contains a line feed\n";
1760     }
1761
1762     if ($type eq 'boolean') {
1763         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1764         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1765         die "type check ('boolean') failed - got '$value'\n";
1766     } elsif ($type eq 'integer') {
1767         return int($1) if $value =~ m/^(\d+)$/;
1768         die "type check ('integer') failed - got '$value'\n";
1769     } elsif ($type eq 'number') {
1770         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1771         die "type check ('number') failed - got '$value'\n";
1772     } elsif ($type eq 'string') {
1773         if (my $fmt = $confdesc->{$key}->{format}) {
1774             if ($fmt eq 'pve-qm-drive') {
1775                 # special case - we need to pass $key to parse_drive()
1776                 my $drive = parse_drive($key, $value);
1777                 return $value if $drive;
1778                 die "unable to parse drive options\n";
1779             }
1780             PVE::JSONSchema::check_format($fmt, $value);
1781             return $value;
1782         }
1783         $value =~ s/^\"(.*)\"$/$1/;
1784         return $value;
1785     } else {
1786         die "internal error"
1787     }
1788 }
1789
1790 sub lock_config_full {
1791     my ($vmid, $timeout, $code, @param) = @_;
1792
1793     my $filename = config_file_lock($vmid);
1794
1795     my $res = lock_file($filename, $timeout, $code, @param);
1796
1797     die $@ if $@;
1798
1799     return $res;
1800 }
1801
1802 sub lock_config_mode {
1803     my ($vmid, $timeout, $shared, $code, @param) = @_;
1804
1805     my $filename = config_file_lock($vmid);
1806
1807     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1808
1809     die $@ if $@;
1810
1811     return $res;
1812 }
1813
1814 sub lock_config {
1815     my ($vmid, $code, @param) = @_;
1816
1817     return lock_config_full($vmid, 10, $code, @param);
1818 }
1819
1820 sub cfs_config_path {
1821     my ($vmid, $node) = @_;
1822
1823     $node = $nodename if !$node;
1824     return "nodes/$node/qemu-server/$vmid.conf";
1825 }
1826
1827 sub check_iommu_support{
1828     #fixme : need to check IOMMU support
1829     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1830
1831     my $iommu=1;
1832     return $iommu;
1833
1834 }
1835
1836 sub config_file {
1837     my ($vmid, $node) = @_;
1838
1839     my $cfspath = cfs_config_path($vmid, $node);
1840     return "/etc/pve/$cfspath";
1841 }
1842
1843 sub config_file_lock {
1844     my ($vmid) = @_;
1845
1846     return "$lock_dir/lock-$vmid.conf";
1847 }
1848
1849 sub touch_config {
1850     my ($vmid) = @_;
1851
1852     my $conf = config_file($vmid);
1853     utime undef, undef, $conf;
1854 }
1855
1856 sub destroy_vm {
1857     my ($storecfg, $vmid, $keep_empty_config) = @_;
1858
1859     my $conffile = config_file($vmid);
1860
1861     my $conf = load_config($vmid);
1862
1863     check_lock($conf);
1864
1865     # only remove disks owned by this VM
1866     foreach_drive($conf, sub {
1867         my ($ds, $drive) = @_;
1868
1869         return if drive_is_cdrom($drive);
1870
1871         my $volid = $drive->{file};
1872
1873         return if !$volid || $volid =~ m|^/|;
1874
1875         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1876         return if !$path || !$owner || ($owner != $vmid);
1877
1878         PVE::Storage::vdisk_free($storecfg, $volid);
1879     });
1880
1881     if ($keep_empty_config) {
1882         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1883     } else {
1884         unlink $conffile;
1885     }
1886
1887     # also remove unused disk
1888     eval {
1889         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1890
1891         eval {
1892             PVE::Storage::foreach_volid($dl, sub {
1893                 my ($volid, $sid, $volname, $d) = @_;
1894                 PVE::Storage::vdisk_free($storecfg, $volid);
1895             });
1896         };
1897         warn $@ if $@;
1898
1899     };
1900     warn $@ if $@;
1901 }
1902
1903 sub load_config {
1904     my ($vmid, $node) = @_;
1905
1906     my $cfspath = cfs_config_path($vmid, $node);
1907
1908     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1909
1910     die "no such VM ('$vmid')\n" if !defined($conf);
1911
1912     return $conf;
1913 }
1914
1915 sub parse_vm_config {
1916     my ($filename, $raw) = @_;
1917
1918     return undef if !defined($raw);
1919
1920     my $res = {
1921         digest => Digest::SHA::sha1_hex($raw),
1922         snapshots => {},
1923         pending => {},
1924     };
1925
1926     $filename =~ m|/qemu-server/(\d+)\.conf$|
1927         || die "got strange filename '$filename'";
1928
1929     my $vmid = $1;
1930
1931     my $conf = $res;
1932     my $descr;
1933     my $section = '';
1934
1935     my @lines = split(/\n/, $raw);
1936     foreach my $line (@lines) {
1937         next if $line =~ m/^\s*$/;
1938
1939         if ($line =~ m/^\[PENDING\]\s*$/i) {
1940             $section = 'pending';
1941             if (defined($descr)) {
1942                 $descr =~ s/\s+$//;
1943                 $conf->{description} = $descr;
1944             }
1945             $descr = undef;
1946             $conf = $res->{$section} = {};
1947             next;
1948
1949         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1950             $section = $1;
1951             if (defined($descr)) {
1952                 $descr =~ s/\s+$//;
1953                 $conf->{description} = $descr;
1954             }
1955             $descr = undef;
1956             $conf = $res->{snapshots}->{$section} = {};
1957             next;
1958         }
1959
1960         if ($line =~ m/^\#(.*)\s*$/) {
1961             $descr = '' if !defined($descr);
1962             $descr .= PVE::Tools::decode_text($1) . "\n";
1963             next;
1964         }
1965
1966         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1967             $descr = '' if !defined($descr);
1968             $descr .= PVE::Tools::decode_text($2);
1969         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1970             $conf->{snapstate} = $1;
1971         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1972             my $key = $1;
1973             my $value = $2;
1974             $conf->{$key} = $value;
1975         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1976             my $value = $1;
1977             if ($section eq 'pending') {
1978                 $conf->{delete} = $value; # we parse this later
1979             } else {
1980                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1981             }
1982         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1983             my $key = $1;
1984             my $value = $2;
1985             eval { $value = check_type($key, $value); };
1986             if ($@) {
1987                 warn "vm $vmid - unable to parse value of '$key' - $@";
1988             } else {
1989                 my $fmt = $confdesc->{$key}->{format};
1990                 if ($fmt && $fmt eq 'pve-qm-drive') {
1991                     my $v = parse_drive($key, $value);
1992                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1993                         $v->{file} = $volid;
1994                         $value = print_drive($vmid, $v);
1995                     } else {
1996                         warn "vm $vmid - unable to parse value of '$key'\n";
1997                         next;
1998                     }
1999                 }
2000
2001                 if ($key eq 'cdrom') {
2002                     $conf->{ide2} = $value;
2003                 } else {
2004                     $conf->{$key} = $value;
2005                 }
2006             }
2007         }
2008     }
2009
2010     if (defined($descr)) {
2011         $descr =~ s/\s+$//;
2012         $conf->{description} = $descr;
2013     }
2014     delete $res->{snapstate}; # just to be sure
2015
2016     return $res;
2017 }
2018
2019 sub write_vm_config {
2020     my ($filename, $conf) = @_;
2021
2022     delete $conf->{snapstate}; # just to be sure
2023
2024     if ($conf->{cdrom}) {
2025         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2026         $conf->{ide2} = $conf->{cdrom};
2027         delete $conf->{cdrom};
2028     }
2029
2030     # we do not use 'smp' any longer
2031     if ($conf->{sockets}) {
2032         delete $conf->{smp};
2033     } elsif ($conf->{smp}) {
2034         $conf->{sockets} = $conf->{smp};
2035         delete $conf->{cores};
2036         delete $conf->{smp};
2037     }
2038
2039     my $used_volids = {};
2040
2041     my $cleanup_config = sub {
2042         my ($cref, $pending, $snapname) = @_;
2043
2044         foreach my $key (keys %$cref) {
2045             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2046                 $key eq 'snapstate' || $key eq 'pending';
2047             my $value = $cref->{$key};
2048             if ($key eq 'delete') {
2049                 die "propertry 'delete' is only allowed in [PENDING]\n"
2050                     if !$pending;
2051                 # fixme: check syntax?
2052                 next;
2053             }
2054             eval { $value = check_type($key, $value); };
2055             die "unable to parse value of '$key' - $@" if $@;
2056
2057             $cref->{$key} = $value;
2058
2059             if (!$snapname && valid_drivename($key)) {
2060                 my $drive = parse_drive($key, $value);
2061                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2062             }
2063         }
2064     };
2065
2066     &$cleanup_config($conf);
2067
2068     &$cleanup_config($conf->{pending}, 1);
2069
2070     foreach my $snapname (keys %{$conf->{snapshots}}) {
2071         die "internal error" if $snapname eq 'pending';
2072         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2073     }
2074
2075     # remove 'unusedX' settings if we re-add a volume
2076     foreach my $key (keys %$conf) {
2077         my $value = $conf->{$key};
2078         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2079             delete $conf->{$key};
2080         }
2081     }
2082
2083     my $generate_raw_config = sub {
2084         my ($conf, $pending) = @_;
2085
2086         my $raw = '';
2087
2088         # add description as comment to top of file
2089         if (defined(my $descr = $conf->{description})) {
2090             if ($descr) {
2091                 foreach my $cl (split(/\n/, $descr)) {
2092                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2093                 }
2094             } else {
2095                 $raw .= "#\n" if $pending;
2096             }
2097         }
2098
2099         foreach my $key (sort keys %$conf) {
2100             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2101             $raw .= "$key: $conf->{$key}\n";
2102         }
2103         return $raw;
2104     };
2105
2106     my $raw = &$generate_raw_config($conf);
2107
2108     if (scalar(keys %{$conf->{pending}})){
2109         $raw .= "\n[PENDING]\n";
2110         $raw .= &$generate_raw_config($conf->{pending}, 1);
2111     }
2112
2113     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2114         $raw .= "\n[$snapname]\n";
2115         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2116     }
2117
2118     return $raw;
2119 }
2120
2121 sub update_config_nolock {
2122     my ($vmid, $conf, $skiplock) = @_;
2123
2124     check_lock($conf) if !$skiplock;
2125
2126     my $cfspath = cfs_config_path($vmid);
2127
2128     PVE::Cluster::cfs_write_file($cfspath, $conf);
2129 }
2130
2131 sub update_config {
2132     my ($vmid, $conf, $skiplock) = @_;
2133
2134     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2135 }
2136
2137 sub load_defaults {
2138
2139     my $res = {};
2140
2141     # we use static defaults from our JSON schema configuration
2142     foreach my $key (keys %$confdesc) {
2143         if (defined(my $default = $confdesc->{$key}->{default})) {
2144             $res->{$key} = $default;
2145         }
2146     }
2147
2148     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2149     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2150
2151     return $res;
2152 }
2153
2154 sub config_list {
2155     my $vmlist = PVE::Cluster::get_vmlist();
2156     my $res = {};
2157     return $res if !$vmlist || !$vmlist->{ids};
2158     my $ids = $vmlist->{ids};
2159
2160     foreach my $vmid (keys %$ids) {
2161         my $d = $ids->{$vmid};
2162         next if !$d->{node} || $d->{node} ne $nodename;
2163         next if !$d->{type} || $d->{type} ne 'qemu';
2164         $res->{$vmid}->{exists} = 1;
2165     }
2166     return $res;
2167 }
2168
2169 # test if VM uses local resources (to prevent migration)
2170 sub check_local_resources {
2171     my ($conf, $noerr) = @_;
2172
2173     my $loc_res = 0;
2174
2175     $loc_res = 1 if $conf->{hostusb}; # old syntax
2176     $loc_res = 1 if $conf->{hostpci}; # old syntax
2177
2178     foreach my $k (keys %$conf) {
2179         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2180         # sockets are safe: they will recreated be on the target side post-migrate
2181         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2182         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2183     }
2184
2185     die "VM uses local resources\n" if $loc_res && !$noerr;
2186
2187     return $loc_res;
2188 }
2189
2190 # check if used storages are available on all nodes (use by migrate)
2191 sub check_storage_availability {
2192     my ($storecfg, $conf, $node) = @_;
2193
2194     foreach_drive($conf, sub {
2195         my ($ds, $drive) = @_;
2196
2197         my $volid = $drive->{file};
2198         return if !$volid;
2199
2200         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2201         return if !$sid;
2202
2203         # check if storage is available on both nodes
2204         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2205         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2206    });
2207 }
2208
2209 # list nodes where all VM images are available (used by has_feature API)
2210 sub shared_nodes {
2211     my ($conf, $storecfg) = @_;
2212
2213     my $nodelist = PVE::Cluster::get_nodelist();
2214     my $nodehash = { map { $_ => 1 } @$nodelist };
2215     my $nodename = PVE::INotify::nodename();
2216
2217     foreach_drive($conf, sub {
2218         my ($ds, $drive) = @_;
2219
2220         my $volid = $drive->{file};
2221         return if !$volid;
2222
2223         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2224         if ($storeid) {
2225             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2226             if ($scfg->{disable}) {
2227                 $nodehash = {};
2228             } elsif (my $avail = $scfg->{nodes}) {
2229                 foreach my $node (keys %$nodehash) {
2230                     delete $nodehash->{$node} if !$avail->{$node};
2231                 }
2232             } elsif (!$scfg->{shared}) {
2233                 foreach my $node (keys %$nodehash) {
2234                     delete $nodehash->{$node} if $node ne $nodename
2235                 }
2236             }
2237         }
2238     });
2239
2240     return $nodehash
2241 }
2242
2243 sub check_lock {
2244     my ($conf) = @_;
2245
2246     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2247 }
2248
2249 sub check_cmdline {
2250     my ($pidfile, $pid) = @_;
2251
2252     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2253     if (defined($fh)) {
2254         my $line = <$fh>;
2255         $fh->close;
2256         return undef if !$line;
2257         my @param = split(/\0/, $line);
2258
2259         my $cmd = $param[0];
2260         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2261
2262         for (my $i = 0; $i < scalar (@param); $i++) {
2263             my $p = $param[$i];
2264             next if !$p;
2265             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2266                 my $p = $param[$i+1];
2267                 return 1 if $p && ($p eq $pidfile);
2268                 return undef;
2269             }
2270         }
2271     }
2272     return undef;
2273 }
2274
2275 sub check_running {
2276     my ($vmid, $nocheck, $node) = @_;
2277
2278     my $filename = config_file($vmid, $node);
2279
2280     die "unable to find configuration file for VM $vmid - no such machine\n"
2281         if !$nocheck && ! -f $filename;
2282
2283     my $pidfile = pidfile_name($vmid);
2284
2285     if (my $fd = IO::File->new("<$pidfile")) {
2286         my $st = stat($fd);
2287         my $line = <$fd>;
2288         close($fd);
2289
2290         my $mtime = $st->mtime;
2291         if ($mtime > time()) {
2292             warn "file '$filename' modified in future\n";
2293         }
2294
2295         if ($line =~ m/^(\d+)$/) {
2296             my $pid = $1;
2297             if (check_cmdline($pidfile, $pid)) {
2298                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2299                     return $pid;
2300                 }
2301             }
2302         }
2303     }
2304
2305     return undef;
2306 }
2307
2308 sub vzlist {
2309
2310     my $vzlist = config_list();
2311
2312     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2313
2314     while (defined(my $de = $fd->read)) {
2315         next if $de !~ m/^(\d+)\.pid$/;
2316         my $vmid = $1;
2317         next if !defined($vzlist->{$vmid});
2318         if (my $pid = check_running($vmid)) {
2319             $vzlist->{$vmid}->{pid} = $pid;
2320         }
2321     }
2322
2323     return $vzlist;
2324 }
2325
2326 sub disksize {
2327     my ($storecfg, $conf) = @_;
2328
2329     my $bootdisk = $conf->{bootdisk};
2330     return undef if !$bootdisk;
2331     return undef if !valid_drivename($bootdisk);
2332
2333     return undef if !$conf->{$bootdisk};
2334
2335     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2336     return undef if !defined($drive);
2337
2338     return undef if drive_is_cdrom($drive);
2339
2340     my $volid = $drive->{file};
2341     return undef if !$volid;
2342
2343     return $drive->{size};
2344 }
2345
2346 my $last_proc_pid_stat;
2347
2348 # get VM status information
2349 # This must be fast and should not block ($full == false)
2350 # We only query KVM using QMP if $full == true (this can be slow)
2351 sub vmstatus {
2352     my ($opt_vmid, $full) = @_;
2353
2354     my $res = {};
2355
2356     my $storecfg = PVE::Storage::config();
2357
2358     my $list = vzlist();
2359     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2360
2361     my $cpucount = $cpuinfo->{cpus} || 1;
2362
2363     foreach my $vmid (keys %$list) {
2364         next if $opt_vmid && ($vmid ne $opt_vmid);
2365
2366         my $cfspath = cfs_config_path($vmid);
2367         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2368
2369         my $d = {};
2370         $d->{pid} = $list->{$vmid}->{pid};
2371
2372         # fixme: better status?
2373         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2374
2375         my $size = disksize($storecfg, $conf);
2376         if (defined($size)) {
2377             $d->{disk} = 0; # no info available
2378             $d->{maxdisk} = $size;
2379         } else {
2380             $d->{disk} = 0;
2381             $d->{maxdisk} = 0;
2382         }
2383
2384         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2385         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2386         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2387
2388         $d->{name} = $conf->{name} || "VM $vmid";
2389         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2390
2391         if ($conf->{balloon}) {
2392             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2393             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2394         }
2395
2396         $d->{uptime} = 0;
2397         $d->{cpu} = 0;
2398         $d->{mem} = 0;
2399
2400         $d->{netout} = 0;
2401         $d->{netin} = 0;
2402
2403         $d->{diskread} = 0;
2404         $d->{diskwrite} = 0;
2405
2406         $d->{template} = is_template($conf);
2407
2408         $res->{$vmid} = $d;
2409     }
2410
2411     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2412     foreach my $dev (keys %$netdev) {
2413         next if $dev !~ m/^tap([1-9]\d*)i/;
2414         my $vmid = $1;
2415         my $d = $res->{$vmid};
2416         next if !$d;
2417
2418         $d->{netout} += $netdev->{$dev}->{receive};
2419         $d->{netin} += $netdev->{$dev}->{transmit};
2420
2421         if ($full) {
2422             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2423             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2424         }
2425
2426     }
2427
2428     my $ctime = gettimeofday;
2429
2430     foreach my $vmid (keys %$list) {
2431
2432         my $d = $res->{$vmid};
2433         my $pid = $d->{pid};
2434         next if !$pid;
2435
2436         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2437         next if !$pstat; # not running
2438
2439         my $used = $pstat->{utime} + $pstat->{stime};
2440
2441         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2442
2443         if ($pstat->{vsize}) {
2444             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2445         }
2446
2447         my $old = $last_proc_pid_stat->{$pid};
2448         if (!$old) {
2449             $last_proc_pid_stat->{$pid} = {
2450                 time => $ctime,
2451                 used => $used,
2452                 cpu => 0,
2453             };
2454             next;
2455         }
2456
2457         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2458
2459         if ($dtime > 1000) {
2460             my $dutime = $used -  $old->{used};
2461
2462             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2463             $last_proc_pid_stat->{$pid} = {
2464                 time => $ctime,
2465                 used => $used,
2466                 cpu => $d->{cpu},
2467             };
2468         } else {
2469             $d->{cpu} = $old->{cpu};
2470         }
2471     }
2472
2473     return $res if !$full;
2474
2475     my $qmpclient = PVE::QMPClient->new();
2476
2477     my $ballooncb = sub {
2478         my ($vmid, $resp) = @_;
2479
2480         my $info = $resp->{'return'};
2481         return if !$info->{max_mem};
2482
2483         my $d = $res->{$vmid};
2484
2485         # use memory assigned to VM
2486         $d->{maxmem} = $info->{max_mem};
2487         $d->{balloon} = $info->{actual};
2488
2489         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2490             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2491             $d->{freemem} = $info->{free_mem};
2492         }
2493
2494         $d->{ballooninfo} = $info;
2495     };
2496
2497     my $blockstatscb = sub {
2498         my ($vmid, $resp) = @_;
2499         my $data = $resp->{'return'} || [];
2500         my $totalrdbytes = 0;
2501         my $totalwrbytes = 0;
2502
2503         for my $blockstat (@$data) {
2504             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2505             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2506
2507             $blockstat->{device} =~ s/drive-//;
2508             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2509         }
2510         $res->{$vmid}->{diskread} = $totalrdbytes;
2511         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2512     };
2513
2514     my $statuscb = sub {
2515         my ($vmid, $resp) = @_;
2516
2517         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2518         # this fails if ballon driver is not loaded, so this must be
2519         # the last commnand (following command are aborted if this fails).
2520         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2521
2522         my $status = 'unknown';
2523         if (!defined($status = $resp->{'return'}->{status})) {
2524             warn "unable to get VM status\n";
2525             return;
2526         }
2527
2528         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2529     };
2530
2531     foreach my $vmid (keys %$list) {
2532         next if $opt_vmid && ($vmid ne $opt_vmid);
2533         next if !$res->{$vmid}->{pid}; # not running
2534         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2535     }
2536
2537     $qmpclient->queue_execute(undef, 1);
2538
2539     foreach my $vmid (keys %$list) {
2540         next if $opt_vmid && ($vmid ne $opt_vmid);
2541         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub foreach_dimm {
2548     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2549
2550     my $dimm_id = 0;
2551     my $current_size = 1024;
2552     my $dimm_size = 512;
2553     return if $current_size == $memory;
2554
2555     for (my $j = 0; $j < 8; $j++) {
2556         for (my $i = 0; $i < 32; $i++) {
2557             my $name = "dimm${dimm_id}";
2558             $dimm_id++;
2559             my $numanode = $i % $sockets;
2560             $current_size += $dimm_size;
2561             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2562             return  $current_size if $current_size >= $memory;
2563         }
2564         $dimm_size *= 2;
2565     }
2566 }
2567
2568 sub foreach_reverse_dimm {
2569     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2570
2571     my $dimm_id = 253;
2572     my $current_size = 4177920;
2573     my $dimm_size = 65536;
2574     return if $current_size == $memory;
2575
2576     for (my $j = 0; $j < 8; $j++) {
2577         for (my $i = 0; $i < 32; $i++) {
2578             my $name = "dimm${dimm_id}";
2579             $dimm_id--;
2580             my $numanode = $i % $sockets;
2581             $current_size -= $dimm_size;
2582             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2583             return  $current_size if $current_size <= $memory;
2584         }
2585         $dimm_size /= 2;
2586     }
2587 }
2588
2589 sub foreach_drive {
2590     my ($conf, $func) = @_;
2591
2592     foreach my $ds (keys %$conf) {
2593         next if !valid_drivename($ds);
2594
2595         my $drive = parse_drive($ds, $conf->{$ds});
2596         next if !$drive;
2597
2598         &$func($ds, $drive);
2599     }
2600 }
2601
2602 sub foreach_volid {
2603     my ($conf, $func) = @_;
2604
2605     my $volhash = {};
2606
2607     my $test_volid = sub {
2608         my ($volid, $is_cdrom) = @_;
2609
2610         return if !$volid;
2611
2612         $volhash->{$volid} = $is_cdrom || 0;
2613     };
2614
2615     foreach_drive($conf, sub {
2616         my ($ds, $drive) = @_;
2617         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2618     });
2619
2620     foreach my $snapname (keys %{$conf->{snapshots}}) {
2621         my $snap = $conf->{snapshots}->{$snapname};
2622         &$test_volid($snap->{vmstate}, 0);
2623         foreach_drive($snap, sub {
2624             my ($ds, $drive) = @_;
2625             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2626         });
2627     }
2628
2629     foreach my $volid (keys %$volhash) {
2630         &$func($volid, $volhash->{$volid});
2631     }
2632 }
2633
2634 sub vga_conf_has_spice {
2635     my ($vga) = @_;
2636
2637     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2638
2639     return $1 || 1;
2640 }
2641
2642 sub config_to_command {
2643     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2644
2645     my $cmd = [];
2646     my $globalFlags = [];
2647     my $machineFlags = [];
2648     my $rtcFlags = [];
2649     my $cpuFlags = [];
2650     my $devices = [];
2651     my $pciaddr = '';
2652     my $bridges = {};
2653     my $kvmver = kvm_user_version();
2654     my $vernum = 0; # unknown
2655     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2656         $vernum = $1*1000000+$2*1000;
2657     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2658         $vernum = $1*1000000+$2*1000+$3;
2659     }
2660
2661     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2662
2663     my $have_ovz = -f '/proc/vz/vestat';
2664
2665     my $q35 = machine_type_is_q35($conf);
2666     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2667     my $machine_type = $forcemachine || $conf->{machine};
2668     my $use_old_bios_files = undef;
2669     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2670
2671     my $cpuunits = defined($conf->{cpuunits}) ?
2672             $conf->{cpuunits} : $defaults->{cpuunits};
2673
2674     push @$cmd, '/usr/bin/systemd-run';
2675     push @$cmd, '--scope';
2676     push @$cmd, '--slice', "qemu";
2677     push @$cmd, '--unit', $vmid;
2678     # set KillMode=none, so that systemd don't kill those scopes
2679     # at shutdown (pve-manager service should stop the VMs instead)
2680     push @$cmd, '-p', "KillMode=none";
2681     push @$cmd, '-p', "CPUShares=$cpuunits";
2682     if ($conf->{cpulimit}) {
2683         my $cpulimit = int($conf->{cpulimit} * 100);
2684         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2685     }
2686
2687     push @$cmd, '/usr/bin/kvm';
2688
2689     push @$cmd, '-id', $vmid;
2690
2691     my $use_virtio = 0;
2692
2693     my $qmpsocket = qmp_socket($vmid);
2694     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2695     push @$cmd, '-mon', "chardev=qmp,mode=control";
2696
2697     my $socket = vnc_socket($vmid);
2698     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2699
2700     push @$cmd, '-pidfile' , pidfile_name($vmid);
2701
2702     push @$cmd, '-daemonize';
2703
2704     if ($conf->{smbios1}) {
2705         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2706     }
2707
2708     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2709         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2710         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2711         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2712         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2713         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2714         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2715     }
2716
2717     if ($q35) {
2718         # the q35 chipset support native usb2, so we enable usb controller
2719         # by default for this machine type
2720         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2721     } else {
2722         $pciaddr = print_pci_addr("piix3", $bridges);
2723         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2724
2725         my $use_usb2 = 0;
2726         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2727             next if !$conf->{"usb$i"};
2728             $use_usb2 = 1;
2729         }
2730         # include usb device config
2731         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2732     }
2733
2734     my $vga = $conf->{vga};
2735
2736     my $qxlnum = vga_conf_has_spice($vga);
2737     $vga = 'qxl' if $qxlnum;
2738
2739     if (!$vga) {
2740         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2741                                 $conf->{ostype} eq 'win7' ||
2742                                 $conf->{ostype} eq 'w2k8')) {
2743             $vga = 'std';
2744         } else {
2745             $vga = 'cirrus';
2746         }
2747     }
2748
2749     # enable absolute mouse coordinates (needed by vnc)
2750     my $tablet;
2751     if (defined($conf->{tablet})) {
2752         $tablet = $conf->{tablet};
2753     } else {
2754         $tablet = $defaults->{tablet};
2755         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2756         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2757     }
2758
2759     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2760
2761     # host pci devices
2762     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2763         my $d = parse_hostpci($conf->{"hostpci$i"});
2764         next if !$d;
2765
2766         my $pcie = $d->{pcie};
2767         if($pcie){
2768             die "q35 machine model is not enabled" if !$q35;
2769             $pciaddr = print_pcie_addr("hostpci$i");
2770         }else{
2771             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2772         }
2773
2774         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2775         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2776         if ($xvga && $xvga ne '') {
2777             push @$cpuFlags, 'kvm=off';
2778             $vga = 'none';
2779         }
2780         my $pcidevices = $d->{pciid};
2781         my $multifunction = 1 if @$pcidevices > 1;
2782
2783         my $j=0;
2784         foreach my $pcidevice (@$pcidevices) {
2785
2786             my $id = "hostpci$i";
2787             $id .= ".$j" if $multifunction;
2788             my $addr = $pciaddr;
2789             $addr .= ".$j" if $multifunction;
2790             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2791
2792             if($j == 0){
2793                 $devicestr .= "$rombar$xvga";
2794                 $devicestr .= ",multifunction=on" if $multifunction;
2795             }
2796
2797             push @$devices, '-device', $devicestr;
2798             $j++;
2799         }
2800     }
2801
2802     # usb devices
2803     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2804         my $d = parse_usb_device($conf->{"usb$i"});
2805         next if !$d;
2806         if ($d->{vendorid} && $d->{productid}) {
2807             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2808         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2809             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2810         } elsif ($d->{spice}) {
2811             # usb redir support for spice
2812             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2813             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2814         }
2815     }
2816
2817     # serial devices
2818     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2819         if (my $path = $conf->{"serial$i"}) {
2820             if ($path eq 'socket') {
2821                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2822                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2823                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2824             } else {
2825                 die "no such serial device\n" if ! -c $path;
2826                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2827                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2828             }
2829         }
2830     }
2831
2832     # parallel devices
2833     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2834         if (my $path = $conf->{"parallel$i"}) {
2835             die "no such parallel device\n" if ! -c $path;
2836             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2837             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2838             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2839         }
2840     }
2841
2842     my $vmname = $conf->{name} || "vm$vmid";
2843
2844     push @$cmd, '-name', $vmname;
2845
2846     my $sockets = 1;
2847     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2848     $sockets = $conf->{sockets} if  $conf->{sockets};
2849
2850     my $cores = $conf->{cores} || 1;
2851
2852     my $maxcpus = $sockets * $cores;
2853
2854     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2855
2856     my $allowed_vcpus = $cpuinfo->{cpus};
2857
2858     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2859         if ($allowed_vcpus < $maxcpus);
2860
2861     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2862
2863     push @$cmd, '-nodefaults';
2864
2865     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2866
2867     my $bootindex_hash = {};
2868     my $i = 1;
2869     foreach my $o (split(//, $bootorder)) {
2870         $bootindex_hash->{$o} = $i*100;
2871         $i++;
2872     }
2873
2874     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2875
2876     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2877
2878     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2879
2880     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2881
2882     # time drift fix
2883     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2884
2885     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2886     my $useLocaltime = $conf->{localtime};
2887
2888     if (my $ost = $conf->{ostype}) {
2889         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2890
2891         if ($ost =~ m/^w/) { # windows
2892             $useLocaltime = 1 if !defined($conf->{localtime});
2893
2894             # use time drift fix when acpi is enabled
2895             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2896                 $tdf = 1 if !defined($conf->{tdf});
2897             }
2898         }
2899
2900         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2901             $ost eq 'wvista') {
2902             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2903             push @$cmd, '-no-hpet';
2904             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2905                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2906                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2907                 push @$cpuFlags , 'hv_time' if !$nokvm;
2908
2909             } else {
2910                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2911             }
2912         }
2913
2914         if ($ost eq 'win7' || $ost eq 'win8') {
2915             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2916         }
2917     }
2918
2919     push @$rtcFlags, 'driftfix=slew' if $tdf;
2920
2921     if ($nokvm) {
2922         push @$machineFlags, 'accel=tcg';
2923     } else {
2924         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2925     }
2926
2927     if ($machine_type) {
2928         push @$machineFlags, "type=${machine_type}";
2929     }
2930
2931     if ($conf->{startdate}) {
2932         push @$rtcFlags, "base=$conf->{startdate}";
2933     } elsif ($useLocaltime) {
2934         push @$rtcFlags, 'base=localtime';
2935     }
2936
2937     my $cpu = $nokvm ? "qemu64" : "kvm64";
2938     $cpu = $conf->{cpu} if $conf->{cpu};
2939
2940     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2941
2942     push @$cpuFlags , '-x2apic'
2943         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2944
2945     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2946
2947     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2948
2949     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2950
2951         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2952         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2953     }
2954
2955     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2956
2957     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2958
2959     push @$cmd, '-cpu', $cpu;
2960
2961     my $memory = $conf->{memory} || $defaults->{memory};
2962     my $static_memory = 0;
2963     my $dimm_memory = 0;
2964
2965     if ($hotplug_features->{memory}) {
2966         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2967         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2968         $static_memory = $STATICMEM;
2969         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2970         $dimm_memory = $memory - $static_memory;
2971         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2972
2973     } else {
2974
2975         $static_memory = $memory;
2976         push @$cmd, '-m', $static_memory;
2977     }
2978
2979     if ($conf->{numa}) {
2980
2981         my $numa_totalmemory = undef;
2982         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2983             next if !$conf->{"numa$i"};
2984             my $numa = parse_numa($conf->{"numa$i"});
2985             next if !$numa;
2986             # memory
2987             die "missing numa node$i memory value\n" if !$numa->{memory};
2988             my $numa_memory = $numa->{memory};
2989             $numa_totalmemory += $numa_memory;
2990             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2991
2992             # cpus
2993             my $cpus_start = $numa->{cpus}->{start};
2994             die "missing numa node$i cpus\n" if !defined($cpus_start);
2995             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2996             my $cpus = $cpus_start;
2997             if (defined($cpus_end)) {
2998                 $cpus .= "-$cpus_end";
2999                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3000             }
3001
3002             # hostnodes
3003             my $hostnodes_start = $numa->{hostnodes}->{start};
3004             if (defined($hostnodes_start)) {
3005                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3006                 my $hostnodes = $hostnodes_start;
3007                 if (defined($hostnodes_end)) {
3008                     $hostnodes .= "-$hostnodes_end";
3009                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3010                 }
3011
3012                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3013                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3014                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3015                 }
3016
3017                 # policy
3018                 my $policy = $numa->{policy};
3019                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3020                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3021             }
3022
3023             push @$cmd, '-object', $numa_object;
3024             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3025         }
3026
3027         die "total memory for NUMA nodes must be equal to vm static memory\n"
3028             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3029
3030         #if no custom tology, we split memory and cores across numa nodes
3031         if(!$numa_totalmemory) {
3032
3033             my $numa_memory = ($static_memory / $sockets) . "M";
3034
3035             for (my $i = 0; $i < $sockets; $i++)  {
3036
3037                 my $cpustart = ($cores * $i);
3038                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3039                 my $cpus = $cpustart;
3040                 $cpus .= "-$cpuend" if $cpuend;
3041
3042                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3043                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3044             }
3045         }
3046     }
3047
3048     if ($hotplug_features->{memory}) {
3049         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3050             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3051             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3052             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3053
3054             #if dimm_memory is not aligned to dimm map
3055             if($current_size > $memory) {
3056                  $conf->{memory} = $current_size;
3057                  update_config_nolock($vmid, $conf, 1);
3058             }
3059         });
3060     }
3061
3062     push @$cmd, '-S' if $conf->{freeze};
3063
3064     # set keyboard layout
3065     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3066     push @$cmd, '-k', $kb if $kb;
3067
3068     # enable sound
3069     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3070     #push @$cmd, '-soundhw', 'es1370';
3071     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3072
3073     if($conf->{agent}) {
3074         my $qgasocket = qmp_socket($vmid, 1);
3075         my $pciaddr = print_pci_addr("qga0", $bridges);
3076         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3077         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3078         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3079     }
3080
3081     my $spice_port;
3082
3083     if ($qxlnum) {
3084         if ($qxlnum > 1) {
3085             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3086                 for(my $i = 1; $i < $qxlnum; $i++){
3087                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3088                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3089                 }
3090             } else {
3091                 # assume other OS works like Linux
3092                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3093                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3094             }
3095         }
3096
3097         my $pciaddr = print_pci_addr("spice", $bridges);
3098
3099         my $nodename = PVE::INotify::nodename();
3100         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3101         $spice_port = PVE::Tools::next_spice_port($pfamily);
3102
3103         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3104
3105         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3106         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3107         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3108     }
3109
3110     # enable balloon by default, unless explicitly disabled
3111     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3112         $pciaddr = print_pci_addr("balloon0", $bridges);
3113         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3114     }
3115
3116     if ($conf->{watchdog}) {
3117         my $wdopts = parse_watchdog($conf->{watchdog});
3118         $pciaddr = print_pci_addr("watchdog", $bridges);
3119         my $watchdog = $wdopts->{model} || 'i6300esb';
3120         push @$devices, '-device', "$watchdog$pciaddr";
3121         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3122     }
3123
3124     my $vollist = [];
3125     my $scsicontroller = {};
3126     my $ahcicontroller = {};
3127     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3128
3129     # Add iscsi initiator name if available
3130     if (my $initiator = get_initiator_name()) {
3131         push @$devices, '-iscsi', "initiator-name=$initiator";
3132     }
3133
3134     foreach_drive($conf, sub {
3135         my ($ds, $drive) = @_;
3136
3137         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3138             push @$vollist, $drive->{file};
3139         }
3140
3141         $use_virtio = 1 if $ds =~ m/^virtio/;
3142
3143         if (drive_is_cdrom ($drive)) {
3144             if ($bootindex_hash->{d}) {
3145                 $drive->{bootindex} = $bootindex_hash->{d};
3146                 $bootindex_hash->{d} += 1;
3147             }
3148         } else {
3149             if ($bootindex_hash->{c}) {
3150                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3151                 $bootindex_hash->{c} += 1;
3152             }
3153         }
3154
3155         if($drive->{interface} eq 'virtio'){
3156            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3157         }
3158
3159         if ($drive->{interface} eq 'scsi') {
3160
3161             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3162
3163             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3164             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3165
3166             my $iothread = '';
3167             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3168                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3169                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3170             }
3171
3172             my $queues = '';
3173             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3174                 $queues = ",num_queues=$drive->{queues}";
3175             }
3176
3177             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3178             $scsicontroller->{$controller}=1;
3179         }
3180
3181         if ($drive->{interface} eq 'sata') {
3182            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3183            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3184            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3185            $ahcicontroller->{$controller}=1;
3186         }
3187
3188         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3189         push @$devices, '-drive',$drive_cmd;
3190         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3191     });
3192
3193     for (my $i = 0; $i < $MAX_NETS; $i++) {
3194          next if !$conf->{"net$i"};
3195          my $d = parse_net($conf->{"net$i"});
3196          next if !$d;
3197
3198          $use_virtio = 1 if $d->{model} eq 'virtio';
3199
3200          if ($bootindex_hash->{n}) {
3201             $d->{bootindex} = $bootindex_hash->{n};
3202             $bootindex_hash->{n} += 1;
3203          }
3204
3205          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3206          push @$devices, '-netdev', $netdevfull;
3207
3208          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3209          push @$devices, '-device', $netdevicefull;
3210     }
3211
3212     if (!$q35) {
3213         # add pci bridges
3214         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3215            $bridges->{1} = 1;
3216            $bridges->{2} = 1;
3217         }
3218
3219         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3220
3221         while (my ($k, $v) = each %$bridges) {
3222             $pciaddr = print_pci_addr("pci.$k");
3223             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3224         }
3225     }
3226
3227     # add custom args
3228     if ($conf->{args}) {
3229         my $aa = PVE::Tools::split_args($conf->{args});
3230         push @$cmd, @$aa;
3231     }
3232
3233     push @$cmd, @$devices;
3234     push @$cmd, '-rtc', join(',', @$rtcFlags)
3235         if scalar(@$rtcFlags);
3236     push @$cmd, '-machine', join(',', @$machineFlags)
3237         if scalar(@$machineFlags);
3238     push @$cmd, '-global', join(',', @$globalFlags)
3239         if scalar(@$globalFlags);
3240
3241     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3242 }
3243
3244 sub vnc_socket {
3245     my ($vmid) = @_;
3246     return "${var_run_tmpdir}/$vmid.vnc";
3247 }
3248
3249 sub spice_port {
3250     my ($vmid) = @_;
3251
3252     my $res = vm_mon_cmd($vmid, 'query-spice');
3253
3254     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3255 }
3256
3257 sub qmp_socket {
3258     my ($vmid, $qga) = @_;
3259     my $sockettype = $qga ? 'qga' : 'qmp';
3260     return "${var_run_tmpdir}/$vmid.$sockettype";
3261 }
3262
3263 sub pidfile_name {
3264     my ($vmid) = @_;
3265     return "${var_run_tmpdir}/$vmid.pid";
3266 }
3267
3268 sub vm_devices_list {
3269     my ($vmid) = @_;
3270
3271     my $res = vm_mon_cmd($vmid, 'query-pci');
3272     my $devices = {};
3273     foreach my $pcibus (@$res) {
3274         foreach my $device (@{$pcibus->{devices}}) {
3275             next if !$device->{'qdev_id'};
3276             if ($device->{'pci_bridge'}) {
3277                 $devices->{$device->{'qdev_id'}} = 1;
3278                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3279                     next if !$bridge_device->{'qdev_id'};
3280                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3281                     $devices->{$device->{'qdev_id'}}++;
3282                 }
3283             } else {
3284                 $devices->{$device->{'qdev_id'}} = 1;
3285             }
3286         }
3287     }
3288
3289     my $resblock = vm_mon_cmd($vmid, 'query-block');
3290     foreach my $block (@$resblock) {
3291         if($block->{device} =~ m/^drive-(\S+)/){
3292                 $devices->{$1} = 1;
3293         }
3294     }
3295
3296     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3297     foreach my $mice (@$resmice) {
3298         if ($mice->{name} eq 'QEMU HID Tablet') {
3299             $devices->{tablet} = 1;
3300             last;
3301         }
3302     }
3303
3304     return $devices;
3305 }
3306
3307 sub vm_deviceplug {
3308     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3309
3310     my $q35 = machine_type_is_q35($conf);
3311
3312     my $devices_list = vm_devices_list($vmid);
3313     return 1 if defined($devices_list->{$deviceid});
3314
3315     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3316
3317     if ($deviceid eq 'tablet') {
3318
3319         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3320
3321     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3322
3323         qemu_iothread_add($vmid, $deviceid, $device);
3324
3325         qemu_driveadd($storecfg, $vmid, $device);
3326         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3327
3328         qemu_deviceadd($vmid, $devicefull);
3329         eval { qemu_deviceaddverify($vmid, $deviceid); };
3330         if (my $err = $@) {
3331             eval { qemu_drivedel($vmid, $deviceid); };
3332             warn $@ if $@;
3333             die $err;
3334         }
3335
3336     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3337
3338
3339         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3340         my $pciaddr = print_pci_addr($deviceid);
3341         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3342
3343         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3344
3345         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3346             qemu_iothread_add($vmid, $deviceid, $device);
3347             $devicefull .= ",iothread=iothread-$deviceid";
3348         }
3349
3350         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3351             $devicefull .= ",num_queues=$device->{queues}";
3352         }
3353
3354         qemu_deviceadd($vmid, $devicefull);
3355         qemu_deviceaddverify($vmid, $deviceid);
3356
3357     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3358
3359         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3360         qemu_driveadd($storecfg, $vmid, $device);
3361
3362         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3363         eval { qemu_deviceadd($vmid, $devicefull); };
3364         if (my $err = $@) {
3365             eval { qemu_drivedel($vmid, $deviceid); };
3366             warn $@ if $@;
3367             die $err;
3368         }
3369
3370     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3371
3372         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3373
3374         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3375         my $use_old_bios_files = undef;
3376         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3377
3378         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3379         qemu_deviceadd($vmid, $netdevicefull);
3380         eval { qemu_deviceaddverify($vmid, $deviceid); };
3381         if (my $err = $@) {
3382             eval { qemu_netdevdel($vmid, $deviceid); };
3383             warn $@ if $@;
3384             die $err;
3385         }
3386
3387     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3388
3389         my $bridgeid = $2;
3390         my $pciaddr = print_pci_addr($deviceid);
3391         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3392
3393         qemu_deviceadd($vmid, $devicefull);
3394         qemu_deviceaddverify($vmid, $deviceid);
3395
3396     } else {
3397         die "can't hotplug device '$deviceid'\n";
3398     }
3399
3400     return 1;
3401 }
3402
3403 # fixme: this should raise exceptions on error!
3404 sub vm_deviceunplug {
3405     my ($vmid, $conf, $deviceid) = @_;
3406
3407     my $devices_list = vm_devices_list($vmid);
3408     return 1 if !defined($devices_list->{$deviceid});
3409
3410     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3411
3412     if ($deviceid eq 'tablet') {
3413
3414         qemu_devicedel($vmid, $deviceid);
3415
3416     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3417
3418         qemu_devicedel($vmid, $deviceid);
3419         qemu_devicedelverify($vmid, $deviceid);
3420         qemu_drivedel($vmid, $deviceid);
3421         qemu_iothread_del($conf, $vmid, $deviceid);
3422
3423     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3424
3425         qemu_devicedel($vmid, $deviceid);
3426         qemu_devicedelverify($vmid, $deviceid);
3427         qemu_iothread_del($conf, $vmid, $deviceid);
3428
3429     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3430
3431         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3432         my $device = parse_drive($deviceid, $conf->{$deviceid});
3433         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3434
3435         qemu_devicedel($vmid, $deviceid);
3436         qemu_drivedel($vmid, $deviceid);
3437         qemu_deletescsihw($conf, $vmid, $deviceid);
3438
3439     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3440
3441         qemu_devicedel($vmid, $deviceid);
3442         qemu_devicedelverify($vmid, $deviceid);
3443         qemu_netdevdel($vmid, $deviceid);
3444
3445     } else {
3446         die "can't unplug device '$deviceid'\n";
3447     }
3448
3449     return 1;
3450 }
3451
3452 sub qemu_deviceadd {
3453     my ($vmid, $devicefull) = @_;
3454
3455     $devicefull = "driver=".$devicefull;
3456     my %options =  split(/[=,]/, $devicefull);
3457
3458     vm_mon_cmd($vmid, "device_add" , %options);
3459 }
3460
3461 sub qemu_devicedel {
3462     my ($vmid, $deviceid) = @_;
3463
3464     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3465 }
3466
3467 sub qemu_iothread_add {
3468     my($vmid, $deviceid, $device) = @_;
3469
3470     if ($device->{iothread}) {
3471         my $iothreads = vm_iothreads_list($vmid);
3472         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3473     }
3474 }
3475
3476 sub qemu_iothread_del {
3477     my($conf, $vmid, $deviceid) = @_;
3478
3479     my $device = parse_drive($deviceid, $conf->{$deviceid});
3480     if ($device->{iothread}) {
3481         my $iothreads = vm_iothreads_list($vmid);
3482         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3483     }
3484 }
3485
3486 sub qemu_objectadd {
3487     my($vmid, $objectid, $qomtype) = @_;
3488
3489     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3490
3491     return 1;
3492 }
3493
3494 sub qemu_objectdel {
3495     my($vmid, $objectid) = @_;
3496
3497     vm_mon_cmd($vmid, "object-del", id => $objectid);
3498
3499     return 1;
3500 }
3501
3502 sub qemu_driveadd {
3503     my ($storecfg, $vmid, $device) = @_;
3504
3505     my $drive = print_drive_full($storecfg, $vmid, $device);
3506     $drive =~ s/\\/\\\\/g;
3507     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3508
3509     # If the command succeeds qemu prints: "OK"
3510     return 1 if $ret =~ m/OK/s;
3511
3512     die "adding drive failed: $ret\n";
3513 }
3514
3515 sub qemu_drivedel {
3516     my($vmid, $deviceid) = @_;
3517
3518     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3519     $ret =~ s/^\s+//;
3520
3521     return 1 if $ret eq "";
3522
3523     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3524     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3525
3526     die "deleting drive $deviceid failed : $ret\n";
3527 }
3528
3529 sub qemu_deviceaddverify {
3530     my ($vmid, $deviceid) = @_;
3531
3532     for (my $i = 0; $i <= 5; $i++) {
3533          my $devices_list = vm_devices_list($vmid);
3534          return 1 if defined($devices_list->{$deviceid});
3535          sleep 1;
3536     }
3537
3538     die "error on hotplug device '$deviceid'\n";
3539 }
3540
3541
3542 sub qemu_devicedelverify {
3543     my ($vmid, $deviceid) = @_;
3544
3545     # need to verify that the device is correctly removed as device_del
3546     # is async and empty return is not reliable
3547
3548     for (my $i = 0; $i <= 5; $i++) {
3549          my $devices_list = vm_devices_list($vmid);
3550          return 1 if !defined($devices_list->{$deviceid});
3551          sleep 1;
3552     }
3553
3554     die "error on hot-unplugging device '$deviceid'\n";
3555 }
3556
3557 sub qemu_findorcreatescsihw {
3558     my ($storecfg, $conf, $vmid, $device) = @_;
3559
3560     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3561
3562     my $scsihwid="$controller_prefix$controller";
3563     my $devices_list = vm_devices_list($vmid);
3564
3565     if(!defined($devices_list->{$scsihwid})) {
3566         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3567     }
3568
3569     return 1;
3570 }
3571
3572 sub qemu_deletescsihw {
3573     my ($conf, $vmid, $opt) = @_;
3574
3575     my $device = parse_drive($opt, $conf->{$opt});
3576
3577     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3578         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3579         return 1;
3580     }
3581
3582     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3583
3584     my $devices_list = vm_devices_list($vmid);
3585     foreach my $opt (keys %{$devices_list}) {
3586         if (PVE::QemuServer::valid_drivename($opt)) {
3587             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3588             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3589                 return 1;
3590             }
3591         }
3592     }
3593
3594     my $scsihwid="scsihw$controller";
3595
3596     vm_deviceunplug($vmid, $conf, $scsihwid);
3597
3598     return 1;
3599 }
3600
3601 sub qemu_add_pci_bridge {
3602     my ($storecfg, $conf, $vmid, $device) = @_;
3603
3604     my $bridges = {};
3605
3606     my $bridgeid;
3607
3608     print_pci_addr($device, $bridges);
3609
3610     while (my ($k, $v) = each %$bridges) {
3611         $bridgeid = $k;
3612     }
3613     return 1 if !defined($bridgeid) || $bridgeid < 1;
3614
3615     my $bridge = "pci.$bridgeid";
3616     my $devices_list = vm_devices_list($vmid);
3617
3618     if (!defined($devices_list->{$bridge})) {
3619         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3620     }
3621
3622     return 1;
3623 }
3624
3625 sub qemu_set_link_status {
3626     my ($vmid, $device, $up) = @_;
3627
3628     vm_mon_cmd($vmid, "set_link", name => $device,
3629                up => $up ? JSON::true : JSON::false);
3630 }
3631
3632 sub qemu_netdevadd {
3633     my ($vmid, $conf, $device, $deviceid) = @_;
3634
3635     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3636     my %options =  split(/[=,]/, $netdev);
3637
3638     vm_mon_cmd($vmid, "netdev_add",  %options);
3639     return 1;
3640 }
3641
3642 sub qemu_netdevdel {
3643     my ($vmid, $deviceid) = @_;
3644
3645     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3646 }
3647
3648 sub qemu_cpu_hotplug {
3649     my ($vmid, $conf, $vcpus) = @_;
3650
3651     my $sockets = 1;
3652     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3653     $sockets = $conf->{sockets} if  $conf->{sockets};
3654     my $cores = $conf->{cores} || 1;
3655     my $maxcpus = $sockets * $cores;
3656
3657     $vcpus = $maxcpus if !$vcpus;
3658
3659     die "you can't add more vcpus than maxcpus\n"
3660         if $vcpus > $maxcpus;
3661
3662     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3663     die "online cpu unplug is not yet possible\n"
3664         if $vcpus < $currentvcpus;
3665
3666     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3667     die "vcpus in running vm is different than configuration\n"
3668         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3669
3670     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3671         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3672     }
3673 }
3674
3675 sub qemu_memory_hotplug {
3676     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3677
3678     return $value if !check_running($vmid);
3679
3680     my $memory = $conf->{memory} || $defaults->{memory};
3681     $value = $defaults->{memory} if !$value;
3682     return $value if $value == $memory;
3683
3684     my $static_memory = $STATICMEM;
3685     my $dimm_memory = $memory - $static_memory;
3686
3687     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3688     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3689
3690
3691     my $sockets = 1;
3692     $sockets = $conf->{sockets} if $conf->{sockets};
3693
3694     if($value > $memory) {
3695
3696         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3697             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3698
3699                 return if $current_size <= $conf->{memory};
3700
3701                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3702                 if (my $err = $@) {
3703                     eval { qemu_objectdel($vmid, "mem-$name"); };
3704                     die $err;
3705                 }
3706
3707                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3708                 if (my $err = $@) {
3709                     eval { qemu_objectdel($vmid, "mem-$name"); };
3710                     die $err;
3711                 }
3712                 #update conf after each succesful module hotplug
3713                 $conf->{memory} = $current_size;
3714                 update_config_nolock($vmid, $conf, 1);
3715         });
3716
3717     } else {
3718
3719         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3720             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3721
3722                 return if $current_size >= $conf->{memory};
3723                 print "try to unplug memory dimm $name\n";
3724
3725                 my $retry = 0;
3726                 while (1) {
3727                     eval { qemu_devicedel($vmid, $name) };
3728                     sleep 3;
3729                     my $dimm_list = qemu_dimm_list($vmid);
3730                     last if !$dimm_list->{$name};
3731                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3732                     $retry++;
3733                 }
3734
3735                 #update conf after each succesful module unplug
3736                 $conf->{memory} = $current_size;
3737
3738                 eval { qemu_objectdel($vmid, "mem-$name"); };
3739                 update_config_nolock($vmid, $conf, 1);
3740         });
3741     }
3742 }
3743
3744 sub qemu_dimm_list {
3745     my ($vmid) = @_;
3746
3747     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3748     my $dimms = {};
3749
3750     foreach my $dimm (@$dimmarray) {
3751
3752         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3753         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3754         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3755         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3756         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3757     }
3758     return $dimms;
3759 }
3760
3761 sub qemu_block_set_io_throttle {
3762     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3763
3764     return if !check_running($vmid) ;
3765
3766     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3767
3768 }
3769
3770 # old code, only used to shutdown old VM after update
3771 sub __read_avail {
3772     my ($fh, $timeout) = @_;
3773
3774     my $sel = new IO::Select;
3775     $sel->add($fh);
3776
3777     my $res = '';
3778     my $buf;
3779
3780     my @ready;
3781     while (scalar (@ready = $sel->can_read($timeout))) {
3782         my $count;
3783         if ($count = $fh->sysread($buf, 8192)) {
3784             if ($buf =~ /^(.*)\(qemu\) $/s) {
3785                 $res .= $1;
3786                 last;
3787             } else {
3788                 $res .= $buf;
3789             }
3790         } else {
3791             if (!defined($count)) {
3792                 die "$!\n";
3793             }
3794             last;
3795         }
3796     }
3797
3798     die "monitor read timeout\n" if !scalar(@ready);
3799
3800     return $res;
3801 }
3802
3803 # old code, only used to shutdown old VM after update
3804 sub vm_monitor_command {
3805     my ($vmid, $cmdstr, $nocheck) = @_;
3806
3807     my $res;
3808
3809     eval {
3810         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3811
3812         my $sname = "${var_run_tmpdir}/$vmid.mon";
3813
3814         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3815             die "unable to connect to VM $vmid socket - $!\n";
3816
3817         my $timeout = 3;
3818
3819         # hack: migrate sometime blocks the monitor (when migrate_downtime
3820         # is set)
3821         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3822             $timeout = 60*60; # 1 hour
3823         }
3824
3825         # read banner;
3826         my $data = __read_avail($sock, $timeout);
3827
3828         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3829             die "got unexpected qemu monitor banner\n";
3830         }
3831
3832         my $sel = new IO::Select;
3833         $sel->add($sock);
3834
3835         if (!scalar(my @ready = $sel->can_write($timeout))) {
3836             die "monitor write error - timeout";
3837         }
3838
3839         my $fullcmd = "$cmdstr\r";
3840
3841         # syslog('info', "VM $vmid monitor command: $cmdstr");
3842
3843         my $b;
3844         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3845             die "monitor write error - $!";
3846         }
3847
3848         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3849
3850         $timeout = 20;
3851
3852         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3853             $timeout = 60*60; # 1 hour
3854         } elsif ($cmdstr =~ m/^(eject|change)/) {
3855             $timeout = 60; # note: cdrom mount command is slow
3856         }
3857         if ($res = __read_avail($sock, $timeout)) {
3858
3859             my @lines = split("\r?\n", $res);
3860
3861             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3862
3863             $res = join("\n", @lines);
3864             $res .= "\n";
3865         }
3866     };
3867
3868     my $err = $@;
3869
3870     if ($err) {
3871         syslog("err", "VM $vmid monitor command failed - $err");
3872         die $err;
3873     }
3874
3875     return $res;
3876 }
3877
3878 sub qemu_block_resize {
3879     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3880
3881     my $running = check_running($vmid);
3882
3883     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3884
3885     return if !$running;
3886
3887     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3888
3889 }
3890
3891 sub qemu_volume_snapshot {
3892     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3893
3894     my $running = check_running($vmid);
3895
3896     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3897         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3898     } else {
3899         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3900     }
3901 }
3902
3903 sub qemu_volume_snapshot_delete {
3904     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3905
3906     my $running = check_running($vmid);
3907
3908     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3909
3910     return if !$running;
3911
3912     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3913 }
3914
3915 sub set_migration_caps {
3916     my ($vmid) = @_;
3917
3918     my $cap_ref = [];
3919
3920     my $enabled_cap = {
3921         "auto-converge" => 1,
3922         "xbzrle" => 1,
3923         "x-rdma-pin-all" => 0,
3924         "zero-blocks" => 0,
3925         "compress" => 0
3926     };
3927
3928     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3929
3930     for my $supported_capability (@$supported_capabilities) {
3931         push @$cap_ref, {
3932             capability => $supported_capability->{capability},
3933             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3934         };
3935     }
3936
3937     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3938 }
3939
3940 my $fast_plug_option = {
3941     'lock' => 1,
3942     'name' => 1,
3943     'onboot' => 1,
3944     'shares' => 1,
3945     'startup' => 1,
3946     'description' => 1,
3947 };
3948
3949 # hotplug changes in [PENDING]
3950 # $selection hash can be used to only apply specified options, for
3951 # example: { cores => 1 } (only apply changed 'cores')
3952 # $errors ref is used to return error messages
3953 sub vmconfig_hotplug_pending {
3954     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3955
3956     my $defaults = load_defaults();
3957
3958     # commit values which do not have any impact on running VM first
3959     # Note: those option cannot raise errors, we we do not care about
3960     # $selection and always apply them.
3961
3962     my $add_error = sub {
3963         my ($opt, $msg) = @_;
3964         $errors->{$opt} = "hotplug problem - $msg";
3965     };
3966
3967     my $changes = 0;
3968     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3969         if ($fast_plug_option->{$opt}) {
3970             $conf->{$opt} = $conf->{pending}->{$opt};
3971             delete $conf->{pending}->{$opt};
3972             $changes = 1;
3973         }
3974     }
3975
3976     if ($changes) {
3977         update_config_nolock($vmid, $conf, 1);
3978         $conf = load_config($vmid); # update/reload
3979     }
3980
3981     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3982
3983     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3984     while (my ($opt, $force) = each %$pending_delete_hash) {
3985         next if $selection && !$selection->{$opt};
3986         eval {
3987             if ($opt eq 'hotplug') {
3988                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3989             } elsif ($opt eq 'tablet') {
3990                 die "skip\n" if !$hotplug_features->{usb};
3991                 if ($defaults->{tablet}) {
3992                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3993                 } else {
3994                     vm_deviceunplug($vmid, $conf, $opt);
3995                 }
3996             } elsif ($opt eq 'vcpus') {
3997                 die "skip\n" if !$hotplug_features->{cpu};
3998                 qemu_cpu_hotplug($vmid, $conf, undef);
3999             } elsif ($opt eq 'balloon') {
4000                 # enable balloon device is not hotpluggable
4001                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4002             } elsif ($fast_plug_option->{$opt}) {
4003                 # do nothing
4004             } elsif ($opt =~ m/^net(\d+)$/) {
4005                 die "skip\n" if !$hotplug_features->{network};
4006                 vm_deviceunplug($vmid, $conf, $opt);
4007             } elsif (valid_drivename($opt)) {
4008                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4009                 vm_deviceunplug($vmid, $conf, $opt);
4010                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4011             } elsif ($opt =~ m/^memory$/) {
4012                 die "skip\n" if !$hotplug_features->{memory};
4013                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4014             } elsif ($opt eq 'cpuunits') {
4015                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4016             } elsif ($opt eq 'cpulimit') {
4017                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4018             } else {
4019                 die "skip\n";
4020             }
4021         };
4022         if (my $err = $@) {
4023             &$add_error($opt, $err) if $err ne "skip\n";
4024         } else {
4025             # save new config if hotplug was successful
4026             delete $conf->{$opt};
4027             vmconfig_undelete_pending_option($conf, $opt);
4028             update_config_nolock($vmid, $conf, 1);
4029             $conf = load_config($vmid); # update/reload
4030         }
4031     }
4032
4033     foreach my $opt (keys %{$conf->{pending}}) {
4034         next if $selection && !$selection->{$opt};
4035         my $value = $conf->{pending}->{$opt};
4036         eval {
4037             if ($opt eq 'hotplug') {
4038                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4039             } elsif ($opt eq 'tablet') {
4040                 die "skip\n" if !$hotplug_features->{usb};
4041                 if ($value == 1) {
4042                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4043                 } elsif ($value == 0) {
4044                     vm_deviceunplug($vmid, $conf, $opt);
4045                 }
4046             } elsif ($opt eq 'vcpus') {
4047                 die "skip\n" if !$hotplug_features->{cpu};
4048                 qemu_cpu_hotplug($vmid, $conf, $value);
4049             } elsif ($opt eq 'balloon') {
4050                 # enable/disable balloning device is not hotpluggable
4051                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4052                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4053                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4054
4055                 # allow manual ballooning if shares is set to zero
4056                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4057                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4058                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4059                 }
4060             } elsif ($opt =~ m/^net(\d+)$/) {
4061                 # some changes can be done without hotplug
4062                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4063                                     $vmid, $opt, $value);
4064             } elsif (valid_drivename($opt)) {
4065                 # some changes can be done without hotplug
4066                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4067                                      $vmid, $opt, $value, 1);
4068             } elsif ($opt =~ m/^memory$/) { #dimms
4069                 die "skip\n" if !$hotplug_features->{memory};
4070                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4071             } elsif ($opt eq 'cpuunits') {
4072                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4073             } elsif ($opt eq 'cpulimit') {
4074                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4075                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4076             } else {
4077                 die "skip\n";  # skip non-hot-pluggable options
4078             }
4079         };
4080         if (my $err = $@) {
4081             &$add_error($opt, $err) if $err ne "skip\n";
4082         } else {
4083             # save new config if hotplug was successful
4084             $conf->{$opt} = $value;
4085             delete $conf->{pending}->{$opt};
4086             update_config_nolock($vmid, $conf, 1);
4087             $conf = load_config($vmid); # update/reload
4088         }
4089     }
4090 }
4091
4092 sub try_deallocate_drive {
4093     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4094
4095     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4096         my $volid = $drive->{file};
4097         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4098             my $sid = PVE::Storage::parse_volume_id($volid);
4099             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4100
4101             # check if the disk is really unused
4102             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4103                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4104             PVE::Storage::vdisk_free($storecfg, $volid);
4105             return 1;
4106         } else {
4107             # If vm is not owner of this disk remove from config
4108             return 1;
4109         }
4110     }
4111
4112     return undef;
4113 }
4114
4115 sub vmconfig_delete_or_detach_drive {
4116     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4117
4118     my $drive = parse_drive($opt, $conf->{$opt});
4119
4120     my $rpcenv = PVE::RPCEnvironment::get();
4121     my $authuser = $rpcenv->get_user();
4122
4123     if ($force) {
4124         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4125         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4126     } else {
4127         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4128     }
4129 }
4130
4131 sub vmconfig_apply_pending {
4132     my ($vmid, $conf, $storecfg) = @_;
4133
4134     # cold plug
4135
4136     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4137     while (my ($opt, $force) = each %$pending_delete_hash) {
4138         die "internal error" if $opt =~ m/^unused/;
4139         $conf = load_config($vmid); # update/reload
4140         if (!defined($conf->{$opt})) {
4141             vmconfig_undelete_pending_option($conf, $opt);
4142             update_config_nolock($vmid, $conf, 1);
4143         } elsif (valid_drivename($opt)) {
4144             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4145             vmconfig_undelete_pending_option($conf, $opt);
4146             delete $conf->{$opt};
4147             update_config_nolock($vmid, $conf, 1);
4148         } else {
4149             vmconfig_undelete_pending_option($conf, $opt);
4150             delete $conf->{$opt};
4151             update_config_nolock($vmid, $conf, 1);
4152         }
4153     }
4154
4155     $conf = load_config($vmid); # update/reload
4156
4157     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4158         $conf = load_config($vmid); # update/reload
4159
4160         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4161             # skip if nothing changed
4162         } elsif (valid_drivename($opt)) {
4163             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4164                 if defined($conf->{$opt});
4165             $conf->{$opt} = $conf->{pending}->{$opt};
4166         } else {
4167             $conf->{$opt} = $conf->{pending}->{$opt};
4168         }
4169
4170         delete $conf->{pending}->{$opt};
4171         update_config_nolock($vmid, $conf, 1);
4172     }
4173 }
4174
4175 my $safe_num_ne = sub {
4176     my ($a, $b) = @_;
4177
4178     return 0 if !defined($a) && !defined($b);
4179     return 1 if !defined($a);
4180     return 1 if !defined($b);
4181
4182     return $a != $b;
4183 };
4184
4185 my $safe_string_ne = sub {
4186     my ($a, $b) = @_;
4187
4188     return 0 if !defined($a) && !defined($b);
4189     return 1 if !defined($a);
4190     return 1 if !defined($b);
4191
4192     return $a ne $b;
4193 };
4194
4195 sub vmconfig_update_net {
4196     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4197
4198     my $newnet = parse_net($value);
4199
4200     if ($conf->{$opt}) {
4201         my $oldnet = parse_net($conf->{$opt});
4202
4203         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4204             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4205             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4206             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4207
4208             # for non online change, we try to hot-unplug
4209             die "skip\n" if !$hotplug;
4210             vm_deviceunplug($vmid, $conf, $opt);
4211         } else {
4212
4213             die "internal error" if $opt !~ m/net(\d+)/;
4214             my $iface = "tap${vmid}i$1";
4215
4216             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4217                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4218             }
4219
4220             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4221                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4222                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4223                 PVE::Network::tap_unplug($iface);
4224                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4225             }
4226
4227             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4228                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4229             }
4230
4231             return 1;
4232         }
4233     }
4234
4235     if ($hotplug) {
4236         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4237     } else {
4238         die "skip\n";
4239     }
4240 }
4241
4242 sub vmconfig_update_disk {
4243     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4244
4245     # fixme: do we need force?
4246
4247     my $drive = parse_drive($opt, $value);
4248
4249     if ($conf->{$opt}) {
4250
4251         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4252
4253             my $media = $drive->{media} || 'disk';
4254             my $oldmedia = $old_drive->{media} || 'disk';
4255             die "unable to change media type\n" if $media ne $oldmedia;
4256
4257             if (!drive_is_cdrom($old_drive)) {
4258
4259                 if ($drive->{file} ne $old_drive->{file}) {
4260
4261                     die "skip\n" if !$hotplug;
4262
4263                     # unplug and register as unused
4264                     vm_deviceunplug($vmid, $conf, $opt);
4265                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4266
4267                 } else {
4268                     # update existing disk
4269
4270                     # skip non hotpluggable value
4271                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4272                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4273                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4274                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4275                         die "skip\n";
4276                     }
4277
4278                     # apply throttle
4279                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4280                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4281                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4282                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4283                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4284                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4285                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4286                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4287                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4288                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4289                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4290                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4291
4292                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4293                                                    ($drive->{mbps} || 0)*1024*1024,
4294                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4295                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4296                                                    $drive->{iops} || 0,
4297                                                    $drive->{iops_rd} || 0,
4298                                                    $drive->{iops_wr} || 0,
4299                                                    ($drive->{mbps_max} || 0)*1024*1024,
4300                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4301                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4302                                                    $drive->{iops_max} || 0,
4303                                                    $drive->{iops_rd_max} || 0,
4304                                                    $drive->{iops_wr_max} || 0);
4305
4306                     }
4307
4308                     return 1;
4309                 }
4310
4311             } else { # cdrom
4312
4313                 if ($drive->{file} eq 'none') {
4314                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4315                 } else {
4316                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4317                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4318                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4319                 }
4320
4321                 return 1;
4322             }
4323         }
4324     }
4325
4326     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4327     # hotplug new disks
4328     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4329     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4330 }
4331
4332 sub vm_start {
4333     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4334         $forcemachine, $spice_ticket) = @_;
4335
4336     lock_config($vmid, sub {
4337         my $conf = load_config($vmid, $migratedfrom);
4338
4339         die "you can't start a vm if it's a template\n" if is_template($conf);
4340
4341         check_lock($conf) if !$skiplock;
4342
4343         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4344
4345         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4346             vmconfig_apply_pending($vmid, $conf, $storecfg);
4347             $conf = load_config($vmid); # update/reload
4348         }
4349
4350         my $defaults = load_defaults();
4351
4352         # set environment variable useful inside network script
4353         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4354
4355         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4356
4357         my $migrate_port = 0;
4358         my $migrate_uri;
4359         if ($statefile) {
4360             if ($statefile eq 'tcp') {
4361                 my $localip = "localhost";
4362                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4363                 my $nodename = PVE::INotify::nodename();
4364                 if ($datacenterconf->{migration_unsecure}) {
4365                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4366                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4367                 }
4368                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4369                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4370                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4371                 push @$cmd, '-incoming', $migrate_uri;
4372                 push @$cmd, '-S';
4373             } else {
4374                 push @$cmd, '-loadstate', $statefile;
4375             }
4376         } elsif ($paused) {
4377             push @$cmd, '-S';
4378         }
4379
4380         # host pci devices
4381         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4382           my $d = parse_hostpci($conf->{"hostpci$i"});
4383           next if !$d;
4384           my $pcidevices = $d->{pciid};
4385           foreach my $pcidevice (@$pcidevices) {
4386                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4387
4388                 my $info = pci_device_info("0000:$pciid");
4389                 die "IOMMU not present\n" if !check_iommu_support();
4390                 die "no pci device info for device '$pciid'\n" if !$info;
4391                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4392                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4393           }
4394         }
4395
4396         PVE::Storage::activate_volumes($storecfg, $vollist);
4397
4398         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4399                     umask => 0077); };
4400         my $err = $@;
4401         die "start failed: $err" if $err;
4402
4403         print "migration listens on $migrate_uri\n" if $migrate_uri;
4404
4405         if ($statefile && $statefile ne 'tcp')  {
4406             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4407             warn $@ if $@;
4408         }
4409
4410         if ($migratedfrom) {
4411
4412             eval {
4413                 set_migration_caps($vmid);
4414             };
4415             warn $@ if $@;
4416
4417             if ($spice_port) {
4418                 print "spice listens on port $spice_port\n";
4419                 if ($spice_ticket) {
4420                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4421                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4422                 }
4423             }
4424
4425         } else {
4426
4427             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4428                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4429                     if $conf->{balloon};
4430             }
4431
4432             foreach my $opt (keys %$conf) {
4433                 next if $opt !~  m/^net\d+$/;
4434                 my $nicconf = parse_net($conf->{$opt});
4435                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4436             }
4437         }
4438
4439         vm_mon_cmd_nocheck($vmid, 'qom-set',
4440                     path => "machine/peripheral/balloon0",
4441                     property => "guest-stats-polling-interval",
4442                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4443
4444     });
4445 }
4446
4447 sub vm_mon_cmd {
4448     my ($vmid, $execute, %params) = @_;
4449
4450     my $cmd = { execute => $execute, arguments => \%params };
4451     vm_qmp_command($vmid, $cmd);
4452 }
4453
4454 sub vm_mon_cmd_nocheck {
4455     my ($vmid, $execute, %params) = @_;
4456
4457     my $cmd = { execute => $execute, arguments => \%params };
4458     vm_qmp_command($vmid, $cmd, 1);
4459 }
4460
4461 sub vm_qmp_command {
4462     my ($vmid, $cmd, $nocheck) = @_;
4463
4464     my $res;
4465
4466     my $timeout;
4467     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4468         $timeout = $cmd->{arguments}->{timeout};
4469         delete $cmd->{arguments}->{timeout};
4470     }
4471
4472     eval {
4473         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4474         my $sname = qmp_socket($vmid);
4475         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4476             my $qmpclient = PVE::QMPClient->new();
4477
4478             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4479         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4480             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4481                 if scalar(%{$cmd->{arguments}});
4482             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4483         } else {
4484             die "unable to open monitor socket\n";
4485         }
4486     };
4487     if (my $err = $@) {
4488         syslog("err", "VM $vmid qmp command failed - $err");
4489         die $err;
4490     }
4491
4492     return $res;
4493 }
4494
4495 sub vm_human_monitor_command {
4496     my ($vmid, $cmdline) = @_;
4497
4498     my $res;
4499
4500     my $cmd = {
4501         execute => 'human-monitor-command',
4502         arguments => { 'command-line' => $cmdline},
4503     };
4504
4505     return vm_qmp_command($vmid, $cmd);
4506 }
4507
4508 sub vm_commandline {
4509     my ($storecfg, $vmid) = @_;
4510
4511     my $conf = load_config($vmid);
4512
4513     my $defaults = load_defaults();
4514
4515     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4516
4517     return join(' ', @$cmd);
4518 }
4519
4520 sub vm_reset {
4521     my ($vmid, $skiplock) = @_;
4522
4523     lock_config($vmid, sub {
4524
4525         my $conf = load_config($vmid);
4526
4527         check_lock($conf) if !$skiplock;
4528
4529         vm_mon_cmd($vmid, "system_reset");
4530     });
4531 }
4532
4533 sub get_vm_volumes {
4534     my ($conf) = @_;
4535
4536     my $vollist = [];
4537     foreach_volid($conf, sub {
4538         my ($volid, $is_cdrom) = @_;
4539
4540         return if $volid =~ m|^/|;
4541
4542         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4543         return if !$sid;
4544
4545         push @$vollist, $volid;
4546     });
4547
4548     return $vollist;
4549 }
4550
4551 sub vm_stop_cleanup {
4552     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4553
4554     eval {
4555
4556         if (!$keepActive) {
4557             my $vollist = get_vm_volumes($conf);
4558             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4559         }
4560
4561         foreach my $ext (qw(mon qmp pid vnc qga)) {
4562             unlink "/var/run/qemu-server/${vmid}.$ext";
4563         }
4564
4565         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4566     };
4567     warn $@ if $@; # avoid errors - just warn
4568 }
4569
4570 # Note: use $nockeck to skip tests if VM configuration file exists.
4571 # We need that when migration VMs to other nodes (files already moved)
4572 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4573 sub vm_stop {
4574     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4575
4576     $force = 1 if !defined($force) && !$shutdown;
4577
4578     if ($migratedfrom){
4579         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4580         kill 15, $pid if $pid;
4581         my $conf = load_config($vmid, $migratedfrom);
4582         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4583         return;
4584     }
4585
4586     lock_config($vmid, sub {
4587
4588         my $pid = check_running($vmid, $nocheck);
4589         return if !$pid;
4590
4591         my $conf;
4592         if (!$nocheck) {
4593             $conf = load_config($vmid);
4594             check_lock($conf) if !$skiplock;
4595             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4596                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4597                 $timeout = $opts->{down} if $opts->{down};
4598             }
4599         }
4600
4601         $timeout = 60 if !defined($timeout);
4602
4603         eval {
4604             if ($shutdown) {
4605                 if (defined($conf) && $conf->{agent}) {
4606                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4607                 } else {
4608                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4609                 }
4610             } else {
4611                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4612             }
4613         };
4614         my $err = $@;
4615
4616         if (!$err) {
4617             my $count = 0;
4618             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4619                 $count++;
4620                 sleep 1;
4621             }
4622
4623             if ($count >= $timeout) {
4624                 if ($force) {
4625                     warn "VM still running - terminating now with SIGTERM\n";
4626                     kill 15, $pid;
4627                 } else {
4628                     die "VM quit/powerdown failed - got timeout\n";
4629                 }
4630             } else {
4631                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4632                 return;
4633             }
4634         } else {
4635             if ($force) {
4636                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4637                 kill 15, $pid;
4638             } else {
4639                 die "VM quit/powerdown failed\n";
4640             }
4641         }
4642
4643         # wait again
4644         $timeout = 10;
4645
4646         my $count = 0;
4647         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4648             $count++;
4649             sleep 1;
4650         }
4651
4652         if ($count >= $timeout) {
4653             warn "VM still running - terminating now with SIGKILL\n";
4654             kill 9, $pid;
4655             sleep 1;
4656         }
4657
4658         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4659    });
4660 }
4661
4662 sub vm_suspend {
4663     my ($vmid, $skiplock) = @_;
4664
4665     lock_config($vmid, sub {
4666
4667         my $conf = load_config($vmid);
4668
4669         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4670
4671         vm_mon_cmd($vmid, "stop");
4672     });
4673 }
4674
4675 sub vm_resume {
4676     my ($vmid, $skiplock, $nocheck) = @_;
4677
4678     lock_config($vmid, sub {
4679
4680         if (!$nocheck) {
4681
4682             my $conf = load_config($vmid);
4683
4684             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4685
4686             vm_mon_cmd($vmid, "cont");
4687
4688         } else {
4689             vm_mon_cmd_nocheck($vmid, "cont");
4690         }
4691     });
4692 }
4693
4694 sub vm_sendkey {
4695     my ($vmid, $skiplock, $key) = @_;
4696
4697     lock_config($vmid, sub {
4698
4699         my $conf = load_config($vmid);
4700
4701         # there is no qmp command, so we use the human monitor command
4702         vm_human_monitor_command($vmid, "sendkey $key");
4703     });
4704 }
4705
4706 sub vm_destroy {
4707     my ($storecfg, $vmid, $skiplock) = @_;
4708
4709     lock_config($vmid, sub {
4710
4711         my $conf = load_config($vmid);
4712
4713         check_lock($conf) if !$skiplock;
4714
4715         if (!check_running($vmid)) {
4716             destroy_vm($storecfg, $vmid);
4717         } else {
4718             die "VM $vmid is running - destroy failed\n";
4719         }
4720     });
4721 }
4722
4723 # pci helpers
4724
4725 sub file_write {
4726     my ($filename, $buf) = @_;
4727
4728     my $fh = IO::File->new($filename, "w");
4729     return undef if !$fh;
4730
4731     my $res = print $fh $buf;
4732
4733     $fh->close();
4734
4735     return $res;
4736 }
4737
4738 sub pci_device_info {
4739     my ($name) = @_;
4740
4741     my $res;
4742
4743     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4744     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4745
4746     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4747     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4748
4749     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4750     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4751
4752     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4753     return undef if !defined($product) || $product !~ s/^0x//;
4754
4755     $res = {
4756         name => $name,
4757         vendor => $vendor,
4758         product => $product,
4759         domain => $domain,
4760         bus => $bus,
4761         slot => $slot,
4762         func => $func,
4763         irq => $irq,
4764         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4765     };
4766
4767     return $res;
4768 }
4769
4770 sub pci_dev_reset {
4771     my ($dev) = @_;
4772
4773     my $name = $dev->{name};
4774
4775     my $fn = "$pcisysfs/devices/$name/reset";
4776
4777     return file_write($fn, "1");
4778 }
4779
4780 sub pci_dev_bind_to_vfio {
4781     my ($dev) = @_;
4782
4783     my $name = $dev->{name};
4784
4785     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4786
4787     if (!-d $vfio_basedir) {
4788         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4789     }
4790     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4791
4792     my $testdir = "$vfio_basedir/$name";
4793     return 1 if -d $testdir;
4794
4795     my $data = "$dev->{vendor} $dev->{product}";
4796     return undef if !file_write("$vfio_basedir/new_id", $data);
4797
4798     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4799     if (!file_write($fn, $name)) {
4800         return undef if -f $fn;
4801     }
4802
4803     $fn = "$vfio_basedir/bind";
4804     if (! -d $testdir) {
4805         return undef if !file_write($fn, $name);
4806     }
4807
4808     return -d $testdir;
4809 }
4810
4811 sub pci_dev_group_bind_to_vfio {
4812     my ($pciid) = @_;
4813
4814     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4815
4816     if (!-d $vfio_basedir) {
4817         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4818     }
4819     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4820
4821     # get IOMMU group devices
4822     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4823       my @devs = grep /^0000:/, readdir($D);
4824     closedir($D);
4825
4826     foreach my $pciid (@devs) {
4827         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4828
4829         # pci bridges, switches or root ports are not supported
4830         # they have a pci_bus subdirectory so skip them
4831         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4832
4833         my $info = pci_device_info($1);
4834         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4835     }
4836
4837     return 1;
4838 }
4839
4840 sub print_pci_addr {
4841     my ($id, $bridges) = @_;
4842
4843     my $res = '';
4844     my $devices = {
4845         piix3 => { bus => 0, addr => 1 },
4846         #addr2 : first videocard
4847         balloon0 => { bus => 0, addr => 3 },
4848         watchdog => { bus => 0, addr => 4 },
4849         scsihw0 => { bus => 0, addr => 5 },
4850         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4851         scsihw1 => { bus => 0, addr => 6 },
4852         ahci0 => { bus => 0, addr => 7 },
4853         qga0 => { bus => 0, addr => 8 },
4854         spice => { bus => 0, addr => 9 },
4855         virtio0 => { bus => 0, addr => 10 },
4856         virtio1 => { bus => 0, addr => 11 },
4857         virtio2 => { bus => 0, addr => 12 },
4858         virtio3 => { bus => 0, addr => 13 },
4859         virtio4 => { bus => 0, addr => 14 },
4860         virtio5 => { bus => 0, addr => 15 },
4861         hostpci0 => { bus => 0, addr => 16 },
4862         hostpci1 => { bus => 0, addr => 17 },
4863         net0 => { bus => 0, addr => 18 },
4864         net1 => { bus => 0, addr => 19 },
4865         net2 => { bus => 0, addr => 20 },
4866         net3 => { bus => 0, addr => 21 },
4867         net4 => { bus => 0, addr => 22 },
4868         net5 => { bus => 0, addr => 23 },
4869         vga1 => { bus => 0, addr => 24 },
4870         vga2 => { bus => 0, addr => 25 },
4871         vga3 => { bus => 0, addr => 26 },
4872         hostpci2 => { bus => 0, addr => 27 },
4873         hostpci3 => { bus => 0, addr => 28 },
4874         #addr29 : usb-host (pve-usb.cfg)
4875         'pci.1' => { bus => 0, addr => 30 },
4876         'pci.2' => { bus => 0, addr => 31 },
4877         'net6' => { bus => 1, addr => 1 },
4878         'net7' => { bus => 1, addr => 2 },
4879         'net8' => { bus => 1, addr => 3 },
4880         'net9' => { bus => 1, addr => 4 },
4881         'net10' => { bus => 1, addr => 5 },
4882         'net11' => { bus => 1, addr => 6 },
4883         'net12' => { bus => 1, addr => 7 },
4884         'net13' => { bus => 1, addr => 8 },
4885         'net14' => { bus => 1, addr => 9 },
4886         'net15' => { bus => 1, addr => 10 },
4887         'net16' => { bus => 1, addr => 11 },
4888         'net17' => { bus => 1, addr => 12 },
4889         'net18' => { bus => 1, addr => 13 },
4890         'net19' => { bus => 1, addr => 14 },
4891         'net20' => { bus => 1, addr => 15 },
4892         'net21' => { bus => 1, addr => 16 },
4893         'net22' => { bus => 1, addr => 17 },
4894         'net23' => { bus => 1, addr => 18 },
4895         'net24' => { bus => 1, addr => 19 },
4896         'net25' => { bus => 1, addr => 20 },
4897         'net26' => { bus => 1, addr => 21 },
4898         'net27' => { bus => 1, addr => 22 },
4899         'net28' => { bus => 1, addr => 23 },
4900         'net29' => { bus => 1, addr => 24 },
4901         'net30' => { bus => 1, addr => 25 },
4902         'net31' => { bus => 1, addr => 26 },
4903         'virtio6' => { bus => 2, addr => 1 },
4904         'virtio7' => { bus => 2, addr => 2 },
4905         'virtio8' => { bus => 2, addr => 3 },
4906         'virtio9' => { bus => 2, addr => 4 },
4907         'virtio10' => { bus => 2, addr => 5 },
4908         'virtio11' => { bus => 2, addr => 6 },
4909         'virtio12' => { bus => 2, addr => 7 },
4910         'virtio13' => { bus => 2, addr => 8 },
4911         'virtio14' => { bus => 2, addr => 9 },
4912         'virtio15' => { bus => 2, addr => 10 },
4913         'virtioscsi0' => { bus => 3, addr => 1 },
4914         'virtioscsi1' => { bus => 3, addr => 2 },
4915         'virtioscsi2' => { bus => 3, addr => 3 },
4916         'virtioscsi3' => { bus => 3, addr => 4 },
4917         'virtioscsi4' => { bus => 3, addr => 5 },
4918         'virtioscsi5' => { bus => 3, addr => 6 },
4919         'virtioscsi6' => { bus => 3, addr => 7 },
4920         'virtioscsi7' => { bus => 3, addr => 8 },
4921         'virtioscsi8' => { bus => 3, addr => 9 },
4922         'virtioscsi9' => { bus => 3, addr => 10 },
4923         'virtioscsi10' => { bus => 3, addr => 11 },
4924         'virtioscsi11' => { bus => 3, addr => 12 },
4925         'virtioscsi12' => { bus => 3, addr => 13 },
4926         'virtioscsi13' => { bus => 3, addr => 14 },
4927         'virtioscsi14' => { bus => 3, addr => 15 },
4928         'virtioscsi15' => { bus => 3, addr => 16 },
4929         'virtioscsi16' => { bus => 3, addr => 17 },
4930         'virtioscsi17' => { bus => 3, addr => 18 },
4931         'virtioscsi18' => { bus => 3, addr => 19 },
4932         'virtioscsi19' => { bus => 3, addr => 20 },
4933         'virtioscsi20' => { bus => 3, addr => 21 },
4934         'virtioscsi21' => { bus => 3, addr => 22 },
4935         'virtioscsi22' => { bus => 3, addr => 23 },
4936         'virtioscsi23' => { bus => 3, addr => 24 },
4937         'virtioscsi24' => { bus => 3, addr => 25 },
4938         'virtioscsi25' => { bus => 3, addr => 26 },
4939         'virtioscsi26' => { bus => 3, addr => 27 },
4940         'virtioscsi27' => { bus => 3, addr => 28 },
4941         'virtioscsi28' => { bus => 3, addr => 29 },
4942         'virtioscsi29' => { bus => 3, addr => 30 },
4943         'virtioscsi30' => { bus => 3, addr => 31 },
4944
4945     };
4946
4947     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4948            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4949            my $bus = $devices->{$id}->{bus};
4950            $res = ",bus=pci.$bus,addr=$addr";
4951            $bridges->{$bus} = 1 if $bridges;
4952     }
4953     return $res;
4954
4955 }
4956
4957 sub print_pcie_addr {
4958     my ($id) = @_;
4959
4960     my $res = '';
4961     my $devices = {
4962         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4963         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4964         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4965         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4966     };
4967
4968     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4969            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4970            my $bus = $devices->{$id}->{bus};
4971            $res = ",bus=$bus,addr=$addr";
4972     }
4973     return $res;
4974
4975 }
4976
4977 # vzdump restore implementaion
4978
4979 sub tar_archive_read_firstfile {
4980     my $archive = shift;
4981
4982     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4983
4984     # try to detect archive type first
4985     my $pid = open (TMP, "tar tf '$archive'|") ||
4986         die "unable to open file '$archive'\n";
4987     my $firstfile = <TMP>;
4988     kill 15, $pid;
4989     close TMP;
4990
4991     die "ERROR: archive contaions no data\n" if !$firstfile;
4992     chomp $firstfile;
4993
4994     return $firstfile;
4995 }
4996
4997 sub tar_restore_cleanup {
4998     my ($storecfg, $statfile) = @_;
4999
5000     print STDERR "starting cleanup\n";
5001
5002     if (my $fd = IO::File->new($statfile, "r")) {
5003         while (defined(my $line = <$fd>)) {
5004             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5005                 my $volid = $2;
5006                 eval {
5007                     if ($volid =~ m|^/|) {
5008                         unlink $volid || die 'unlink failed\n';
5009                     } else {
5010                         PVE::Storage::vdisk_free($storecfg, $volid);
5011                     }
5012                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5013                 };
5014                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5015             } else {
5016                 print STDERR "unable to parse line in statfile - $line";
5017             }
5018         }
5019         $fd->close();
5020     }
5021 }
5022
5023 sub restore_archive {
5024     my ($archive, $vmid, $user, $opts) = @_;
5025
5026     my $format = $opts->{format};
5027     my $comp;
5028
5029     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5030         $format = 'tar' if !$format;
5031         $comp = 'gzip';
5032     } elsif ($archive =~ m/\.tar$/) {
5033         $format = 'tar' if !$format;
5034     } elsif ($archive =~ m/.tar.lzo$/) {
5035         $format = 'tar' if !$format;
5036         $comp = 'lzop';
5037     } elsif ($archive =~ m/\.vma$/) {
5038         $format = 'vma' if !$format;
5039     } elsif ($archive =~ m/\.vma\.gz$/) {
5040         $format = 'vma' if !$format;
5041         $comp = 'gzip';
5042     } elsif ($archive =~ m/\.vma\.lzo$/) {
5043         $format = 'vma' if !$format;
5044         $comp = 'lzop';
5045     } else {
5046         $format = 'vma' if !$format; # default
5047     }
5048
5049     # try to detect archive format
5050     if ($format eq 'tar') {
5051         return restore_tar_archive($archive, $vmid, $user, $opts);
5052     } else {
5053         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5054     }
5055 }
5056
5057 sub restore_update_config_line {
5058     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5059
5060     return if $line =~ m/^\#qmdump\#/;
5061     return if $line =~ m/^\#vzdump\#/;
5062     return if $line =~ m/^lock:/;
5063     return if $line =~ m/^unused\d+:/;
5064     return if $line =~ m/^parent:/;
5065     return if $line =~ m/^template:/; # restored VM is never a template
5066
5067     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5068         # try to convert old 1.X settings
5069         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5070         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5071             my ($model, $macaddr) = split(/\=/, $devconfig);
5072             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5073             my $net = {
5074                 model => $model,
5075                 bridge => "vmbr$ind",
5076                 macaddr => $macaddr,
5077             };
5078             my $netstr = print_net($net);
5079
5080             print $outfd "net$cookie->{netcount}: $netstr\n";
5081             $cookie->{netcount}++;
5082         }
5083     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5084         my ($id, $netstr) = ($1, $2);
5085         my $net = parse_net($netstr);
5086         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5087         $netstr = print_net($net);
5088         print $outfd "$id: $netstr\n";
5089     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5090         my $virtdev = $1;
5091         my $value = $3;
5092         if ($line =~ m/backup=no/) {
5093             print $outfd "#$line";
5094         } elsif ($virtdev && $map->{$virtdev}) {
5095             my $di = parse_drive($virtdev, $value);
5096             delete $di->{format}; # format can change on restore
5097             $di->{file} = $map->{$virtdev};
5098             $value = print_drive($vmid, $di);
5099             print $outfd "$virtdev: $value\n";
5100         } else {
5101             print $outfd $line;
5102         }
5103     } else {
5104         print $outfd $line;
5105     }
5106 }
5107
5108 sub scan_volids {
5109     my ($cfg, $vmid) = @_;
5110
5111     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5112
5113     my $volid_hash = {};
5114     foreach my $storeid (keys %$info) {
5115         foreach my $item (@{$info->{$storeid}}) {
5116             next if !($item->{volid} && $item->{size});
5117             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5118             $volid_hash->{$item->{volid}} = $item;
5119         }
5120     }
5121
5122     return $volid_hash;
5123 }
5124
5125 sub is_volume_in_use {
5126     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5127
5128     my $path = PVE::Storage::path($storecfg, $volid);
5129
5130     my $scan_config = sub {
5131         my ($cref, $snapname) = @_;
5132
5133         foreach my $key (keys %$cref) {
5134             my $value = $cref->{$key};
5135             if (valid_drivename($key)) {
5136                 next if $skip_drive && $key eq $skip_drive;
5137                 my $drive = parse_drive($key, $value);
5138                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5139                 return 1 if $volid eq $drive->{file};
5140                 if ($drive->{file} =~ m!^/!) {
5141                     return 1 if $drive->{file} eq $path;
5142                 } else {
5143                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5144                     next if !$storeid;
5145                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5146                     next if !$scfg;
5147                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5148                 }
5149             }
5150         }
5151
5152         return 0;
5153     };
5154
5155     return 1 if &$scan_config($conf);
5156
5157     undef $skip_drive;
5158
5159     foreach my $snapname (keys %{$conf->{snapshots}}) {
5160         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5161     }
5162
5163     return 0;
5164 }
5165
5166 sub update_disksize {
5167     my ($vmid, $conf, $volid_hash) = @_;
5168
5169     my $changes;
5170
5171     my $used = {};
5172
5173     # Note: it is allowed to define multiple storages with same path (alias), so
5174     # we need to check both 'volid' and real 'path' (two different volid can point
5175     # to the same path).
5176
5177     my $usedpath = {};
5178
5179     # update size info
5180     foreach my $opt (keys %$conf) {
5181         if (valid_drivename($opt)) {
5182             my $drive = parse_drive($opt, $conf->{$opt});
5183             my $volid = $drive->{file};
5184             next if !$volid;
5185
5186             $used->{$volid} = 1;
5187             if ($volid_hash->{$volid} &&
5188                 (my $path = $volid_hash->{$volid}->{path})) {
5189                 $usedpath->{$path} = 1;
5190             }
5191
5192             next if drive_is_cdrom($drive);
5193             next if !$volid_hash->{$volid};
5194
5195             $drive->{size} = $volid_hash->{$volid}->{size};
5196             my $new = print_drive($vmid, $drive);
5197             if ($new ne $conf->{$opt}) {
5198                 $changes = 1;
5199                 $conf->{$opt} = $new;
5200             }
5201         }
5202     }
5203
5204     # remove 'unusedX' entry if volume is used
5205     foreach my $opt (keys %$conf) {
5206         next if $opt !~ m/^unused\d+$/;
5207         my $volid = $conf->{$opt};
5208         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5209         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5210             $changes = 1;
5211             delete $conf->{$opt};
5212         }
5213     }
5214
5215     foreach my $volid (sort keys %$volid_hash) {
5216         next if $volid =~ m/vm-$vmid-state-/;
5217         next if $used->{$volid};
5218         my $path = $volid_hash->{$volid}->{path};
5219         next if !$path; # just to be sure
5220         next if $usedpath->{$path};
5221         $changes = 1;
5222         add_unused_volume($conf, $volid);
5223         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5224     }
5225
5226     return $changes;
5227 }
5228
5229 sub rescan {
5230     my ($vmid, $nolock) = @_;
5231
5232     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5233
5234     my $volid_hash = scan_volids($cfg, $vmid);
5235
5236     my $updatefn =  sub {
5237         my ($vmid) = @_;
5238
5239         my $conf = load_config($vmid);
5240
5241         check_lock($conf);
5242
5243         my $vm_volids = {};
5244         foreach my $volid (keys %$volid_hash) {
5245             my $info = $volid_hash->{$volid};
5246             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5247         }
5248
5249         my $changes = update_disksize($vmid, $conf, $vm_volids);
5250
5251         update_config_nolock($vmid, $conf, 1) if $changes;
5252     };
5253
5254     if (defined($vmid)) {
5255         if ($nolock) {
5256             &$updatefn($vmid);
5257         } else {
5258             lock_config($vmid, $updatefn, $vmid);
5259         }
5260     } else {
5261         my $vmlist = config_list();
5262         foreach my $vmid (keys %$vmlist) {
5263             if ($nolock) {
5264                 &$updatefn($vmid);
5265             } else {
5266                 lock_config($vmid, $updatefn, $vmid);
5267             }
5268         }
5269     }
5270 }
5271
5272 sub restore_vma_archive {
5273     my ($archive, $vmid, $user, $opts, $comp) = @_;
5274
5275     my $input = $archive eq '-' ? "<&STDIN" : undef;
5276     my $readfrom = $archive;
5277
5278     my $uncomp = '';
5279     if ($comp) {
5280         $readfrom = '-';
5281         my $qarchive = PVE::Tools::shellquote($archive);
5282         if ($comp eq 'gzip') {
5283             $uncomp = "zcat $qarchive|";
5284         } elsif ($comp eq 'lzop') {
5285             $uncomp = "lzop -d -c $qarchive|";
5286         } else {
5287             die "unknown compression method '$comp'\n";
5288         }
5289
5290     }
5291
5292     my $tmpdir = "/var/tmp/vzdumptmp$$";
5293     rmtree $tmpdir;
5294
5295     # disable interrupts (always do cleanups)
5296     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5297         warn "got interrupt - ignored\n";
5298     };
5299
5300     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5301     POSIX::mkfifo($mapfifo, 0600);
5302     my $fifofh;
5303
5304     my $openfifo = sub {
5305         open($fifofh, '>', $mapfifo) || die $!;
5306     };
5307
5308     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5309
5310     my $oldtimeout;
5311     my $timeout = 5;
5312
5313     my $devinfo = {};
5314
5315     my $rpcenv = PVE::RPCEnvironment::get();
5316
5317     my $conffile = config_file($vmid);
5318     my $tmpfn = "$conffile.$$.tmp";
5319
5320     # Note: $oldconf is undef if VM does not exists
5321     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5322
5323     my $print_devmap = sub {
5324         my $virtdev_hash = {};
5325
5326         my $cfgfn = "$tmpdir/qemu-server.conf";
5327
5328         # we can read the config - that is already extracted
5329         my $fh = IO::File->new($cfgfn, "r") ||
5330             "unable to read qemu-server.conf - $!\n";
5331
5332         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5333         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5334             if -f $fwcfgfn;
5335
5336         while (defined(my $line = <$fh>)) {
5337             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5338                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5339                 die "archive does not contain data for drive '$virtdev'\n"
5340                     if !$devinfo->{$devname};
5341                 if (defined($opts->{storage})) {
5342                     $storeid = $opts->{storage} || 'local';
5343                 } elsif (!$storeid) {
5344                     $storeid = 'local';
5345                 }
5346                 $format = 'raw' if !$format;
5347                 $devinfo->{$devname}->{devname} = $devname;
5348                 $devinfo->{$devname}->{virtdev} = $virtdev;
5349                 $devinfo->{$devname}->{format} = $format;
5350                 $devinfo->{$devname}->{storeid} = $storeid;
5351
5352                 # check permission on storage
5353                 my $pool = $opts->{pool}; # todo: do we need that?
5354                 if ($user ne 'root@pam') {
5355                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5356                 }
5357
5358                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5359             }
5360         }
5361
5362         foreach my $devname (keys %$devinfo) {
5363             die "found no device mapping information for device '$devname'\n"
5364                 if !$devinfo->{$devname}->{virtdev};
5365         }
5366
5367         my $cfg = cfs_read_file('storage.cfg');
5368
5369         # create empty/temp config
5370         if ($oldconf) {
5371             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5372             foreach_drive($oldconf, sub {
5373                 my ($ds, $drive) = @_;
5374
5375                 return if drive_is_cdrom($drive);
5376
5377                 my $volid = $drive->{file};
5378
5379                 return if !$volid || $volid =~ m|^/|;
5380
5381                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5382                 return if !$path || !$owner || ($owner != $vmid);
5383
5384                 # Note: only delete disk we want to restore
5385                 # other volumes will become unused
5386                 if ($virtdev_hash->{$ds}) {
5387                     PVE::Storage::vdisk_free($cfg, $volid);
5388                 }
5389             });
5390         }
5391
5392         my $map = {};
5393         foreach my $virtdev (sort keys %$virtdev_hash) {
5394             my $d = $virtdev_hash->{$virtdev};
5395             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5396             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5397
5398             # test if requested format is supported
5399             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5400             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5401             $d->{format} = $defFormat if !$supported;
5402
5403             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5404                                                   $d->{format}, undef, $alloc_size);
5405             print STDERR "new volume ID is '$volid'\n";
5406             $d->{volid} = $volid;
5407             my $path = PVE::Storage::path($cfg, $volid);
5408
5409             PVE::Storage::activate_volumes($cfg,[$volid]);
5410
5411             my $write_zeros = 1;
5412             # fixme: what other storages types initialize volumes with zero?
5413             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5414                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5415                 $write_zeros = 0;
5416             }
5417
5418             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5419
5420             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5421             $map->{$virtdev} = $volid;
5422         }
5423
5424         $fh->seek(0, 0) || die "seek failed - $!\n";
5425
5426         my $outfd = new IO::File ($tmpfn, "w") ||
5427             die "unable to write config for VM $vmid\n";
5428
5429         my $cookie = { netcount => 0 };
5430         while (defined(my $line = <$fh>)) {
5431             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5432         }
5433
5434         $fh->close();
5435         $outfd->close();
5436     };
5437
5438     eval {
5439         # enable interrupts
5440         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5441             die "interrupted by signal\n";
5442         };
5443         local $SIG{ALRM} = sub { die "got timeout\n"; };
5444
5445         $oldtimeout = alarm($timeout);
5446
5447         my $parser = sub {
5448             my $line = shift;
5449
5450             print "$line\n";
5451
5452             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5453                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5454                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5455             } elsif ($line =~ m/^CTIME: /) {
5456                 # we correctly received the vma config, so we can disable
5457                 # the timeout now for disk allocation (set to 10 minutes, so
5458                 # that we always timeout if something goes wrong)
5459                 alarm(600);
5460                 &$print_devmap();
5461                 print $fifofh "done\n";
5462                 my $tmp = $oldtimeout || 0;
5463                 $oldtimeout = undef;
5464                 alarm($tmp);
5465                 close($fifofh);
5466             }
5467         };
5468
5469         print "restore vma archive: $cmd\n";
5470         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5471     };
5472     my $err = $@;
5473
5474     alarm($oldtimeout) if $oldtimeout;
5475
5476     my $vollist = [];
5477     foreach my $devname (keys %$devinfo) {
5478         my $volid = $devinfo->{$devname}->{volid};
5479         push @$vollist, $volid if $volid;
5480     }
5481
5482     my $cfg = cfs_read_file('storage.cfg');
5483     PVE::Storage::deactivate_volumes($cfg, $vollist);
5484
5485     unlink $mapfifo;
5486
5487     if ($err) {
5488         rmtree $tmpdir;
5489         unlink $tmpfn;
5490
5491         foreach my $devname (keys %$devinfo) {
5492             my $volid = $devinfo->{$devname}->{volid};
5493             next if !$volid;
5494             eval {
5495                 if ($volid =~ m|^/|) {
5496                     unlink $volid || die 'unlink failed\n';
5497                 } else {
5498                     PVE::Storage::vdisk_free($cfg, $volid);
5499                 }
5500                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5501             };
5502             print STDERR "unable to cleanup '$volid' - $@" if $@;
5503         }
5504         die $err;
5505     }
5506
5507     rmtree $tmpdir;
5508
5509     rename($tmpfn, $conffile) ||
5510         die "unable to commit configuration file '$conffile'\n";
5511
5512     PVE::Cluster::cfs_update(); # make sure we read new file
5513
5514     eval { rescan($vmid, 1); };
5515     warn $@ if $@;
5516 }
5517
5518 sub restore_tar_archive {
5519     my ($archive, $vmid, $user, $opts) = @_;
5520
5521     if ($archive ne '-') {
5522         my $firstfile = tar_archive_read_firstfile($archive);
5523         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5524             if $firstfile ne 'qemu-server.conf';
5525     }
5526
5527     my $storecfg = cfs_read_file('storage.cfg');
5528
5529     # destroy existing data - keep empty config
5530     my $vmcfgfn = config_file($vmid);
5531     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5532
5533     my $tocmd = "/usr/lib/qemu-server/qmextract";
5534
5535     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5536     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5537     $tocmd .= ' --prealloc' if $opts->{prealloc};
5538     $tocmd .= ' --info' if $opts->{info};
5539
5540     # tar option "xf" does not autodetect compression when read from STDIN,
5541     # so we pipe to zcat
5542     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5543         PVE::Tools::shellquote("--to-command=$tocmd");
5544
5545     my $tmpdir = "/var/tmp/vzdumptmp$$";
5546     mkpath $tmpdir;
5547
5548     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5549     local $ENV{VZDUMP_VMID} = $vmid;
5550     local $ENV{VZDUMP_USER} = $user;
5551
5552     my $conffile = config_file($vmid);
5553     my $tmpfn = "$conffile.$$.tmp";
5554
5555     # disable interrupts (always do cleanups)
5556     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5557         print STDERR "got interrupt - ignored\n";
5558     };
5559
5560     eval {
5561         # enable interrupts
5562         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5563             die "interrupted by signal\n";
5564         };
5565
5566         if ($archive eq '-') {
5567             print "extracting archive from STDIN\n";
5568             run_command($cmd, input => "<&STDIN");
5569         } else {
5570             print "extracting archive '$archive'\n";
5571             run_command($cmd);
5572         }
5573
5574         return if $opts->{info};
5575
5576         # read new mapping
5577         my $map = {};
5578         my $statfile = "$tmpdir/qmrestore.stat";
5579         if (my $fd = IO::File->new($statfile, "r")) {
5580             while (defined (my $line = <$fd>)) {
5581                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5582                     $map->{$1} = $2 if $1;
5583                 } else {
5584                     print STDERR "unable to parse line in statfile - $line\n";
5585                 }
5586             }
5587             $fd->close();
5588         }
5589
5590         my $confsrc = "$tmpdir/qemu-server.conf";
5591
5592         my $srcfd = new IO::File($confsrc, "r") ||
5593             die "unable to open file '$confsrc'\n";
5594
5595         my $outfd = new IO::File ($tmpfn, "w") ||
5596             die "unable to write config for VM $vmid\n";
5597
5598         my $cookie = { netcount => 0 };
5599         while (defined (my $line = <$srcfd>)) {
5600             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5601         }
5602
5603         $srcfd->close();
5604         $outfd->close();
5605     };
5606     my $err = $@;
5607
5608     if ($err) {
5609
5610         unlink $tmpfn;
5611
5612         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5613
5614         die $err;
5615     }
5616
5617     rmtree $tmpdir;
5618
5619     rename $tmpfn, $conffile ||
5620         die "unable to commit configuration file '$conffile'\n";
5621
5622     PVE::Cluster::cfs_update(); # make sure we read new file
5623
5624     eval { rescan($vmid, 1); };
5625     warn $@ if $@;
5626 };
5627
5628
5629 # Internal snapshots
5630
5631 # NOTE: Snapshot create/delete involves several non-atomic
5632 # action, and can take a long time.
5633 # So we try to avoid locking the file and use 'lock' variable
5634 # inside the config file instead.
5635
5636 my $snapshot_copy_config = sub {
5637     my ($source, $dest) = @_;
5638
5639     foreach my $k (keys %$source) {
5640         next if $k eq 'snapshots';
5641         next if $k eq 'snapstate';
5642         next if $k eq 'snaptime';
5643         next if $k eq 'vmstate';
5644         next if $k eq 'lock';
5645         next if $k eq 'digest';
5646         next if $k eq 'description';
5647         next if $k =~ m/^unused\d+$/;
5648
5649         $dest->{$k} = $source->{$k};
5650     }
5651 };
5652
5653 my $snapshot_apply_config = sub {
5654     my ($conf, $snap) = @_;
5655
5656     # copy snapshot list
5657     my $newconf = {
5658         snapshots => $conf->{snapshots},
5659     };
5660
5661     # keep description and list of unused disks
5662     foreach my $k (keys %$conf) {
5663         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5664         $newconf->{$k} = $conf->{$k};
5665     }
5666
5667     &$snapshot_copy_config($snap, $newconf);
5668
5669     return $newconf;
5670 };
5671
5672 sub foreach_writable_storage {
5673     my ($conf, $func) = @_;
5674
5675     my $sidhash = {};
5676
5677     foreach my $ds (keys %$conf) {
5678         next if !valid_drivename($ds);
5679
5680         my $drive = parse_drive($ds, $conf->{$ds});
5681         next if !$drive;
5682         next if drive_is_cdrom($drive);
5683
5684         my $volid = $drive->{file};
5685
5686         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5687         $sidhash->{$sid} = $sid if $sid;
5688     }
5689
5690     foreach my $sid (sort keys %$sidhash) {
5691         &$func($sid);
5692     }
5693 }
5694
5695 my $alloc_vmstate_volid = sub {
5696     my ($storecfg, $vmid, $conf, $snapname) = @_;
5697
5698     # Note: we try to be smart when selecting a $target storage
5699
5700     my $target;
5701
5702     # search shared storage first
5703     foreach_writable_storage($conf, sub {
5704         my ($sid) = @_;
5705         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5706         return if !$scfg->{shared};
5707
5708         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5709     });
5710
5711     if (!$target) {
5712         # now search local storage
5713         foreach_writable_storage($conf, sub {
5714             my ($sid) = @_;
5715             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5716             return if $scfg->{shared};
5717
5718             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5719         });
5720     }
5721
5722     $target = 'local' if !$target;
5723
5724     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5725     # we abort live save after $conf->{memory}, so we need at max twice that space
5726     my $size = $conf->{memory}*2 + $driver_state_size;
5727
5728     my $name = "vm-$vmid-state-$snapname";
5729     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5730     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5731     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5732
5733     return $volid;
5734 };
5735
5736 my $snapshot_prepare = sub {
5737     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5738
5739     my $snap;
5740
5741     my $updatefn =  sub {
5742
5743         my $conf = load_config($vmid);
5744
5745         die "you can't take a snapshot if it's a template\n"
5746             if is_template($conf);
5747
5748         check_lock($conf);
5749
5750         $conf->{lock} = 'snapshot';
5751
5752         die "snapshot name '$snapname' already used\n"
5753             if defined($conf->{snapshots}->{$snapname});
5754
5755         my $storecfg = PVE::Storage::config();
5756         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5757
5758         $snap = $conf->{snapshots}->{$snapname} = {};
5759
5760         if ($save_vmstate && check_running($vmid)) {
5761             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5762         }
5763
5764         &$snapshot_copy_config($conf, $snap);
5765
5766         $snap->{snapstate} = "prepare";
5767         $snap->{snaptime} = time();
5768         $snap->{description} = $comment if $comment;
5769
5770         # always overwrite machine if we save vmstate. This makes sure we
5771         # can restore it later using correct machine type
5772         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5773
5774         update_config_nolock($vmid, $conf, 1);
5775     };
5776
5777     lock_config($vmid, $updatefn);
5778
5779     return $snap;
5780 };
5781
5782 my $snapshot_commit = sub {
5783     my ($vmid, $snapname) = @_;
5784
5785     my $updatefn = sub {
5786
5787         my $conf = load_config($vmid);
5788
5789         die "missing snapshot lock\n"
5790             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5791
5792         my $has_machine_config = defined($conf->{machine});
5793
5794         my $snap = $conf->{snapshots}->{$snapname};
5795
5796         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5797
5798         die "wrong snapshot state\n"
5799             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5800
5801         delete $snap->{snapstate};
5802         delete $conf->{lock};
5803
5804         my $newconf = &$snapshot_apply_config($conf, $snap);
5805
5806         delete $newconf->{machine} if !$has_machine_config;
5807
5808         $newconf->{parent} = $snapname;
5809
5810         update_config_nolock($vmid, $newconf, 1);
5811     };
5812
5813     lock_config($vmid, $updatefn);
5814 };
5815
5816 sub snapshot_rollback {
5817     my ($vmid, $snapname) = @_;
5818
5819     my $prepare = 1;
5820
5821     my $storecfg = PVE::Storage::config();
5822
5823     my $conf = load_config($vmid);
5824
5825     my $get_snapshot_config = sub {
5826
5827         die "you can't rollback if vm is a template\n" if is_template($conf);
5828
5829         my $res = $conf->{snapshots}->{$snapname};
5830
5831         die "snapshot '$snapname' does not exist\n" if !defined($res);
5832
5833         return $res;
5834     };
5835
5836     my $snap = &$get_snapshot_config();
5837
5838     foreach_drive($snap, sub {
5839         my ($ds, $drive) = @_;
5840
5841         return if drive_is_cdrom($drive);
5842
5843         my $volid = $drive->{file};
5844
5845         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5846     });
5847
5848     my $updatefn = sub {
5849
5850         $conf = load_config($vmid);
5851
5852         $snap = &$get_snapshot_config();
5853
5854         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5855             if $snap->{snapstate};
5856
5857         if ($prepare) {
5858             check_lock($conf);
5859             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5860         }
5861
5862         die "unable to rollback vm $vmid: vm is running\n"
5863             if check_running($vmid);
5864
5865         if ($prepare) {
5866             $conf->{lock} = 'rollback';
5867         } else {
5868             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5869             delete $conf->{lock};
5870         }
5871
5872         my $forcemachine;
5873
5874         if (!$prepare) {
5875             my $has_machine_config = defined($conf->{machine});
5876
5877             # copy snapshot config to current config
5878             $conf = &$snapshot_apply_config($conf, $snap);
5879             $conf->{parent} = $snapname;
5880
5881             # Note: old code did not store 'machine', so we try to be smart
5882             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5883             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5884             # we remove the 'machine' configuration if not explicitly specified
5885             # in the original config.
5886             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5887         }
5888
5889         update_config_nolock($vmid, $conf, 1);
5890
5891         if (!$prepare && $snap->{vmstate}) {
5892             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5893             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5894         }
5895     };
5896
5897     lock_config($vmid, $updatefn);
5898
5899     foreach_drive($snap, sub {
5900         my ($ds, $drive) = @_;
5901
5902         return if drive_is_cdrom($drive);
5903
5904         my $volid = $drive->{file};
5905         my $device = "drive-$ds";
5906
5907         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5908     });
5909
5910     $prepare = 0;
5911     lock_config($vmid, $updatefn);
5912 }
5913
5914 my $savevm_wait = sub {
5915     my ($vmid) = @_;
5916
5917     for(;;) {
5918         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5919         if (!$stat->{status}) {
5920             die "savevm not active\n";
5921         } elsif ($stat->{status} eq 'active') {
5922             sleep(1);
5923             next;
5924         } elsif ($stat->{status} eq 'completed') {
5925             last;
5926         } else {
5927             die "query-savevm returned status '$stat->{status}'\n";
5928         }
5929     }
5930 };
5931
5932 sub do_snapshots_with_qemu {
5933     my ($storecfg, $volid) = @_;
5934
5935     my $storage_name = PVE::Storage::parse_volume_id($volid);
5936
5937     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5938         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5939         return 1;
5940     }
5941
5942     if ($volid =~ m/\.(qcow2|qed)$/){
5943         return 1;
5944     }
5945
5946     return undef;
5947 }
5948
5949 sub snapshot_create {
5950     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5951
5952     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5953
5954     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5955
5956     my $config = load_config($vmid);
5957
5958     my $running = check_running($vmid);
5959
5960     my $freezefs = $running && $config->{agent};
5961     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5962
5963     my $drivehash = {};
5964
5965     if ($freezefs) {
5966         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5967         warn "guest-fsfreeze-freeze problems - $@" if $@;
5968     }
5969
5970     eval {
5971         # create internal snapshots of all drives
5972
5973         my $storecfg = PVE::Storage::config();
5974
5975         if ($running) {
5976             if ($snap->{vmstate}) {
5977                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5978                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5979                 &$savevm_wait($vmid);
5980             } else {
5981                 vm_mon_cmd($vmid, "savevm-start");
5982             }
5983         };
5984
5985         foreach_drive($snap, sub {
5986             my ($ds, $drive) = @_;
5987
5988             return if drive_is_cdrom($drive);
5989
5990             my $volid = $drive->{file};
5991             my $device = "drive-$ds";
5992
5993             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5994             $drivehash->{$ds} = 1;
5995        });
5996     };
5997     my $err = $@;
5998
5999     if ($running) {
6000         eval { vm_mon_cmd($vmid, "savevm-end")  };
6001         warn $@ if $@;
6002
6003         if ($freezefs) {
6004             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6005             warn "guest-fsfreeze-thaw problems - $@" if $@;
6006         }
6007
6008         # savevm-end is async, we need to wait
6009         for (;;) {
6010             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6011             if (!$stat->{bytes}) {
6012                 last;
6013             } else {
6014                 print "savevm not yet finished\n";
6015                 sleep(1);
6016                 next;
6017             }
6018         }
6019     }
6020
6021     if ($err) {
6022         warn "snapshot create failed: starting cleanup\n";
6023         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6024         warn $@ if $@;
6025         die $err;
6026     }
6027
6028     &$snapshot_commit($vmid, $snapname);
6029 }
6030
6031 # Note: $drivehash is only set when called from snapshot_create.
6032 sub snapshot_delete {
6033     my ($vmid, $snapname, $force, $drivehash) = @_;
6034
6035     my $prepare = 1;
6036
6037     my $snap;
6038     my $unused = [];
6039
6040     my $unlink_parent = sub {
6041         my ($confref, $new_parent) = @_;
6042
6043         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6044             if ($new_parent) {
6045                 $confref->{parent} = $new_parent;
6046             } else {
6047                 delete $confref->{parent};
6048             }
6049         }
6050     };
6051
6052     my $updatefn =  sub {
6053         my ($remove_drive) = @_;
6054
6055         my $conf = load_config($vmid);
6056
6057         if (!$drivehash) {
6058             check_lock($conf);
6059             die "you can't delete a snapshot if vm is a template\n"
6060                 if is_template($conf);
6061         }
6062
6063         $snap = $conf->{snapshots}->{$snapname};
6064
6065         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6066
6067         # remove parent refs
6068         if (!$prepare) {
6069             &$unlink_parent($conf, $snap->{parent});
6070             foreach my $sn (keys %{$conf->{snapshots}}) {
6071                 next if $sn eq $snapname;
6072                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6073             }
6074         }
6075
6076         if ($remove_drive) {
6077             if ($remove_drive eq 'vmstate') {
6078                 delete $snap->{$remove_drive};
6079             } else {
6080                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6081                 my $volid = $drive->{file};
6082                 delete $snap->{$remove_drive};
6083                 add_unused_volume($conf, $volid);
6084             }
6085         }
6086
6087         if ($prepare) {
6088             $snap->{snapstate} = 'delete';
6089         } else {
6090             delete $conf->{snapshots}->{$snapname};
6091             delete $conf->{lock} if $drivehash;
6092             foreach my $volid (@$unused) {
6093                 add_unused_volume($conf, $volid);
6094             }
6095         }
6096
6097         update_config_nolock($vmid, $conf, 1);
6098     };
6099
6100     lock_config($vmid, $updatefn);
6101
6102     # now remove vmstate file
6103
6104     my $storecfg = PVE::Storage::config();
6105
6106     if ($snap->{vmstate}) {
6107         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6108         if (my $err = $@) {
6109             die $err if !$force;
6110             warn $err;
6111         }
6112         # save changes (remove vmstate from snapshot)
6113         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6114     };
6115
6116     # now remove all internal snapshots
6117     foreach_drive($snap, sub {
6118         my ($ds, $drive) = @_;
6119
6120         return if drive_is_cdrom($drive);
6121
6122         my $volid = $drive->{file};
6123         my $device = "drive-$ds";
6124
6125         if (!$drivehash || $drivehash->{$ds}) {
6126             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6127             if (my $err = $@) {
6128                 die $err if !$force;
6129                 warn $err;
6130             }
6131         }
6132
6133         # save changes (remove drive fron snapshot)
6134         lock_config($vmid, $updatefn, $ds) if !$force;
6135         push @$unused, $volid;
6136     });
6137
6138     # now cleanup config
6139     $prepare = 0;
6140     lock_config($vmid, $updatefn);
6141 }
6142
6143 sub has_feature {
6144     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6145
6146     my $err;
6147     foreach_drive($conf, sub {
6148         my ($ds, $drive) = @_;
6149
6150         return if drive_is_cdrom($drive);
6151         my $volid = $drive->{file};
6152         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6153     });
6154
6155     return $err ? 0 : 1;
6156 }
6157
6158 sub template_create {
6159     my ($vmid, $conf, $disk) = @_;
6160
6161     my $storecfg = PVE::Storage::config();
6162
6163     foreach_drive($conf, sub {
6164         my ($ds, $drive) = @_;
6165
6166         return if drive_is_cdrom($drive);
6167         return if $disk && $ds ne $disk;
6168
6169         my $volid = $drive->{file};
6170         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6171
6172         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6173         $drive->{file} = $voliddst;
6174         $conf->{$ds} = print_drive($vmid, $drive);
6175         update_config_nolock($vmid, $conf, 1);
6176     });
6177 }
6178
6179 sub is_template {
6180     my ($conf) = @_;
6181
6182     return 1 if defined $conf->{template} && $conf->{template} == 1;
6183 }
6184
6185 sub qemu_img_convert {
6186     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6187
6188     my $storecfg = PVE::Storage::config();
6189     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6190     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6191
6192     if ($src_storeid && $dst_storeid) {
6193
6194         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6195
6196         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6197         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6198
6199         my $src_format = qemu_img_format($src_scfg, $src_volname);
6200         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6201
6202         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6203         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6204
6205         my $cmd = [];
6206         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6207         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6208         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6209
6210         my $parser = sub {
6211             my $line = shift;
6212             if($line =~ m/\((\S+)\/100\%\)/){
6213                 my $percent = $1;
6214                 my $transferred = int($size * $percent / 100);
6215                 my $remaining = $size - $transferred;
6216
6217                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6218             }
6219
6220         };
6221
6222         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6223         my $err = $@;
6224         die "copy failed: $err" if $err;
6225     }
6226 }
6227
6228 sub qemu_img_format {
6229     my ($scfg, $volname) = @_;
6230
6231     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6232         return $1;
6233     } else {
6234         return "raw";
6235     }
6236 }
6237
6238 sub qemu_drive_mirror {
6239     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6240
6241     my $storecfg = PVE::Storage::config();
6242     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6243
6244     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6245
6246     my $format = qemu_img_format($dst_scfg, $dst_volname);
6247
6248     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6249
6250     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6251     $opts->{format} = $format if $format;
6252
6253     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6254
6255     eval {
6256     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6257         while (1) {
6258             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6259             my $stat = @$stats[0];
6260             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6261             die "error job is not mirroring" if $stat->{type} ne "mirror";
6262
6263             my $busy = $stat->{busy};
6264             my $ready = $stat->{ready};
6265
6266             if (my $total = $stat->{len}) {
6267                 my $transferred = $stat->{offset} || 0;
6268                 my $remaining = $total - $transferred;
6269                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6270
6271                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6272             }
6273
6274
6275             if ($stat->{ready} eq 'true') {
6276
6277                 last if $vmiddst != $vmid;
6278
6279                 # try to switch the disk if source and destination are on the same guest
6280                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6281                 last if !$@;
6282                 die $@ if $@ !~ m/cannot be completed/;
6283             }
6284             sleep 1;
6285         }
6286
6287
6288     };
6289     my $err = $@;
6290
6291     my $cancel_job = sub {
6292         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6293         while (1) {
6294             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6295             my $stat = @$stats[0];
6296             last if !$stat;
6297             sleep 1;
6298         }
6299     };
6300
6301     if ($err) {
6302         eval { &$cancel_job(); };
6303         die "mirroring error: $err";
6304     }
6305
6306     if ($vmiddst != $vmid) {
6307         # if we clone a disk for a new target vm, we don't switch the disk
6308         &$cancel_job(); # so we call block-job-cancel
6309     }
6310 }
6311
6312 sub clone_disk {
6313     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6314         $newvmid, $storage, $format, $full, $newvollist) = @_;
6315
6316     my $newvolid;
6317
6318     if (!$full) {
6319         print "create linked clone of drive $drivename ($drive->{file})\n";
6320         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6321         push @$newvollist, $newvolid;
6322     } else {
6323         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6324         $storeid = $storage if $storage;
6325
6326         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6327         if (!$format) {
6328             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6329             $format = qemu_img_format($scfg, $volname);
6330         }
6331
6332         # test if requested format is supported - else use default
6333         my $supported = grep { $_ eq $format } @$validFormats;
6334         $format = $defFormat if !$supported;
6335
6336         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6337
6338         print "create full clone of drive $drivename ($drive->{file})\n";
6339         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6340         push @$newvollist, $newvolid;
6341
6342         PVE::Storage::activate_volumes($storecfg, $newvollist);
6343
6344         if (!$running || $snapname) {
6345             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6346         } else {
6347             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6348         }
6349     }
6350
6351     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6352
6353     my $disk = $drive;
6354     $disk->{format} = undef;
6355     $disk->{file} = $newvolid;
6356     $disk->{size} = $size;
6357
6358     return $disk;
6359 }
6360
6361 # this only works if VM is running
6362 sub get_current_qemu_machine {
6363     my ($vmid) = @_;
6364
6365     my $cmd = { execute => 'query-machines', arguments => {} };
6366     my $res = vm_qmp_command($vmid, $cmd);
6367
6368     my ($current, $default);
6369     foreach my $e (@$res) {
6370         $default = $e->{name} if $e->{'is-default'};
6371         $current = $e->{name} if $e->{'is-current'};
6372     }
6373
6374     # fallback to the default machine if current is not supported by qemu
6375     return $current || $default || 'pc';
6376 }
6377
6378 sub qemu_machine_feature_enabled {
6379     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6380
6381     my $current_major;
6382     my $current_minor;
6383
6384     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6385
6386         $current_major = $3;
6387         $current_minor = $4;
6388
6389     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6390
6391         $current_major = $1;
6392         $current_minor = $2;
6393     }
6394
6395     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6396
6397
6398 }
6399
6400 sub qemu_machine_pxe {
6401     my ($vmid, $conf, $machine) = @_;
6402
6403     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6404
6405     foreach my $opt (keys %$conf) {
6406         next if $opt !~ m/^net(\d+)$/;
6407         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6408         next if !$net;
6409         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6410         return $machine.".pxe" if $romfile =~ m/pxe/;
6411         last;
6412     }
6413
6414 }
6415
6416 sub qemu_use_old_bios_files {
6417     my ($machine_type) = @_;
6418
6419     return if !$machine_type;
6420
6421     my $use_old_bios_files = undef;
6422
6423     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6424         $machine_type = $1;
6425         $use_old_bios_files = 1;
6426     } else {
6427         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6428         # load new efi bios files on migration. So this hack is required to allow
6429         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6430         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6431         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6432     }
6433
6434     return ($use_old_bios_files, $machine_type);
6435 }
6436
6437 sub lspci {
6438
6439     my $devices = {};
6440
6441     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6442             my (undef, $id, $function) = @_;
6443             my $res = { id => $id, function => $function};
6444             push @{$devices->{$id}}, $res;
6445     });
6446
6447     return $devices;
6448 }
6449
6450 sub vm_iothreads_list {
6451     my ($vmid) = @_;
6452
6453     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6454
6455     my $iothreads = {};
6456     foreach my $iothread (@$res) {
6457         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6458     }
6459
6460     return $iothreads;
6461 }
6462
6463 sub scsihw_infos {
6464     my ($conf, $drive) = @_;
6465
6466     my $maxdev = 0;
6467
6468     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6469         $maxdev = 7;
6470     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6471         $maxdev = 1;
6472     } else {
6473         $maxdev = 256;
6474     }
6475
6476     my $controller = int($drive->{index} / $maxdev);
6477     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6478
6479     return ($maxdev, $controller, $controller_prefix);
6480 }
6481
6482 # bash completion helper
6483
6484 sub complete_backup_archives {
6485     my ($cmdname, $pname, $cvalue) = @_;
6486
6487     my $cfg = PVE::Storage::config();
6488
6489     my $storeid;
6490
6491     if ($cvalue =~ m/^([^:]+):/) {
6492         $storeid = $1;
6493     }
6494
6495     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6496
6497     my $res = [];
6498     foreach my $id (keys %$data) {
6499         foreach my $item (@{$data->{$id}}) {
6500             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6501             push @$res, $item->{volid} if defined($item->{volid});
6502         }
6503     }
6504
6505     return $res;
6506 }
6507
6508 my $complete_vmid_full = sub {
6509     my ($running) = @_;
6510
6511     my $idlist = vmstatus();
6512
6513     my $res = [];
6514
6515     foreach my $id (keys %$idlist) {
6516         my $d = $idlist->{$id};
6517         if (defined($running)) {
6518             next if $d->{template};
6519             next if $running && $d->{status} ne 'running';
6520             next if !$running && $d->{status} eq 'running';
6521         }
6522         push @$res, $id;
6523
6524     }
6525     return $res;
6526 };
6527
6528 sub complete_vmid {
6529     return &$complete_vmid_full();
6530 }
6531
6532 sub complete_vmid_stopped {
6533     return &$complete_vmid_full(0);
6534 }
6535
6536 sub complete_vmid_running {
6537     return &$complete_vmid_full(1);
6538 }
6539
6540 sub complete_storage {
6541
6542     my $cfg = PVE::Storage::config();
6543     my $ids = $cfg->{ids};
6544
6545     my $res = [];
6546     foreach my $sid (keys %$ids) {
6547         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6548         next if !$ids->{$sid}->{content}->{images};
6549         push @$res, $sid;
6550     }
6551
6552     return $res;
6553 }
6554
6555 1;