PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     return undef if $res->{cache} &&
 917         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 918     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 919     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 920     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 921     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 922     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 923     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 924     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 925     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 926     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 927     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 928     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 929     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 930     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 931     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 932
 933     return undef if $res->{mbps_rd} && $res->{mbps};
 934     return undef if $res->{mbps_wr} && $res->{mbps};
 935
 936     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 937     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 938     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 939     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 942
 943     return undef if $res->{iops_rd} && $res->{iops};
 944     return undef if $res->{iops_wr} && $res->{iops};
 945
 946
 947     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 948     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 949     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 950     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 951     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 952     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 953
 954
 955     if ($res->{size}) {
 956         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 957     }
 958
 959     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 960         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 961         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 962         return undef if $res->{interface} eq 'virtio';
 963     }
 964
 965     # rerror does not work with scsi drives
 966     if ($res->{rerror}) {
 967         return undef if $res->{interface} eq 'scsi';
 968     }
 969
 970     return $res;
 971 }
 972
 973 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 974
 975 sub print_drive {
 976     my ($vmid, $drive) = @_;
 977
 978     my $opts = '';
 979     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 980         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 981     }
 982
 983     if ($drive->{size}) {
 984         $opts .= ",size=" . &$format_size($drive->{size});
 985     }
 986
 987     return "$drive->{file}$opts";
 988 }
 989
 990 sub scsi_inquiry {
 991     my($fh, $noerr) = @_;
 992
 993     my $SG_IO = 0x2285;
 994     my $SG_GET_VERSION_NUM = 0x2282;
 995
 996     my $versionbuf = "\x00" x 8;
 997     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
 998     if (!$ret) {
 999         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1000         return undef;
1001     }
1002     my $version = unpack("I", $versionbuf);
1003     if ($version < 30000) {
1004         die "scsi generic interface too old\n"  if !$noerr;
1005         return undef;
1006     }
1007
1008     my $buf = "\x00" x 36;
1009     my $sensebuf = "\x00" x 8;
1010     my $cmd = pack("C x3 C x1", 0x12, 36);
1011
1012     # see /usr/include/scsi/sg.h
1013     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1014
1015     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1016                       length($sensebuf), 0, length($buf), $buf,
1017                       $cmd, $sensebuf, 6000);
1018
1019     $ret = ioctl($fh, $SG_IO, $packet);
1020     if (!$ret) {
1021         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024
1025     my @res = unpack($sg_io_hdr_t, $packet);
1026     if ($res[17] || $res[18]) {
1027         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my $res = {};
1032     (my $byte0, my $byte1, $res->{vendor},
1033      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1034
1035     $res->{removable} = $byte1 & 128 ? 1 : 0;
1036     $res->{type} = $byte0 & 31;
1037
1038     return $res;
1039 }
1040
1041 sub path_is_scsi {
1042     my ($path) = @_;
1043
1044     my $fh = IO::File->new("+<$path") || return undef;
1045     my $res = scsi_inquiry($fh, 1);
1046     close($fh);
1047
1048     return $res;
1049 }
1050
1051 sub machine_type_is_q35 {
1052     my ($conf) = @_;
1053
1054     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1055 }
1056
1057 sub print_tabletdevice_full {
1058     my ($conf) = @_;
1059
1060     my $q35 = machine_type_is_q35($conf);
1061
1062     # we use uhci for old VMs because tablet driver was buggy in older qemu
1063     my $usbbus = $q35 ? "ehci" : "uhci";
1064
1065     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1066 }
1067
1068 sub print_drivedevice_full {
1069     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1070
1071     my $device = '';
1072     my $maxdev = 0;
1073
1074     if ($drive->{interface} eq 'virtio') {
1075         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1076         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1077         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1078     } elsif ($drive->{interface} eq 'scsi') {
1079
1080         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1081         my $unit = $drive->{index} % $maxdev;
1082         my $devicetype = 'hd';
1083         my $path = '';
1084         if (drive_is_cdrom($drive)) {
1085               $devicetype = 'cd';
1086         } else {
1087               if ($drive->{file} =~ m|^/|) {
1088                   $path = $drive->{file};
1089               } else {
1090                   $path = PVE::Storage::path($storecfg, $drive->{file});
1091               }
1092
1093               if($path =~ m/^iscsi\:\/\//){
1094                   $devicetype = 'generic';
1095               } else {
1096                   if (my $info = path_is_scsi($path)) {
1097                       if ($info->{type} == 0) {
1098                           $devicetype = 'block';
1099                       } elsif ($info->{type} == 1) { # tape
1100                           $devicetype = 'generic';
1101                       }
1102                   }
1103               }
1104          }
1105
1106         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1107             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1108         } else {
1109             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1110         }
1111
1112     } elsif ($drive->{interface} eq 'ide'){
1113         $maxdev = 2;
1114         my $controller = int($drive->{index} / $maxdev);
1115         my $unit = $drive->{index} % $maxdev;
1116         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1117
1118         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1119     } elsif ($drive->{interface} eq 'sata'){
1120         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1121         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1122         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'usb') {
1124         die "implement me";
1125         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1126     } else {
1127         die "unsupported interface type";
1128     }
1129
1130     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1131
1132     return $device;
1133 }
1134
1135 sub get_initiator_name {
1136     my $initiator;
1137
1138     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1139     while (defined(my $line = <$fh>)) {
1140         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1141         $initiator = $1;
1142         last;
1143     }
1144     $fh->close();
1145
1146     return $initiator;
1147 }
1148
1149 sub print_drive_full {
1150     my ($storecfg, $vmid, $drive) = @_;
1151
1152     my $path;
1153     my $volid = $drive->{file};
1154     my $format;
1155
1156     if (drive_is_cdrom($drive)) {
1157         $path = get_iso_path($storecfg, $vmid, $volid);
1158     } else {
1159         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1160         if ($storeid) {
1161             $path = PVE::Storage::path($storecfg, $volid);
1162             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1163             $format = qemu_img_format($scfg, $volname);
1164         } else {
1165             $path = $volid;
1166         }
1167    }
1168
1169     my $opts = '';
1170     foreach my $o (@qemu_drive_options) {
1171         next if $o eq 'bootindex';
1172         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1173     }
1174
1175     $opts .= ",format=$format" if $format && !$drive->{format};
1176
1177     foreach my $o (qw(bps bps_rd bps_wr)) {
1178         my $v = $drive->{"m$o"};
1179         $opts .= ",$o=" . int($v*1024*1024) if $v;
1180     }
1181
1182     my $cache_direct = 0;
1183
1184     if (my $cache = $drive->{cache}) {
1185         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1186     } elsif (!drive_is_cdrom($drive)) {
1187         $opts .= ",cache=none";
1188         $cache_direct = 1;
1189     }
1190
1191     # aio native works only with O_DIRECT
1192     if (!$drive->{aio}) {
1193         if($cache_direct) {
1194             $opts .= ",aio=native";
1195         } else {
1196             $opts .= ",aio=threads";
1197         }
1198     }
1199
1200     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1201     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1202
1203     my $pathinfo = $path ? "file=$path," : '';
1204
1205     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1206 }
1207
1208 sub print_netdevice_full {
1209     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1210
1211     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1212
1213     my $device = $net->{model};
1214     if ($net->{model} eq 'virtio') {
1215          $device = 'virtio-net-pci';
1216      };
1217
1218     my $pciaddr = print_pci_addr("$netid", $bridges);
1219     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1220     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1221         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1222         my $vectors = $net->{queues} * 2 + 2;
1223         $tmpstr .= ",vectors=$vectors,mq=on";
1224     }
1225     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1226     return $tmpstr;
1227 }
1228
1229 sub print_netdev_full {
1230     my ($vmid, $conf, $net, $netid) = @_;
1231
1232     my $i = '';
1233     if ($netid =~ m/^net(\d+)$/) {
1234         $i = int($1);
1235     }
1236
1237     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1238
1239     my $ifname = "tap${vmid}i$i";
1240
1241     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1242     die "interface name '$ifname' is too long (max 15 character)\n"
1243         if length($ifname) >= 16;
1244
1245     my $vhostparam = '';
1246     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1247
1248     my $vmname = $conf->{name} || "vm$vmid";
1249
1250     my $netdev = "";
1251
1252     if ($net->{bridge}) {
1253         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1254     } else {
1255         $netdev = "type=user,id=$netid,hostname=$vmname";
1256     }
1257
1258     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1259
1260     return $netdev;
1261 }
1262
1263 sub drive_is_cdrom {
1264     my ($drive) = @_;
1265
1266     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1267
1268 }
1269
1270 sub parse_numa {
1271     my ($data) = @_;
1272
1273     my $res = {};
1274
1275     foreach my $kvp (split(/,/, $data)) {
1276
1277         if ($kvp =~ m/^memory=(\S+)$/) {
1278             $res->{memory} = $1;
1279         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1280             $res->{policy} = $1;
1281         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1282             $res->{cpus}->{start} = $1;
1283             $res->{cpus}->{end} = $3;
1284         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1285             $res->{hostnodes}->{start} = $1;
1286             $res->{hostnodes}->{end} = $3;
1287         } else {
1288             return undef;
1289         }
1290     }
1291
1292     return $res;
1293 }
1294
1295 sub parse_hostpci {
1296     my ($value) = @_;
1297
1298     return undef if !$value;
1299
1300
1301     my @list = split(/,/, $value);
1302     my $found;
1303
1304     my $res = {};
1305     foreach my $kv (@list) {
1306
1307         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1308             $found = 1;
1309             if(defined($4)){
1310                 push @{$res->{pciid}}, { id => $2 , function => $4};
1311
1312             }else{
1313                 my $pcidevices = lspci($2);
1314                 $res->{pciid} = $pcidevices->{$2};
1315             }
1316         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1317             $res->{driver} = $1;
1318         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1319             $res->{rombar} = $1;
1320         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1321             $res->{'x-vga'} = $1;
1322         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1323             $res->{pcie} = 1 if $1 == 1;
1324         } else {
1325             warn "unknown hostpci setting '$kv'\n";
1326         }
1327     }
1328
1329     return undef if !$found;
1330
1331     return $res;
1332 }
1333
1334 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1335 sub parse_net {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1343             my $model = lc($1);
1344             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1345             $res->{model} = $model;
1346             $res->{macaddr} = $mac;
1347         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1348             $res->{bridge} = $1;
1349         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1350             $res->{queues} = $1;
1351         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1352             $res->{rate} = $1;
1353         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1354             $res->{tag} = $1;
1355         } elsif ($kvp =~ m/^firewall=([01])$/) {
1356             $res->{firewall} = $1;
1357         } elsif ($kvp =~ m/^link_down=([01])$/) {
1358             $res->{link_down} = $1;
1359         } else {
1360             return undef;
1361         }
1362
1363     }
1364
1365     return undef if !$res->{model};
1366
1367     return $res;
1368 }
1369
1370 sub print_net {
1371     my $net = shift;
1372
1373     my $res = "$net->{model}";
1374     $res .= "=$net->{macaddr}" if $net->{macaddr};
1375     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1376     $res .= ",rate=$net->{rate}" if $net->{rate};
1377     $res .= ",tag=$net->{tag}" if $net->{tag};
1378     $res .= ",firewall=1" if $net->{firewall};
1379     $res .= ",link_down=1" if $net->{link_down};
1380     $res .= ",queues=$net->{queues}" if $net->{queues};
1381
1382     return $res;
1383 }
1384
1385 sub add_random_macs {
1386     my ($settings) = @_;
1387
1388     foreach my $opt (keys %$settings) {
1389         next if $opt !~ m/^net(\d+)$/;
1390         my $net = parse_net($settings->{$opt});
1391         next if !$net;
1392         $settings->{$opt} = print_net($net);
1393     }
1394 }
1395
1396 sub add_unused_volume {
1397     my ($config, $volid) = @_;
1398
1399     my $key;
1400     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1401         my $test = "unused$ind";
1402         if (my $vid = $config->{$test}) {
1403             return if $vid eq $volid; # do not add duplicates
1404         } else {
1405             $key = $test;
1406         }
1407     }
1408
1409     die "To many unused volume - please delete them first.\n" if !$key;
1410
1411     $config->{$key} = $volid;
1412
1413     return $key;
1414 }
1415
1416 sub vm_is_volid_owner {
1417     my ($storecfg, $vmid, $volid) = @_;
1418
1419     if ($volid !~  m|^/|) {
1420         my ($path, $owner);
1421         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1422         if ($owner && ($owner == $vmid)) {
1423             return 1;
1424         }
1425     }
1426
1427     return undef;
1428 }
1429
1430 sub vmconfig_delete_pending_option {
1431     my ($conf, $key) = @_;
1432
1433     delete $conf->{pending}->{$key};
1434     my $pending_delete_hash = { $key => 1 };
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1439 }
1440
1441 sub vmconfig_undelete_pending_option {
1442     my ($conf, $key) = @_;
1443
1444     my $pending_delete_hash = {};
1445     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1446         $pending_delete_hash->{$opt} = 1;
1447     }
1448     delete $pending_delete_hash->{$key};
1449
1450     my @keylist = keys %$pending_delete_hash;
1451     if (scalar(@keylist)) {
1452         $conf->{pending}->{delete} = join(',', @keylist);
1453     } else {
1454         delete $conf->{pending}->{delete};
1455     }
1456 }
1457
1458 sub vmconfig_register_unused_drive {
1459     my ($storecfg, $vmid, $conf, $drive) = @_;
1460
1461     if (!drive_is_cdrom($drive)) {
1462         my $volid = $drive->{file};
1463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1464             add_unused_volume($conf, $volid, $vmid);
1465         }
1466     }
1467 }
1468
1469 sub vmconfig_cleanup_pending {
1470     my ($conf) = @_;
1471
1472     # remove pending changes when nothing changed
1473     my $changes;
1474     foreach my $opt (keys %{$conf->{pending}}) {
1475         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1476             $changes = 1;
1477             delete $conf->{pending}->{$opt};
1478         }
1479     }
1480
1481     # remove delete if option is not set
1482     my $pending_delete_hash = {};
1483     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1484         if (defined($conf->{$opt})) {
1485             $pending_delete_hash->{$opt} = 1;
1486         } else {
1487             $changes = 1;
1488         }
1489     }
1490
1491     my @keylist = keys %$pending_delete_hash;
1492     if (scalar(@keylist)) {
1493         $conf->{pending}->{delete} = join(',', @keylist);
1494     } else {
1495         delete $conf->{pending}->{delete};
1496     }
1497
1498     return $changes;
1499 }
1500
1501 my $valid_smbios1_options = {
1502     manufacturer => '\S+',
1503     product => '\S+',
1504     version => '\S+',
1505     serial => '\S+',
1506     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1507     sku => '\S+',
1508     family => '\S+',
1509 };
1510
1511 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1512 sub parse_smbios1 {
1513     my ($data) = @_;
1514
1515     my $res = {};
1516
1517     foreach my $kvp (split(/,/, $data)) {
1518         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1519         my ($k, $v) = split(/=/, $kvp);
1520         return undef if !defined($k) || !defined($v);
1521         return undef if !$valid_smbios1_options->{$k};
1522         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1523         $res->{$k} = $v;
1524     }
1525
1526     return $res;
1527 }
1528
1529 sub print_smbios1 {
1530     my ($smbios1) = @_;
1531
1532     my $data = '';
1533     foreach my $k (keys %$smbios1) {
1534         next if !defined($smbios1->{$k});
1535         next if !$valid_smbios1_options->{$k};
1536         $data .= ',' if $data;
1537         $data .= "$k=$smbios1->{$k}";
1538     }
1539     return $data;
1540 }
1541
1542 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1543 sub verify_smbios1 {
1544     my ($value, $noerr) = @_;
1545
1546     return $value if parse_smbios1($value);
1547
1548     return undef if $noerr;
1549
1550     die "unable to parse smbios (type 1) options\n";
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1554 sub verify_bootdisk {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if valid_drivename($value);
1558
1559     return undef if $noerr;
1560
1561     die "invalid boot disk '$value'\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1565 sub verify_numa {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_numa($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse numa options\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1576 sub verify_net {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_net($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse network options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1587 sub verify_drive {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_drive(undef, $value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse drive options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1598 sub verify_hostpci {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_hostpci($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse pci id\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1609 sub verify_watchdog {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_watchdog($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse watchdog options\n";
1617 }
1618
1619 sub parse_watchdog {
1620     my ($value) = @_;
1621
1622     return undef if !$value;
1623
1624     my $res = {};
1625
1626     foreach my $p (split(/,/, $value)) {
1627         next if $p =~ m/^\s*$/;
1628
1629         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1630             $res->{model} = $2;
1631         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1632             $res->{action} = $2;
1633         } else {
1634             return undef;
1635         }
1636     }
1637
1638     return $res;
1639 }
1640
1641 sub parse_usb_device {
1642     my ($value) = @_;
1643
1644     return undef if !$value;
1645
1646     my @dl = split(/,/, $value);
1647     my $found;
1648
1649     my $res = {};
1650     foreach my $v (@dl) {
1651         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1652             $found = 1;
1653             $res->{vendorid} = $2;
1654             $res->{productid} = $4;
1655         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1656             $found = 1;
1657             $res->{hostbus} = $1;
1658             $res->{hostport} = $2;
1659         } elsif ($v =~ m/^spice$/) {
1660             $found = 1;
1661             $res->{spice} = 1;
1662         } else {
1663             return undef;
1664         }
1665     }
1666     return undef if !$found;
1667
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1672 sub verify_usb_device {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_usb_device($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse usb device\n";
1680 }
1681
1682 # add JSON properties for create and set function
1683 sub json_config_properties {
1684     my $prop = shift;
1685
1686     foreach my $opt (keys %$confdesc) {
1687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1688         $prop->{$opt} = $confdesc->{$opt};
1689     }
1690
1691     return $prop;
1692 }
1693
1694 sub check_type {
1695     my ($key, $value) = @_;
1696
1697     die "unknown setting '$key'\n" if !$confdesc->{$key};
1698
1699     my $type = $confdesc->{$key}->{type};
1700
1701     if (!defined($value)) {
1702         die "got undefined value\n";
1703     }
1704
1705     if ($value =~ m/[\n\r]/) {
1706         die "property contains a line feed\n";
1707     }
1708
1709     if ($type eq 'boolean') {
1710         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1711         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1712         die "type check ('boolean') failed - got '$value'\n";
1713     } elsif ($type eq 'integer') {
1714         return int($1) if $value =~ m/^(\d+)$/;
1715         die "type check ('integer') failed - got '$value'\n";
1716     } elsif ($type eq 'number') {
1717         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1718         die "type check ('number') failed - got '$value'\n";
1719     } elsif ($type eq 'string') {
1720         if (my $fmt = $confdesc->{$key}->{format}) {
1721             if ($fmt eq 'pve-qm-drive') {
1722                 # special case - we need to pass $key to parse_drive()
1723                 my $drive = parse_drive($key, $value);
1724                 return $value if $drive;
1725                 die "unable to parse drive options\n";
1726             }
1727             PVE::JSONSchema::check_format($fmt, $value);
1728             return $value;
1729         }
1730         $value =~ s/^\"(.*)\"$/$1/;
1731         return $value;
1732     } else {
1733         die "internal error"
1734     }
1735 }
1736
1737 sub lock_config_full {
1738     my ($vmid, $timeout, $code, @param) = @_;
1739
1740     my $filename = config_file_lock($vmid);
1741
1742     my $res = lock_file($filename, $timeout, $code, @param);
1743
1744     die $@ if $@;
1745
1746     return $res;
1747 }
1748
1749 sub lock_config_mode {
1750     my ($vmid, $timeout, $shared, $code, @param) = @_;
1751
1752     my $filename = config_file_lock($vmid);
1753
1754     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1755
1756     die $@ if $@;
1757
1758     return $res;
1759 }
1760
1761 sub lock_config {
1762     my ($vmid, $code, @param) = @_;
1763
1764     return lock_config_full($vmid, 10, $code, @param);
1765 }
1766
1767 sub cfs_config_path {
1768     my ($vmid, $node) = @_;
1769
1770     $node = $nodename if !$node;
1771     return "nodes/$node/qemu-server/$vmid.conf";
1772 }
1773
1774 sub check_iommu_support{
1775     #fixme : need to check IOMMU support
1776     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1777
1778     my $iommu=1;
1779     return $iommu;
1780
1781 }
1782
1783 sub config_file {
1784     my ($vmid, $node) = @_;
1785
1786     my $cfspath = cfs_config_path($vmid, $node);
1787     return "/etc/pve/$cfspath";
1788 }
1789
1790 sub config_file_lock {
1791     my ($vmid) = @_;
1792
1793     return "$lock_dir/lock-$vmid.conf";
1794 }
1795
1796 sub touch_config {
1797     my ($vmid) = @_;
1798
1799     my $conf = config_file($vmid);
1800     utime undef, undef, $conf;
1801 }
1802
1803 sub destroy_vm {
1804     my ($storecfg, $vmid, $keep_empty_config) = @_;
1805
1806     my $conffile = config_file($vmid);
1807
1808     my $conf = load_config($vmid);
1809
1810     check_lock($conf);
1811
1812     # only remove disks owned by this VM
1813     foreach_drive($conf, sub {
1814         my ($ds, $drive) = @_;
1815
1816         return if drive_is_cdrom($drive);
1817
1818         my $volid = $drive->{file};
1819
1820         return if !$volid || $volid =~ m|^/|;
1821
1822         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1823         return if !$path || !$owner || ($owner != $vmid);
1824
1825         PVE::Storage::vdisk_free($storecfg, $volid);
1826     });
1827
1828     if ($keep_empty_config) {
1829         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1830     } else {
1831         unlink $conffile;
1832     }
1833
1834     # also remove unused disk
1835     eval {
1836         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1837
1838         eval {
1839             PVE::Storage::foreach_volid($dl, sub {
1840                 my ($volid, $sid, $volname, $d) = @_;
1841                 PVE::Storage::vdisk_free($storecfg, $volid);
1842             });
1843         };
1844         warn $@ if $@;
1845
1846     };
1847     warn $@ if $@;
1848 }
1849
1850 sub load_config {
1851     my ($vmid, $node) = @_;
1852
1853     my $cfspath = cfs_config_path($vmid, $node);
1854
1855     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1856
1857     die "no such VM ('$vmid')\n" if !defined($conf);
1858
1859     return $conf;
1860 }
1861
1862 sub parse_vm_config {
1863     my ($filename, $raw) = @_;
1864
1865     return undef if !defined($raw);
1866
1867     my $res = {
1868         digest => Digest::SHA::sha1_hex($raw),
1869         snapshots => {},
1870         pending => {},
1871     };
1872
1873     $filename =~ m|/qemu-server/(\d+)\.conf$|
1874         || die "got strange filename '$filename'";
1875
1876     my $vmid = $1;
1877
1878     my $conf = $res;
1879     my $descr;
1880     my $section = '';
1881
1882     my @lines = split(/\n/, $raw);
1883     foreach my $line (@lines) {
1884         next if $line =~ m/^\s*$/;
1885
1886         if ($line =~ m/^\[PENDING\]\s*$/i) {
1887             $section = 'pending';
1888             if (defined($descr)) {
1889                 $descr =~ s/\s+$//;
1890                 $conf->{description} = $descr;
1891             }
1892             $descr = undef;
1893             $conf = $res->{$section} = {};
1894             next;
1895
1896         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1897             $section = $1;
1898             if (defined($descr)) {
1899                 $descr =~ s/\s+$//;
1900                 $conf->{description} = $descr;
1901             }
1902             $descr = undef;
1903             $conf = $res->{snapshots}->{$section} = {};
1904             next;
1905         }
1906
1907         if ($line =~ m/^\#(.*)\s*$/) {
1908             $descr = '' if !defined($descr);
1909             $descr .= PVE::Tools::decode_text($1) . "\n";
1910             next;
1911         }
1912
1913         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1914             $descr = '' if !defined($descr);
1915             $descr .= PVE::Tools::decode_text($2);
1916         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1917             $conf->{snapstate} = $1;
1918         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1919             my $key = $1;
1920             my $value = $2;
1921             $conf->{$key} = $value;
1922         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1923             my $value = $1;
1924             if ($section eq 'pending') {
1925                 $conf->{delete} = $value; # we parse this later
1926             } else {
1927                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1928             }
1929         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1930             my $key = $1;
1931             my $value = $2;
1932             eval { $value = check_type($key, $value); };
1933             if ($@) {
1934                 warn "vm $vmid - unable to parse value of '$key' - $@";
1935             } else {
1936                 my $fmt = $confdesc->{$key}->{format};
1937                 if ($fmt && $fmt eq 'pve-qm-drive') {
1938                     my $v = parse_drive($key, $value);
1939                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1940                         $v->{file} = $volid;
1941                         $value = print_drive($vmid, $v);
1942                     } else {
1943                         warn "vm $vmid - unable to parse value of '$key'\n";
1944                         next;
1945                     }
1946                 }
1947
1948                 if ($key eq 'cdrom') {
1949                     $conf->{ide2} = $value;
1950                 } else {
1951                     $conf->{$key} = $value;
1952                 }
1953             }
1954         }
1955     }
1956
1957     if (defined($descr)) {
1958         $descr =~ s/\s+$//;
1959         $conf->{description} = $descr;
1960     }
1961     delete $res->{snapstate}; # just to be sure
1962
1963     return $res;
1964 }
1965
1966 sub write_vm_config {
1967     my ($filename, $conf) = @_;
1968
1969     delete $conf->{snapstate}; # just to be sure
1970
1971     if ($conf->{cdrom}) {
1972         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1973         $conf->{ide2} = $conf->{cdrom};
1974         delete $conf->{cdrom};
1975     }
1976
1977     # we do not use 'smp' any longer
1978     if ($conf->{sockets}) {
1979         delete $conf->{smp};
1980     } elsif ($conf->{smp}) {
1981         $conf->{sockets} = $conf->{smp};
1982         delete $conf->{cores};
1983         delete $conf->{smp};
1984     }
1985
1986     my $used_volids = {};
1987
1988     my $cleanup_config = sub {
1989         my ($cref, $pending, $snapname) = @_;
1990
1991         foreach my $key (keys %$cref) {
1992             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1993                 $key eq 'snapstate' || $key eq 'pending';
1994             my $value = $cref->{$key};
1995             if ($key eq 'delete') {
1996                 die "propertry 'delete' is only allowed in [PENDING]\n"
1997                     if !$pending;
1998                 # fixme: check syntax?
1999                 next;
2000             }
2001             eval { $value = check_type($key, $value); };
2002             die "unable to parse value of '$key' - $@" if $@;
2003
2004             $cref->{$key} = $value;
2005
2006             if (!$snapname && valid_drivename($key)) {
2007                 my $drive = parse_drive($key, $value);
2008                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2009             }
2010         }
2011     };
2012
2013     &$cleanup_config($conf);
2014
2015     &$cleanup_config($conf->{pending}, 1);
2016
2017     foreach my $snapname (keys %{$conf->{snapshots}}) {
2018         die "internal error" if $snapname eq 'pending';
2019         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2020     }
2021
2022     # remove 'unusedX' settings if we re-add a volume
2023     foreach my $key (keys %$conf) {
2024         my $value = $conf->{$key};
2025         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2026             delete $conf->{$key};
2027         }
2028     }
2029
2030     my $generate_raw_config = sub {
2031         my ($conf, $pending) = @_;
2032
2033         my $raw = '';
2034
2035         # add description as comment to top of file
2036         if (defined(my $descr = $conf->{description})) {
2037             if ($descr) {
2038                 foreach my $cl (split(/\n/, $descr)) {
2039                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2040                 }
2041             } else {
2042                 $raw .= "#\n" if $pending;
2043             }
2044         }
2045
2046         foreach my $key (sort keys %$conf) {
2047             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2048             $raw .= "$key: $conf->{$key}\n";
2049         }
2050         return $raw;
2051     };
2052
2053     my $raw = &$generate_raw_config($conf);
2054
2055     if (scalar(keys %{$conf->{pending}})){
2056         $raw .= "\n[PENDING]\n";
2057         $raw .= &$generate_raw_config($conf->{pending}, 1);
2058     }
2059
2060     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2061         $raw .= "\n[$snapname]\n";
2062         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2063     }
2064
2065     return $raw;
2066 }
2067
2068 sub update_config_nolock {
2069     my ($vmid, $conf, $skiplock) = @_;
2070
2071     check_lock($conf) if !$skiplock;
2072
2073     my $cfspath = cfs_config_path($vmid);
2074
2075     PVE::Cluster::cfs_write_file($cfspath, $conf);
2076 }
2077
2078 sub update_config {
2079     my ($vmid, $conf, $skiplock) = @_;
2080
2081     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2082 }
2083
2084 sub load_defaults {
2085
2086     my $res = {};
2087
2088     # we use static defaults from our JSON schema configuration
2089     foreach my $key (keys %$confdesc) {
2090         if (defined(my $default = $confdesc->{$key}->{default})) {
2091             $res->{$key} = $default;
2092         }
2093     }
2094
2095     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2096     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2097
2098     return $res;
2099 }
2100
2101 sub config_list {
2102     my $vmlist = PVE::Cluster::get_vmlist();
2103     my $res = {};
2104     return $res if !$vmlist || !$vmlist->{ids};
2105     my $ids = $vmlist->{ids};
2106
2107     foreach my $vmid (keys %$ids) {
2108         my $d = $ids->{$vmid};
2109         next if !$d->{node} || $d->{node} ne $nodename;
2110         next if !$d->{type} || $d->{type} ne 'qemu';
2111         $res->{$vmid}->{exists} = 1;
2112     }
2113     return $res;
2114 }
2115
2116 # test if VM uses local resources (to prevent migration)
2117 sub check_local_resources {
2118     my ($conf, $noerr) = @_;
2119
2120     my $loc_res = 0;
2121
2122     $loc_res = 1 if $conf->{hostusb}; # old syntax
2123     $loc_res = 1 if $conf->{hostpci}; # old syntax
2124
2125     foreach my $k (keys %$conf) {
2126         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2127         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2128     }
2129
2130     die "VM uses local resources\n" if $loc_res && !$noerr;
2131
2132     return $loc_res;
2133 }
2134
2135 # check if used storages are available on all nodes (use by migrate)
2136 sub check_storage_availability {
2137     my ($storecfg, $conf, $node) = @_;
2138
2139     foreach_drive($conf, sub {
2140         my ($ds, $drive) = @_;
2141
2142         my $volid = $drive->{file};
2143         return if !$volid;
2144
2145         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2146         return if !$sid;
2147
2148         # check if storage is available on both nodes
2149         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2150         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2151    });
2152 }
2153
2154 # list nodes where all VM images are available (used by has_feature API)
2155 sub shared_nodes {
2156     my ($conf, $storecfg) = @_;
2157
2158     my $nodelist = PVE::Cluster::get_nodelist();
2159     my $nodehash = { map { $_ => 1 } @$nodelist };
2160     my $nodename = PVE::INotify::nodename();
2161
2162     foreach_drive($conf, sub {
2163         my ($ds, $drive) = @_;
2164
2165         my $volid = $drive->{file};
2166         return if !$volid;
2167
2168         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2169         if ($storeid) {
2170             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2171             if ($scfg->{disable}) {
2172                 $nodehash = {};
2173             } elsif (my $avail = $scfg->{nodes}) {
2174                 foreach my $node (keys %$nodehash) {
2175                     delete $nodehash->{$node} if !$avail->{$node};
2176                 }
2177             } elsif (!$scfg->{shared}) {
2178                 foreach my $node (keys %$nodehash) {
2179                     delete $nodehash->{$node} if $node ne $nodename
2180                 }
2181             }
2182         }
2183     });
2184
2185     return $nodehash
2186 }
2187
2188 sub check_lock {
2189     my ($conf) = @_;
2190
2191     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2192 }
2193
2194 sub check_cmdline {
2195     my ($pidfile, $pid) = @_;
2196
2197     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2198     if (defined($fh)) {
2199         my $line = <$fh>;
2200         $fh->close;
2201         return undef if !$line;
2202         my @param = split(/\0/, $line);
2203
2204         my $cmd = $param[0];
2205         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2206
2207         for (my $i = 0; $i < scalar (@param); $i++) {
2208             my $p = $param[$i];
2209             next if !$p;
2210             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2211                 my $p = $param[$i+1];
2212                 return 1 if $p && ($p eq $pidfile);
2213                 return undef;
2214             }
2215         }
2216     }
2217     return undef;
2218 }
2219
2220 sub check_running {
2221     my ($vmid, $nocheck, $node) = @_;
2222
2223     my $filename = config_file($vmid, $node);
2224
2225     die "unable to find configuration file for VM $vmid - no such machine\n"
2226         if !$nocheck && ! -f $filename;
2227
2228     my $pidfile = pidfile_name($vmid);
2229
2230     if (my $fd = IO::File->new("<$pidfile")) {
2231         my $st = stat($fd);
2232         my $line = <$fd>;
2233         close($fd);
2234
2235         my $mtime = $st->mtime;
2236         if ($mtime > time()) {
2237             warn "file '$filename' modified in future\n";
2238         }
2239
2240         if ($line =~ m/^(\d+)$/) {
2241             my $pid = $1;
2242             if (check_cmdline($pidfile, $pid)) {
2243                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2244                     return $pid;
2245                 }
2246             }
2247         }
2248     }
2249
2250     return undef;
2251 }
2252
2253 sub vzlist {
2254
2255     my $vzlist = config_list();
2256
2257     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2258
2259     while (defined(my $de = $fd->read)) {
2260         next if $de !~ m/^(\d+)\.pid$/;
2261         my $vmid = $1;
2262         next if !defined($vzlist->{$vmid});
2263         if (my $pid = check_running($vmid)) {
2264             $vzlist->{$vmid}->{pid} = $pid;
2265         }
2266     }
2267
2268     return $vzlist;
2269 }
2270
2271 sub disksize {
2272     my ($storecfg, $conf) = @_;
2273
2274     my $bootdisk = $conf->{bootdisk};
2275     return undef if !$bootdisk;
2276     return undef if !valid_drivename($bootdisk);
2277
2278     return undef if !$conf->{$bootdisk};
2279
2280     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2281     return undef if !defined($drive);
2282
2283     return undef if drive_is_cdrom($drive);
2284
2285     my $volid = $drive->{file};
2286     return undef if !$volid;
2287
2288     return $drive->{size};
2289 }
2290
2291 my $last_proc_pid_stat;
2292
2293 # get VM status information
2294 # This must be fast and should not block ($full == false)
2295 # We only query KVM using QMP if $full == true (this can be slow)
2296 sub vmstatus {
2297     my ($opt_vmid, $full) = @_;
2298
2299     my $res = {};
2300
2301     my $storecfg = PVE::Storage::config();
2302
2303     my $list = vzlist();
2304     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2305
2306     my $cpucount = $cpuinfo->{cpus} || 1;
2307
2308     foreach my $vmid (keys %$list) {
2309         next if $opt_vmid && ($vmid ne $opt_vmid);
2310
2311         my $cfspath = cfs_config_path($vmid);
2312         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2313
2314         my $d = {};
2315         $d->{pid} = $list->{$vmid}->{pid};
2316
2317         # fixme: better status?
2318         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2319
2320         my $size = disksize($storecfg, $conf);
2321         if (defined($size)) {
2322             $d->{disk} = 0; # no info available
2323             $d->{maxdisk} = $size;
2324         } else {
2325             $d->{disk} = 0;
2326             $d->{maxdisk} = 0;
2327         }
2328
2329         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2330         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2331         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2332
2333         $d->{name} = $conf->{name} || "VM $vmid";
2334         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2335
2336         if ($conf->{balloon}) {
2337             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2338             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2339         }
2340
2341         $d->{uptime} = 0;
2342         $d->{cpu} = 0;
2343         $d->{mem} = 0;
2344
2345         $d->{netout} = 0;
2346         $d->{netin} = 0;
2347
2348         $d->{diskread} = 0;
2349         $d->{diskwrite} = 0;
2350
2351         $d->{template} = is_template($conf);
2352
2353         $res->{$vmid} = $d;
2354     }
2355
2356     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2357     foreach my $dev (keys %$netdev) {
2358         next if $dev !~ m/^tap([1-9]\d*)i/;
2359         my $vmid = $1;
2360         my $d = $res->{$vmid};
2361         next if !$d;
2362
2363         $d->{netout} += $netdev->{$dev}->{receive};
2364         $d->{netin} += $netdev->{$dev}->{transmit};
2365
2366         if ($full) {
2367             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2368             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2369         }
2370
2371     }
2372
2373     my $ctime = gettimeofday;
2374
2375     foreach my $vmid (keys %$list) {
2376
2377         my $d = $res->{$vmid};
2378         my $pid = $d->{pid};
2379         next if !$pid;
2380
2381         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2382         next if !$pstat; # not running
2383
2384         my $used = $pstat->{utime} + $pstat->{stime};
2385
2386         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2387
2388         if ($pstat->{vsize}) {
2389             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2390         }
2391
2392         my $old = $last_proc_pid_stat->{$pid};
2393         if (!$old) {
2394             $last_proc_pid_stat->{$pid} = {
2395                 time => $ctime,
2396                 used => $used,
2397                 cpu => 0,
2398             };
2399             next;
2400         }
2401
2402         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2403
2404         if ($dtime > 1000) {
2405             my $dutime = $used -  $old->{used};
2406
2407             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2408             $last_proc_pid_stat->{$pid} = {
2409                 time => $ctime,
2410                 used => $used,
2411                 cpu => $d->{cpu},
2412             };
2413         } else {
2414             $d->{cpu} = $old->{cpu};
2415         }
2416     }
2417
2418     return $res if !$full;
2419
2420     my $qmpclient = PVE::QMPClient->new();
2421
2422     my $ballooncb = sub {
2423         my ($vmid, $resp) = @_;
2424
2425         my $info = $resp->{'return'};
2426         return if !$info->{max_mem};
2427
2428         my $d = $res->{$vmid};
2429
2430         # use memory assigned to VM
2431         $d->{maxmem} = $info->{max_mem};
2432         $d->{balloon} = $info->{actual};
2433
2434         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2435             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2436             $d->{freemem} = $info->{free_mem};
2437         }
2438
2439         $d->{ballooninfo} = $info;
2440     };
2441
2442     my $blockstatscb = sub {
2443         my ($vmid, $resp) = @_;
2444         my $data = $resp->{'return'} || [];
2445         my $totalrdbytes = 0;
2446         my $totalwrbytes = 0;
2447
2448         for my $blockstat (@$data) {
2449             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2450             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2451
2452             $blockstat->{device} =~ s/drive-//;
2453             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2454         }
2455         $res->{$vmid}->{diskread} = $totalrdbytes;
2456         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2457     };
2458
2459     my $statuscb = sub {
2460         my ($vmid, $resp) = @_;
2461
2462         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2463         # this fails if ballon driver is not loaded, so this must be
2464         # the last commnand (following command are aborted if this fails).
2465         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2466
2467         my $status = 'unknown';
2468         if (!defined($status = $resp->{'return'}->{status})) {
2469             warn "unable to get VM status\n";
2470             return;
2471         }
2472
2473         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2474     };
2475
2476     foreach my $vmid (keys %$list) {
2477         next if $opt_vmid && ($vmid ne $opt_vmid);
2478         next if !$res->{$vmid}->{pid}; # not running
2479         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2480     }
2481
2482     $qmpclient->queue_execute(undef, 1);
2483
2484     foreach my $vmid (keys %$list) {
2485         next if $opt_vmid && ($vmid ne $opt_vmid);
2486         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2487     }
2488
2489     return $res;
2490 }
2491
2492 sub foreach_dimm {
2493     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2494
2495     my $dimm_id = 0;
2496     my $current_size = 1024;
2497     my $dimm_size = 512;
2498     return if $current_size == $memory;
2499
2500     for (my $j = 0; $j < 8; $j++) {
2501         for (my $i = 0; $i < 32; $i++) {
2502             my $name = "dimm${dimm_id}";
2503             $dimm_id++;
2504             my $numanode = $i % $sockets;
2505             $current_size += $dimm_size;
2506             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2507             return  $current_size if $current_size >= $memory;
2508         }
2509         $dimm_size *= 2;
2510     }
2511 }
2512
2513 sub foreach_reverse_dimm {
2514     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2515
2516     my $dimm_id = 253;
2517     my $current_size = 4177920;
2518     my $dimm_size = 65536;
2519     return if $current_size == $memory;
2520
2521     for (my $j = 0; $j < 8; $j++) {
2522         for (my $i = 0; $i < 32; $i++) {
2523             my $name = "dimm${dimm_id}";
2524             $dimm_id--;
2525             my $numanode = $i % $sockets;
2526             $current_size -= $dimm_size;
2527             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2528             return  $current_size if $current_size <= $memory;
2529         }
2530         $dimm_size /= 2;
2531     }
2532 }
2533
2534 sub foreach_drive {
2535     my ($conf, $func) = @_;
2536
2537     foreach my $ds (keys %$conf) {
2538         next if !valid_drivename($ds);
2539
2540         my $drive = parse_drive($ds, $conf->{$ds});
2541         next if !$drive;
2542
2543         &$func($ds, $drive);
2544     }
2545 }
2546
2547 sub foreach_volid {
2548     my ($conf, $func) = @_;
2549
2550     my $volhash = {};
2551
2552     my $test_volid = sub {
2553         my ($volid, $is_cdrom) = @_;
2554
2555         return if !$volid;
2556
2557         $volhash->{$volid} = $is_cdrom || 0;
2558     };
2559
2560     foreach_drive($conf, sub {
2561         my ($ds, $drive) = @_;
2562         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2563     });
2564
2565     foreach my $snapname (keys %{$conf->{snapshots}}) {
2566         my $snap = $conf->{snapshots}->{$snapname};
2567         &$test_volid($snap->{vmstate}, 0);
2568         foreach_drive($snap, sub {
2569             my ($ds, $drive) = @_;
2570             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2571         });
2572     }
2573
2574     foreach my $volid (keys %$volhash) {
2575         &$func($volid, $volhash->{$volid});
2576     }
2577 }
2578
2579 sub vga_conf_has_spice {
2580     my ($vga) = @_;
2581
2582     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2583
2584     return $1 || 1;
2585 }
2586
2587 sub config_to_command {
2588     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2589
2590     my $cmd = [];
2591     my $globalFlags = [];
2592     my $machineFlags = [];
2593     my $rtcFlags = [];
2594     my $cpuFlags = [];
2595     my $devices = [];
2596     my $pciaddr = '';
2597     my $bridges = {};
2598     my $kvmver = kvm_user_version();
2599     my $vernum = 0; # unknown
2600     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2601         $vernum = $1*1000000+$2*1000;
2602     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2603         $vernum = $1*1000000+$2*1000+$3;
2604     }
2605
2606     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2607
2608     my $have_ovz = -f '/proc/vz/vestat';
2609
2610     my $q35 = machine_type_is_q35($conf);
2611     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2612     my $machine_type = $forcemachine || $conf->{machine};
2613
2614     my $cpuunits = defined($conf->{cpuunits}) ?
2615             $conf->{cpuunits} : $defaults->{cpuunits};
2616
2617     push @$cmd, '/usr/bin/systemd-run';
2618     push @$cmd, '--scope';
2619     push @$cmd, '--slice', "qemu";
2620     push @$cmd, '--unit', $vmid;
2621     push @$cmd, '-p', "CPUShares=$cpuunits";
2622     if ($conf->{cpulimit}) {
2623         my $cpulimit = int($conf->{cpulimit} * 100);
2624         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2625     }
2626
2627     push @$cmd, '/usr/bin/kvm';
2628
2629     push @$cmd, '-id', $vmid;
2630
2631     my $use_virtio = 0;
2632
2633     my $qmpsocket = qmp_socket($vmid);
2634     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2635     push @$cmd, '-mon', "chardev=qmp,mode=control";
2636
2637     my $socket = vnc_socket($vmid);
2638     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2639
2640     push @$cmd, '-pidfile' , pidfile_name($vmid);
2641
2642     push @$cmd, '-daemonize';
2643
2644     if ($conf->{smbios1}) {
2645         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2646     }
2647
2648     if ($q35) {
2649         # the q35 chipset support native usb2, so we enable usb controller
2650         # by default for this machine type
2651         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2652     } else {
2653         $pciaddr = print_pci_addr("piix3", $bridges);
2654         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2655
2656         my $use_usb2 = 0;
2657         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2658             next if !$conf->{"usb$i"};
2659             $use_usb2 = 1;
2660         }
2661         # include usb device config
2662         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2663     }
2664
2665     my $vga = $conf->{vga};
2666
2667     my $qxlnum = vga_conf_has_spice($vga);
2668     $vga = 'qxl' if $qxlnum;
2669
2670     if (!$vga) {
2671         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2672                                 $conf->{ostype} eq 'win7' ||
2673                                 $conf->{ostype} eq 'w2k8')) {
2674             $vga = 'std';
2675         } else {
2676             $vga = 'cirrus';
2677         }
2678     }
2679
2680     # enable absolute mouse coordinates (needed by vnc)
2681     my $tablet;
2682     if (defined($conf->{tablet})) {
2683         $tablet = $conf->{tablet};
2684     } else {
2685         $tablet = $defaults->{tablet};
2686         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2687         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2688     }
2689
2690     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2691
2692     # host pci devices
2693     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2694         my $d = parse_hostpci($conf->{"hostpci$i"});
2695         next if !$d;
2696
2697         my $pcie = $d->{pcie};
2698         if($pcie){
2699             die "q35 machine model is not enabled" if !$q35;
2700             $pciaddr = print_pcie_addr("hostpci$i");
2701         }else{
2702             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2703         }
2704
2705         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2706         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2707         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2708         if ($xvga && $xvga ne '') {
2709             push @$cpuFlags, 'kvm=off';
2710             $vga = 'none';
2711         }
2712         $driver = "vfio-pci" if $xvga ne '';
2713         my $pcidevices = $d->{pciid};
2714         my $multifunction = 1 if @$pcidevices > 1;
2715
2716         my $j=0;
2717         foreach my $pcidevice (@$pcidevices) {
2718
2719             my $id = "hostpci$i";
2720             $id .= ".$j" if $multifunction;
2721             my $addr = $pciaddr;
2722             $addr .= ".$j" if $multifunction;
2723             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2724
2725             if($j == 0){
2726                 $devicestr .= "$rombar$xvga";
2727                 $devicestr .= ",multifunction=on" if $multifunction;
2728             }
2729
2730             push @$devices, '-device', $devicestr;
2731             $j++;
2732         }
2733     }
2734
2735     # usb devices
2736     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2737         my $d = parse_usb_device($conf->{"usb$i"});
2738         next if !$d;
2739         if ($d->{vendorid} && $d->{productid}) {
2740             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2741         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2742             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2743         } elsif ($d->{spice}) {
2744             # usb redir support for spice
2745             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2746             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2747         }
2748     }
2749
2750     # serial devices
2751     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2752         if (my $path = $conf->{"serial$i"}) {
2753             if ($path eq 'socket') {
2754                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2755                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2756                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2757             } else {
2758                 die "no such serial device\n" if ! -c $path;
2759                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2760                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2761             }
2762         }
2763     }
2764
2765     # parallel devices
2766     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2767         if (my $path = $conf->{"parallel$i"}) {
2768             die "no such parallel device\n" if ! -c $path;
2769             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2770             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2771             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2772         }
2773     }
2774
2775     my $vmname = $conf->{name} || "vm$vmid";
2776
2777     push @$cmd, '-name', $vmname;
2778
2779     my $sockets = 1;
2780     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2781     $sockets = $conf->{sockets} if  $conf->{sockets};
2782
2783     my $cores = $conf->{cores} || 1;
2784
2785     my $maxcpus = $sockets * $cores;
2786
2787     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2788
2789     my $allowed_vcpus = $cpuinfo->{cpus};
2790
2791     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2792         if ($allowed_vcpus < $maxcpus);
2793
2794     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2795
2796     push @$cmd, '-nodefaults';
2797
2798     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2799
2800     my $bootindex_hash = {};
2801     my $i = 1;
2802     foreach my $o (split(//, $bootorder)) {
2803         $bootindex_hash->{$o} = $i*100;
2804         $i++;
2805     }
2806
2807     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2808
2809     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2810
2811     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2812
2813     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2814
2815     # time drift fix
2816     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2817
2818     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2819     my $useLocaltime = $conf->{localtime};
2820
2821     if (my $ost = $conf->{ostype}) {
2822         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2823
2824         if ($ost =~ m/^w/) { # windows
2825             $useLocaltime = 1 if !defined($conf->{localtime});
2826
2827             # use time drift fix when acpi is enabled
2828             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2829                 $tdf = 1 if !defined($conf->{tdf});
2830             }
2831         }
2832
2833         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2834             $ost eq 'wvista') {
2835             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2836             push @$cmd, '-no-hpet';
2837             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2838                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2839                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2840                 push @$cpuFlags , 'hv_time' if !$nokvm;
2841
2842             } else {
2843                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2844             }
2845         }
2846
2847         if ($ost eq 'win7' || $ost eq 'win8') {
2848             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2849         }
2850     }
2851
2852     push @$rtcFlags, 'driftfix=slew' if $tdf;
2853
2854     if ($nokvm) {
2855         push @$machineFlags, 'accel=tcg';
2856     } else {
2857         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2858     }
2859
2860     if ($machine_type) {
2861         push @$machineFlags, "type=${machine_type}";
2862     }
2863
2864     if ($conf->{startdate}) {
2865         push @$rtcFlags, "base=$conf->{startdate}";
2866     } elsif ($useLocaltime) {
2867         push @$rtcFlags, 'base=localtime';
2868     }
2869
2870     my $cpu = $nokvm ? "qemu64" : "kvm64";
2871     $cpu = $conf->{cpu} if $conf->{cpu};
2872
2873     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2874
2875     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2876
2877     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2878
2879     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2880
2881     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2882
2883     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2884
2885         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2886         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2887     }
2888
2889     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2890
2891     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2892
2893     push @$cmd, '-cpu', $cpu;
2894
2895     my $memory = $conf->{memory} || $defaults->{memory};
2896     my $static_memory = 0;
2897     my $dimm_memory = 0;
2898
2899     if ($hotplug_features->{memory}) {
2900         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2901         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2902         $static_memory = $STATICMEM;
2903         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2904         $dimm_memory = $memory - $static_memory;
2905         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2906
2907     } else {
2908
2909         $static_memory = $memory;
2910         push @$cmd, '-m', $static_memory;
2911     }
2912
2913     if ($conf->{numa}) {
2914
2915         my $numa_totalmemory = undef;
2916         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2917             next if !$conf->{"numa$i"};
2918             my $numa = parse_numa($conf->{"numa$i"});
2919             next if !$numa;
2920             # memory
2921             die "missing numa node$i memory value\n" if !$numa->{memory};
2922             my $numa_memory = $numa->{memory};
2923             $numa_totalmemory += $numa_memory;
2924             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2925
2926             # cpus
2927             my $cpus_start = $numa->{cpus}->{start};
2928             die "missing numa node$i cpus\n" if !defined($cpus_start);
2929             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2930             my $cpus = $cpus_start;
2931             if (defined($cpus_end)) {
2932                 $cpus .= "-$cpus_end";
2933                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2934             }
2935
2936             # hostnodes
2937             my $hostnodes_start = $numa->{hostnodes}->{start};
2938             if (defined($hostnodes_start)) {
2939                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2940                 my $hostnodes = $hostnodes_start;
2941                 if (defined($hostnodes_end)) {
2942                     $hostnodes .= "-$hostnodes_end";
2943                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2944                 }
2945
2946                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2947                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2948                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2949                 }
2950
2951                 # policy
2952                 my $policy = $numa->{policy};
2953                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2954                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2955             }
2956
2957             push @$cmd, '-object', $numa_object;
2958             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2959         }
2960
2961         die "total memory for NUMA nodes must be equal to vm static memory\n"
2962             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2963
2964         #if no custom tology, we split memory and cores across numa nodes
2965         if(!$numa_totalmemory) {
2966
2967             my $numa_memory = ($static_memory / $sockets) . "M";
2968
2969             for (my $i = 0; $i < $sockets; $i++)  {
2970
2971                 my $cpustart = ($cores * $i);
2972                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2973                 my $cpus = $cpustart;
2974                 $cpus .= "-$cpuend" if $cpuend;
2975
2976                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2977                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2978             }
2979         }
2980     }
2981
2982     if ($hotplug_features->{memory}) {
2983         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2984             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2985             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2986             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2987
2988             #if dimm_memory is not aligned to dimm map
2989             if($current_size > $memory) {
2990                  $conf->{memory} = $current_size;
2991                  update_config_nolock($vmid, $conf, 1);
2992             }
2993         });
2994     }
2995
2996     push @$cmd, '-S' if $conf->{freeze};
2997
2998     # set keyboard layout
2999     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3000     push @$cmd, '-k', $kb if $kb;
3001
3002     # enable sound
3003     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3004     #push @$cmd, '-soundhw', 'es1370';
3005     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3006
3007     if($conf->{agent}) {
3008         my $qgasocket = qmp_socket($vmid, 1);
3009         my $pciaddr = print_pci_addr("qga0", $bridges);
3010         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3011         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3012         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3013     }
3014
3015     my $spice_port;
3016
3017     if ($qxlnum) {
3018         if ($qxlnum > 1) {
3019             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3020                 for(my $i = 1; $i < $qxlnum; $i++){
3021                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3022                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3023                 }
3024             } else {
3025                 # assume other OS works like Linux
3026                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3027                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3028             }
3029         }
3030
3031         my $pciaddr = print_pci_addr("spice", $bridges);
3032
3033         my $nodename = PVE::INotify::nodename();
3034         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3035         $spice_port = PVE::Tools::next_spice_port($pfamily);
3036
3037         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3038
3039         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3040         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3041         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3042     }
3043
3044     # enable balloon by default, unless explicitly disabled
3045     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3046         $pciaddr = print_pci_addr("balloon0", $bridges);
3047         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3048     }
3049
3050     if ($conf->{watchdog}) {
3051         my $wdopts = parse_watchdog($conf->{watchdog});
3052         $pciaddr = print_pci_addr("watchdog", $bridges);
3053         my $watchdog = $wdopts->{model} || 'i6300esb';
3054         push @$devices, '-device', "$watchdog$pciaddr";
3055         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3056     }
3057
3058     my $vollist = [];
3059     my $scsicontroller = {};
3060     my $ahcicontroller = {};
3061     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3062
3063     # Add iscsi initiator name if available
3064     if (my $initiator = get_initiator_name()) {
3065         push @$devices, '-iscsi', "initiator-name=$initiator";
3066     }
3067
3068     foreach_drive($conf, sub {
3069         my ($ds, $drive) = @_;
3070
3071         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3072             push @$vollist, $drive->{file};
3073         }
3074
3075         $use_virtio = 1 if $ds =~ m/^virtio/;
3076
3077         if (drive_is_cdrom ($drive)) {
3078             if ($bootindex_hash->{d}) {
3079                 $drive->{bootindex} = $bootindex_hash->{d};
3080                 $bootindex_hash->{d} += 1;
3081             }
3082         } else {
3083             if ($bootindex_hash->{c}) {
3084                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3085                 $bootindex_hash->{c} += 1;
3086             }
3087         }
3088
3089         if($drive->{interface} eq 'virtio'){
3090            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3091         }
3092
3093         if ($drive->{interface} eq 'scsi') {
3094
3095             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3096
3097             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3098             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3099
3100             my $iothread = '';
3101             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3102                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3103                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3104             }
3105
3106             my $queues = '';
3107             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3108                 $queues = ",num_queues=$drive->{queues}";
3109             }
3110
3111             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3112             $scsicontroller->{$controller}=1;
3113         }
3114
3115         if ($drive->{interface} eq 'sata') {
3116            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3117            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3118            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3119            $ahcicontroller->{$controller}=1;
3120         }
3121
3122         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3123         push @$devices, '-drive',$drive_cmd;
3124         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3125     });
3126
3127     for (my $i = 0; $i < $MAX_NETS; $i++) {
3128          next if !$conf->{"net$i"};
3129          my $d = parse_net($conf->{"net$i"});
3130          next if !$d;
3131
3132          $use_virtio = 1 if $d->{model} eq 'virtio';
3133
3134          if ($bootindex_hash->{n}) {
3135             $d->{bootindex} = $bootindex_hash->{n};
3136             $bootindex_hash->{n} += 1;
3137          }
3138
3139          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3140          push @$devices, '-netdev', $netdevfull;
3141
3142          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3143          push @$devices, '-device', $netdevicefull;
3144     }
3145
3146     if (!$q35) {
3147         # add pci bridges
3148         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3149            $bridges->{1} = 1;
3150            $bridges->{2} = 1;
3151         }
3152
3153         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3154
3155         while (my ($k, $v) = each %$bridges) {
3156             $pciaddr = print_pci_addr("pci.$k");
3157             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3158         }
3159     }
3160
3161     # add custom args
3162     if ($conf->{args}) {
3163         my $aa = PVE::Tools::split_args($conf->{args});
3164         push @$cmd, @$aa;
3165     }
3166
3167     push @$cmd, @$devices;
3168     push @$cmd, '-rtc', join(',', @$rtcFlags)
3169         if scalar(@$rtcFlags);
3170     push @$cmd, '-machine', join(',', @$machineFlags)
3171         if scalar(@$machineFlags);
3172     push @$cmd, '-global', join(',', @$globalFlags)
3173         if scalar(@$globalFlags);
3174
3175     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3176 }
3177
3178 sub vnc_socket {
3179     my ($vmid) = @_;
3180     return "${var_run_tmpdir}/$vmid.vnc";
3181 }
3182
3183 sub spice_port {
3184     my ($vmid) = @_;
3185
3186     my $res = vm_mon_cmd($vmid, 'query-spice');
3187
3188     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3189 }
3190
3191 sub qmp_socket {
3192     my ($vmid, $qga) = @_;
3193     my $sockettype = $qga ? 'qga' : 'qmp';
3194     return "${var_run_tmpdir}/$vmid.$sockettype";
3195 }
3196
3197 sub pidfile_name {
3198     my ($vmid) = @_;
3199     return "${var_run_tmpdir}/$vmid.pid";
3200 }
3201
3202 sub vm_devices_list {
3203     my ($vmid) = @_;
3204
3205     my $res = vm_mon_cmd($vmid, 'query-pci');
3206     my $devices = {};
3207     foreach my $pcibus (@$res) {
3208         foreach my $device (@{$pcibus->{devices}}) {
3209             next if !$device->{'qdev_id'};
3210             if ($device->{'pci_bridge'}) {
3211                 $devices->{$device->{'qdev_id'}} = 1;
3212                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3213                     next if !$bridge_device->{'qdev_id'};
3214                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3215                     $devices->{$device->{'qdev_id'}}++;
3216                 }
3217             } else {
3218                 $devices->{$device->{'qdev_id'}} = 1;
3219             }
3220         }
3221     }
3222
3223     my $resblock = vm_mon_cmd($vmid, 'query-block');
3224     foreach my $block (@$resblock) {
3225         if($block->{device} =~ m/^drive-(\S+)/){
3226                 $devices->{$1} = 1;
3227         }
3228     }
3229
3230     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3231     foreach my $mice (@$resmice) {
3232         if ($mice->{name} eq 'QEMU HID Tablet') {
3233             $devices->{tablet} = 1;
3234             last;
3235         }
3236     }
3237
3238     return $devices;
3239 }
3240
3241 sub vm_deviceplug {
3242     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3243
3244     my $q35 = machine_type_is_q35($conf);
3245
3246     my $devices_list = vm_devices_list($vmid);
3247     return 1 if defined($devices_list->{$deviceid});
3248
3249     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3250
3251     if ($deviceid eq 'tablet') {
3252
3253         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3254
3255     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3256
3257         qemu_iothread_add($vmid, $deviceid, $device);
3258
3259         qemu_driveadd($storecfg, $vmid, $device);
3260         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3261
3262         qemu_deviceadd($vmid, $devicefull);
3263         eval { qemu_deviceaddverify($vmid, $deviceid); };
3264         if (my $err = $@) {
3265             eval { qemu_drivedel($vmid, $deviceid); };
3266             warn $@ if $@;
3267             die $err;
3268         }
3269
3270     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3271
3272
3273         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3274         my $pciaddr = print_pci_addr($deviceid);
3275         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3276
3277         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3278
3279         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3280             qemu_iothread_add($vmid, $deviceid, $device);
3281             $devicefull .= ",iothread=iothread-$deviceid";
3282         }
3283
3284         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3285             $devicefull .= ",num_queues=$device->{queues}";
3286         }
3287
3288         qemu_deviceadd($vmid, $devicefull);
3289         qemu_deviceaddverify($vmid, $deviceid);
3290
3291     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3292
3293         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3294         qemu_driveadd($storecfg, $vmid, $device);
3295
3296         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3297         eval { qemu_deviceadd($vmid, $devicefull); };
3298         if (my $err = $@) {
3299             eval { qemu_drivedel($vmid, $deviceid); };
3300             warn $@ if $@;
3301             die $err;
3302         }
3303
3304     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3305
3306         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3307         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3308         qemu_deviceadd($vmid, $netdevicefull);
3309         eval { qemu_deviceaddverify($vmid, $deviceid); };
3310         if (my $err = $@) {
3311             eval { qemu_netdevdel($vmid, $deviceid); };
3312             warn $@ if $@;
3313             die $err;
3314         }
3315
3316     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3317
3318         my $bridgeid = $2;
3319         my $pciaddr = print_pci_addr($deviceid);
3320         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3321
3322         qemu_deviceadd($vmid, $devicefull);
3323         qemu_deviceaddverify($vmid, $deviceid);
3324
3325     } else {
3326         die "can't hotplug device '$deviceid'\n";
3327     }
3328
3329     return 1;
3330 }
3331
3332 # fixme: this should raise exceptions on error!
3333 sub vm_deviceunplug {
3334     my ($vmid, $conf, $deviceid) = @_;
3335
3336     my $devices_list = vm_devices_list($vmid);
3337     return 1 if !defined($devices_list->{$deviceid});
3338
3339     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3340
3341     if ($deviceid eq 'tablet') {
3342
3343         qemu_devicedel($vmid, $deviceid);
3344
3345     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3346
3347         qemu_devicedel($vmid, $deviceid);
3348         qemu_devicedelverify($vmid, $deviceid);
3349         qemu_drivedel($vmid, $deviceid);
3350         qemu_iothread_del($conf, $vmid, $deviceid);
3351
3352     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3353
3354         qemu_devicedel($vmid, $deviceid);
3355         qemu_devicedelverify($vmid, $deviceid);
3356         qemu_iothread_del($conf, $vmid, $deviceid);
3357
3358     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3359
3360         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3361         my $device = parse_drive($deviceid, $conf->{$deviceid});
3362         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3363
3364         qemu_devicedel($vmid, $deviceid);
3365         qemu_drivedel($vmid, $deviceid);
3366         qemu_deletescsihw($conf, $vmid, $deviceid);
3367
3368     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3369
3370         qemu_devicedel($vmid, $deviceid);
3371         qemu_devicedelverify($vmid, $deviceid);
3372         qemu_netdevdel($vmid, $deviceid);
3373
3374     } else {
3375         die "can't unplug device '$deviceid'\n";
3376     }
3377
3378     return 1;
3379 }
3380
3381 sub qemu_deviceadd {
3382     my ($vmid, $devicefull) = @_;
3383
3384     $devicefull = "driver=".$devicefull;
3385     my %options =  split(/[=,]/, $devicefull);
3386
3387     vm_mon_cmd($vmid, "device_add" , %options);
3388 }
3389
3390 sub qemu_devicedel {
3391     my ($vmid, $deviceid) = @_;
3392
3393     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3394 }
3395
3396 sub qemu_iothread_add {
3397     my($vmid, $deviceid, $device) = @_;
3398
3399     if ($device->{iothread}) {
3400         my $iothreads = vm_iothreads_list($vmid);
3401         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3402     }
3403 }
3404
3405 sub qemu_iothread_del {
3406     my($conf, $vmid, $deviceid) = @_;
3407
3408     my $device = parse_drive($deviceid, $conf->{$deviceid});
3409     if ($device->{iothread}) {
3410         my $iothreads = vm_iothreads_list($vmid);
3411         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3412     }
3413 }
3414
3415 sub qemu_objectadd {
3416     my($vmid, $objectid, $qomtype) = @_;
3417
3418     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3419
3420     return 1;
3421 }
3422
3423 sub qemu_objectdel {
3424     my($vmid, $objectid) = @_;
3425
3426     vm_mon_cmd($vmid, "object-del", id => $objectid);
3427
3428     return 1;
3429 }
3430
3431 sub qemu_driveadd {
3432     my ($storecfg, $vmid, $device) = @_;
3433
3434     my $drive = print_drive_full($storecfg, $vmid, $device);
3435     $drive =~ s/\\/\\\\/g;
3436     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3437
3438     # If the command succeeds qemu prints: "OK"
3439     return 1 if $ret =~ m/OK/s;
3440
3441     die "adding drive failed: $ret\n";
3442 }
3443
3444 sub qemu_drivedel {
3445     my($vmid, $deviceid) = @_;
3446
3447     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3448     $ret =~ s/^\s+//;
3449
3450     return 1 if $ret eq "";
3451
3452     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3453     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3454
3455     die "deleting drive $deviceid failed : $ret\n";
3456 }
3457
3458 sub qemu_deviceaddverify {
3459     my ($vmid, $deviceid) = @_;
3460
3461     for (my $i = 0; $i <= 5; $i++) {
3462          my $devices_list = vm_devices_list($vmid);
3463          return 1 if defined($devices_list->{$deviceid});
3464          sleep 1;
3465     }
3466
3467     die "error on hotplug device '$deviceid'\n";
3468 }
3469
3470
3471 sub qemu_devicedelverify {
3472     my ($vmid, $deviceid) = @_;
3473
3474     # need to verify that the device is correctly removed as device_del
3475     # is async and empty return is not reliable
3476
3477     for (my $i = 0; $i <= 5; $i++) {
3478          my $devices_list = vm_devices_list($vmid);
3479          return 1 if !defined($devices_list->{$deviceid});
3480          sleep 1;
3481     }
3482
3483     die "error on hot-unplugging device '$deviceid'\n";
3484 }
3485
3486 sub qemu_findorcreatescsihw {
3487     my ($storecfg, $conf, $vmid, $device) = @_;
3488
3489     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3490
3491     my $scsihwid="$controller_prefix$controller";
3492     my $devices_list = vm_devices_list($vmid);
3493
3494     if(!defined($devices_list->{$scsihwid})) {
3495         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3496     }
3497
3498     return 1;
3499 }
3500
3501 sub qemu_deletescsihw {
3502     my ($conf, $vmid, $opt) = @_;
3503
3504     my $device = parse_drive($opt, $conf->{$opt});
3505
3506     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3507         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3508         return 1;
3509     }
3510
3511     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3512
3513     my $devices_list = vm_devices_list($vmid);
3514     foreach my $opt (keys %{$devices_list}) {
3515         if (PVE::QemuServer::valid_drivename($opt)) {
3516             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3517             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3518                 return 1;
3519             }
3520         }
3521     }
3522
3523     my $scsihwid="scsihw$controller";
3524
3525     vm_deviceunplug($vmid, $conf, $scsihwid);
3526
3527     return 1;
3528 }
3529
3530 sub qemu_add_pci_bridge {
3531     my ($storecfg, $conf, $vmid, $device) = @_;
3532
3533     my $bridges = {};
3534
3535     my $bridgeid;
3536
3537     print_pci_addr($device, $bridges);
3538
3539     while (my ($k, $v) = each %$bridges) {
3540         $bridgeid = $k;
3541     }
3542     return 1 if !defined($bridgeid) || $bridgeid < 1;
3543
3544     my $bridge = "pci.$bridgeid";
3545     my $devices_list = vm_devices_list($vmid);
3546
3547     if (!defined($devices_list->{$bridge})) {
3548         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3549     }
3550
3551     return 1;
3552 }
3553
3554 sub qemu_set_link_status {
3555     my ($vmid, $device, $up) = @_;
3556
3557     vm_mon_cmd($vmid, "set_link", name => $device,
3558                up => $up ? JSON::true : JSON::false);
3559 }
3560
3561 sub qemu_netdevadd {
3562     my ($vmid, $conf, $device, $deviceid) = @_;
3563
3564     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3565     my %options =  split(/[=,]/, $netdev);
3566
3567     vm_mon_cmd($vmid, "netdev_add",  %options);
3568     return 1;
3569 }
3570
3571 sub qemu_netdevdel {
3572     my ($vmid, $deviceid) = @_;
3573
3574     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3575 }
3576
3577 sub qemu_cpu_hotplug {
3578     my ($vmid, $conf, $vcpus) = @_;
3579
3580     my $sockets = 1;
3581     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3582     $sockets = $conf->{sockets} if  $conf->{sockets};
3583     my $cores = $conf->{cores} || 1;
3584     my $maxcpus = $sockets * $cores;
3585
3586     $vcpus = $maxcpus if !$vcpus;
3587
3588     die "you can't add more vcpus than maxcpus\n"
3589         if $vcpus > $maxcpus;
3590
3591     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3592     die "online cpu unplug is not yet possible\n"
3593         if $vcpus < $currentvcpus;
3594
3595     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3596     die "vcpus in running vm is different than configuration\n"
3597         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3598
3599     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3600         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3601     }
3602 }
3603
3604 sub qemu_memory_hotplug {
3605     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3606
3607     return $value if !check_running($vmid);
3608
3609     my $memory = $conf->{memory} || $defaults->{memory};
3610     $value = $defaults->{memory} if !$value;
3611     return $value if $value == $memory;
3612
3613     my $static_memory = $STATICMEM;
3614     my $dimm_memory = $memory - $static_memory;
3615
3616     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3617     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3618
3619
3620     my $sockets = 1;
3621     $sockets = $conf->{sockets} if $conf->{sockets};
3622
3623     if($value > $memory) {
3624
3625         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3626             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3627
3628                 return if $current_size <= $conf->{memory};
3629
3630                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3631                 if (my $err = $@) {
3632                     eval { qemu_objectdel($vmid, "mem-$name"); };
3633                     die $err;
3634                 }
3635
3636                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3637                 if (my $err = $@) {
3638                     eval { qemu_objectdel($vmid, "mem-$name"); };
3639                     die $err;
3640                 }
3641                 #update conf after each succesful module hotplug
3642                 $conf->{memory} = $current_size;
3643                 update_config_nolock($vmid, $conf, 1);
3644         });
3645
3646     } else {
3647
3648         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3649             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3650
3651                 return if $current_size >= $conf->{memory};
3652                 print "try to unplug memory dimm $name\n";
3653
3654                 my $retry = 0;
3655                 while (1) {
3656                     eval { qemu_devicedel($vmid, $name) };
3657                     sleep 3;
3658                     my $dimm_list = qemu_dimm_list($vmid);
3659                     last if !$dimm_list->{$name};
3660                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3661                     $retry++;
3662                 }
3663
3664                 #update conf after each succesful module unplug
3665                 $conf->{memory} = $current_size;
3666
3667                 eval { qemu_objectdel($vmid, "mem-$name"); };
3668                 update_config_nolock($vmid, $conf, 1);
3669         });
3670     }
3671 }
3672
3673 sub qemu_dimm_list {
3674     my ($vmid) = @_;
3675
3676     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3677     my $dimms = {};
3678
3679     foreach my $dimm (@$dimmarray) {
3680
3681         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3682         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3683         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3684         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3685         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3686     }
3687     return $dimms;
3688 }
3689
3690 sub qemu_block_set_io_throttle {
3691     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3692
3693     return if !check_running($vmid) ;
3694
3695     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3696
3697 }
3698
3699 # old code, only used to shutdown old VM after update
3700 sub __read_avail {
3701     my ($fh, $timeout) = @_;
3702
3703     my $sel = new IO::Select;
3704     $sel->add($fh);
3705
3706     my $res = '';
3707     my $buf;
3708
3709     my @ready;
3710     while (scalar (@ready = $sel->can_read($timeout))) {
3711         my $count;
3712         if ($count = $fh->sysread($buf, 8192)) {
3713             if ($buf =~ /^(.*)\(qemu\) $/s) {
3714                 $res .= $1;
3715                 last;
3716             } else {
3717                 $res .= $buf;
3718             }
3719         } else {
3720             if (!defined($count)) {
3721                 die "$!\n";
3722             }
3723             last;
3724         }
3725     }
3726
3727     die "monitor read timeout\n" if !scalar(@ready);
3728
3729     return $res;
3730 }
3731
3732 # old code, only used to shutdown old VM after update
3733 sub vm_monitor_command {
3734     my ($vmid, $cmdstr, $nocheck) = @_;
3735
3736     my $res;
3737
3738     eval {
3739         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3740
3741         my $sname = "${var_run_tmpdir}/$vmid.mon";
3742
3743         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3744             die "unable to connect to VM $vmid socket - $!\n";
3745
3746         my $timeout = 3;
3747
3748         # hack: migrate sometime blocks the monitor (when migrate_downtime
3749         # is set)
3750         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3751             $timeout = 60*60; # 1 hour
3752         }
3753
3754         # read banner;
3755         my $data = __read_avail($sock, $timeout);
3756
3757         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3758             die "got unexpected qemu monitor banner\n";
3759         }
3760
3761         my $sel = new IO::Select;
3762         $sel->add($sock);
3763
3764         if (!scalar(my @ready = $sel->can_write($timeout))) {
3765             die "monitor write error - timeout";
3766         }
3767
3768         my $fullcmd = "$cmdstr\r";
3769
3770         # syslog('info', "VM $vmid monitor command: $cmdstr");
3771
3772         my $b;
3773         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3774             die "monitor write error - $!";
3775         }
3776
3777         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3778
3779         $timeout = 20;
3780
3781         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3782             $timeout = 60*60; # 1 hour
3783         } elsif ($cmdstr =~ m/^(eject|change)/) {
3784             $timeout = 60; # note: cdrom mount command is slow
3785         }
3786         if ($res = __read_avail($sock, $timeout)) {
3787
3788             my @lines = split("\r?\n", $res);
3789
3790             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3791
3792             $res = join("\n", @lines);
3793             $res .= "\n";
3794         }
3795     };
3796
3797     my $err = $@;
3798
3799     if ($err) {
3800         syslog("err", "VM $vmid monitor command failed - $err");
3801         die $err;
3802     }
3803
3804     return $res;
3805 }
3806
3807 sub qemu_block_resize {
3808     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3809
3810     my $running = check_running($vmid);
3811
3812     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3813
3814     return if !$running;
3815
3816     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3817
3818 }
3819
3820 sub qemu_volume_snapshot {
3821     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3822
3823     my $running = check_running($vmid);
3824
3825     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3826         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3827     } else {
3828         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3829     }
3830 }
3831
3832 sub qemu_volume_snapshot_delete {
3833     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3834
3835     my $running = check_running($vmid);
3836
3837     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3838
3839     return if !$running;
3840
3841     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3842 }
3843
3844 sub set_migration_caps {
3845     my ($vmid) = @_;
3846
3847     my $cap_ref = [];
3848
3849     my $enabled_cap = {
3850         "auto-converge" => 1,
3851         "xbzrle" => 0,
3852         "x-rdma-pin-all" => 0,
3853         "zero-blocks" => 0,
3854     };
3855
3856     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3857
3858     for my $supported_capability (@$supported_capabilities) {
3859         push @$cap_ref, {
3860             capability => $supported_capability->{capability},
3861             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3862         };
3863     }
3864
3865     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3866 }
3867
3868 my $fast_plug_option = {
3869     'lock' => 1,
3870     'name' => 1,
3871     'onboot' => 1,
3872     'shares' => 1,
3873     'startup' => 1,
3874     'description' => 1,
3875 };
3876
3877 # hotplug changes in [PENDING]
3878 # $selection hash can be used to only apply specified options, for
3879 # example: { cores => 1 } (only apply changed 'cores')
3880 # $errors ref is used to return error messages
3881 sub vmconfig_hotplug_pending {
3882     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3883
3884     my $defaults = load_defaults();
3885
3886     # commit values which do not have any impact on running VM first
3887     # Note: those option cannot raise errors, we we do not care about
3888     # $selection and always apply them.
3889
3890     my $add_error = sub {
3891         my ($opt, $msg) = @_;
3892         $errors->{$opt} = "hotplug problem - $msg";
3893     };
3894
3895     my $changes = 0;
3896     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3897         if ($fast_plug_option->{$opt}) {
3898             $conf->{$opt} = $conf->{pending}->{$opt};
3899             delete $conf->{pending}->{$opt};
3900             $changes = 1;
3901         }
3902     }
3903
3904     if ($changes) {
3905         update_config_nolock($vmid, $conf, 1);
3906         $conf = load_config($vmid); # update/reload
3907     }
3908
3909     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3910
3911     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3912     foreach my $opt (@delete) {
3913         next if $selection && !$selection->{$opt};
3914         eval {
3915             if ($opt eq 'hotplug') {
3916                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3917             } elsif ($opt eq 'tablet') {
3918                 die "skip\n" if !$hotplug_features->{usb};
3919                 if ($defaults->{tablet}) {
3920                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3921                 } else {
3922                     vm_deviceunplug($vmid, $conf, $opt);
3923                 }
3924             } elsif ($opt eq 'vcpus') {
3925                 die "skip\n" if !$hotplug_features->{cpu};
3926                 qemu_cpu_hotplug($vmid, $conf, undef);
3927             } elsif ($opt eq 'balloon') {
3928                 # enable balloon device is not hotpluggable
3929                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3930             } elsif ($fast_plug_option->{$opt}) {
3931                 # do nothing
3932             } elsif ($opt =~ m/^net(\d+)$/) {
3933                 die "skip\n" if !$hotplug_features->{network};
3934                 vm_deviceunplug($vmid, $conf, $opt);
3935             } elsif (valid_drivename($opt)) {
3936                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3937                 vm_deviceunplug($vmid, $conf, $opt);
3938                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3939             } elsif ($opt =~ m/^memory$/) {
3940                 die "skip\n" if !$hotplug_features->{memory};
3941                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3942             } elsif ($opt eq 'cpuunits') {
3943                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3944             } elsif ($opt eq 'cpulimit') {
3945                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3946             } else {
3947                 die "skip\n";
3948             }
3949         };
3950         if (my $err = $@) {
3951             &$add_error($opt, $err) if $err ne "skip\n";
3952         } else {
3953             # save new config if hotplug was successful
3954             delete $conf->{$opt};
3955             vmconfig_undelete_pending_option($conf, $opt);
3956             update_config_nolock($vmid, $conf, 1);
3957             $conf = load_config($vmid); # update/reload
3958         }
3959     }
3960
3961     foreach my $opt (keys %{$conf->{pending}}) {
3962         next if $selection && !$selection->{$opt};
3963         my $value = $conf->{pending}->{$opt};
3964         eval {
3965             if ($opt eq 'hotplug') {
3966                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3967             } elsif ($opt eq 'tablet') {
3968                 die "skip\n" if !$hotplug_features->{usb};
3969                 if ($value == 1) {
3970                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3971                 } elsif ($value == 0) {
3972                     vm_deviceunplug($vmid, $conf, $opt);
3973                 }
3974             } elsif ($opt eq 'vcpus') {
3975                 die "skip\n" if !$hotplug_features->{cpu};
3976                 qemu_cpu_hotplug($vmid, $conf, $value);
3977             } elsif ($opt eq 'balloon') {
3978                 # enable/disable balloning device is not hotpluggable
3979                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3980                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3981                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3982
3983                 # allow manual ballooning if shares is set to zero
3984                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3985                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3986                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3987                 }
3988             } elsif ($opt =~ m/^net(\d+)$/) {
3989                 # some changes can be done without hotplug
3990                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3991                                     $vmid, $opt, $value);
3992             } elsif (valid_drivename($opt)) {
3993                 # some changes can be done without hotplug
3994                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3995                                      $vmid, $opt, $value, 1);
3996             } elsif ($opt =~ m/^memory$/) { #dimms
3997                 die "skip\n" if !$hotplug_features->{memory};
3998                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3999             } elsif ($opt eq 'cpuunits') {
4000                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4001             } elsif ($opt eq 'cpulimit') {
4002                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4003                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4004             } else {
4005                 die "skip\n";  # skip non-hot-pluggable options
4006             }
4007         };
4008         if (my $err = $@) {
4009             &$add_error($opt, $err) if $err ne "skip\n";
4010         } else {
4011             # save new config if hotplug was successful
4012             $conf->{$opt} = $value;
4013             delete $conf->{pending}->{$opt};
4014             update_config_nolock($vmid, $conf, 1);
4015             $conf = load_config($vmid); # update/reload
4016         }
4017     }
4018 }
4019
4020 sub vmconfig_apply_pending {
4021     my ($vmid, $conf, $storecfg) = @_;
4022
4023     # cold plug
4024
4025     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
4026     foreach my $opt (@delete) { # delete
4027         die "internal error" if $opt =~ m/^unused/;
4028         $conf = load_config($vmid); # update/reload
4029         if (!defined($conf->{$opt})) {
4030             vmconfig_undelete_pending_option($conf, $opt);
4031             update_config_nolock($vmid, $conf, 1);
4032         } elsif (valid_drivename($opt)) {
4033             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
4034             vmconfig_undelete_pending_option($conf, $opt);
4035             delete $conf->{$opt};
4036             update_config_nolock($vmid, $conf, 1);
4037         } else {
4038             vmconfig_undelete_pending_option($conf, $opt);
4039             delete $conf->{$opt};
4040             update_config_nolock($vmid, $conf, 1);
4041         }
4042     }
4043
4044     $conf = load_config($vmid); # update/reload
4045
4046     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4047         $conf = load_config($vmid); # update/reload
4048
4049         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4050             # skip if nothing changed
4051         } elsif (valid_drivename($opt)) {
4052             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4053                 if defined($conf->{$opt});
4054             $conf->{$opt} = $conf->{pending}->{$opt};
4055         } else {
4056             $conf->{$opt} = $conf->{pending}->{$opt};
4057         }
4058
4059         delete $conf->{pending}->{$opt};
4060         update_config_nolock($vmid, $conf, 1);
4061     }
4062 }
4063
4064 my $safe_num_ne = sub {
4065     my ($a, $b) = @_;
4066
4067     return 0 if !defined($a) && !defined($b);
4068     return 1 if !defined($a);
4069     return 1 if !defined($b);
4070
4071     return $a != $b;
4072 };
4073
4074 my $safe_string_ne = sub {
4075     my ($a, $b) = @_;
4076
4077     return 0 if !defined($a) && !defined($b);
4078     return 1 if !defined($a);
4079     return 1 if !defined($b);
4080
4081     return $a ne $b;
4082 };
4083
4084 sub vmconfig_update_net {
4085     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4086
4087     my $newnet = parse_net($value);
4088
4089     if ($conf->{$opt}) {
4090         my $oldnet = parse_net($conf->{$opt});
4091
4092         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4093             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4094             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4095             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4096
4097             # for non online change, we try to hot-unplug
4098             die "skip\n" if !$hotplug;
4099             vm_deviceunplug($vmid, $conf, $opt);
4100         } else {
4101
4102             die "internal error" if $opt !~ m/net(\d+)/;
4103             my $iface = "tap${vmid}i$1";
4104
4105             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4106                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4107             }
4108
4109             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4110                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4111                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4112                 PVE::Network::tap_unplug($iface);
4113                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4114             }
4115
4116             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4117                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4118             }
4119
4120             return 1;
4121         }
4122     }
4123
4124     if ($hotplug) {
4125         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4126     } else {
4127         die "skip\n";
4128     }
4129 }
4130
4131 sub vmconfig_update_disk {
4132     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4133
4134     # fixme: do we need force?
4135
4136     my $drive = parse_drive($opt, $value);
4137
4138     if ($conf->{$opt}) {
4139
4140         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4141
4142             my $media = $drive->{media} || 'disk';
4143             my $oldmedia = $old_drive->{media} || 'disk';
4144             die "unable to change media type\n" if $media ne $oldmedia;
4145
4146             if (!drive_is_cdrom($old_drive)) {
4147
4148                 if ($drive->{file} ne $old_drive->{file}) {
4149
4150                     die "skip\n" if !$hotplug;
4151
4152                     # unplug and register as unused
4153                     vm_deviceunplug($vmid, $conf, $opt);
4154                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4155
4156                 } else {
4157                     # update existing disk
4158
4159                     # skip non hotpluggable value
4160                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4161                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4162                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4163                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4164                         die "skip\n";
4165                     }
4166
4167                     # apply throttle
4168                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4169                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4170                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4171                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4172                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4173                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4174                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4175                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4176                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4177                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4178                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4179                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4180
4181                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4182                                                    ($drive->{mbps} || 0)*1024*1024,
4183                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4184                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4185                                                    $drive->{iops} || 0,
4186                                                    $drive->{iops_rd} || 0,
4187                                                    $drive->{iops_wr} || 0,
4188                                                    ($drive->{mbps_max} || 0)*1024*1024,
4189                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4190                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4191                                                    $drive->{iops_max} || 0,
4192                                                    $drive->{iops_rd_max} || 0,
4193                                                    $drive->{iops_wr_max} || 0);
4194
4195                     }
4196
4197                     return 1;
4198                 }
4199
4200             } else { # cdrom
4201
4202                 if ($drive->{file} eq 'none') {
4203                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4204                 } else {
4205                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4206                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4207                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4208                 }
4209
4210                 return 1;
4211             }
4212         }
4213     }
4214
4215     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4216     # hotplug new disks
4217     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4218 }
4219
4220 sub vm_start {
4221     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4222
4223     lock_config($vmid, sub {
4224         my $conf = load_config($vmid, $migratedfrom);
4225
4226         die "you can't start a vm if it's a template\n" if is_template($conf);
4227
4228         check_lock($conf) if !$skiplock;
4229
4230         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4231
4232         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4233             vmconfig_apply_pending($vmid, $conf, $storecfg);
4234             $conf = load_config($vmid); # update/reload
4235         }
4236
4237         my $defaults = load_defaults();
4238
4239         # set environment variable useful inside network script
4240         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4241
4242         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4243
4244         my $migrate_port = 0;
4245         my $migrate_uri;
4246         if ($statefile) {
4247             if ($statefile eq 'tcp') {
4248                 my $localip = "localhost";
4249                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4250                 my $nodename = PVE::INotify::nodename();
4251                 if ($datacenterconf->{migration_unsecure}) {
4252                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4253                 }
4254                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4255                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4256                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4257                 push @$cmd, '-incoming', $migrate_uri;
4258                 push @$cmd, '-S';
4259             } else {
4260                 push @$cmd, '-loadstate', $statefile;
4261             }
4262         } elsif ($paused) {
4263             push @$cmd, '-S';
4264         }
4265
4266         # host pci devices
4267         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4268           my $d = parse_hostpci($conf->{"hostpci$i"});
4269           next if !$d;
4270           my $pcidevices = $d->{pciid};
4271           foreach my $pcidevice (@$pcidevices) {
4272                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4273
4274                 my $info = pci_device_info("0000:$pciid");
4275                 die "IOMMU not present\n" if !check_iommu_support();
4276                 die "no pci device info for device '$pciid'\n" if !$info;
4277
4278                 if ($d->{driver} && $d->{driver} eq "vfio") {
4279                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4280                 } else {
4281                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4282                 }
4283
4284                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4285           }
4286         }
4287
4288         PVE::Storage::activate_volumes($storecfg, $vollist);
4289
4290         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4291                     umask => 0077); };
4292         my $err = $@;
4293         die "start failed: $err" if $err;
4294
4295         print "migration listens on $migrate_uri\n" if $migrate_uri;
4296
4297         if ($statefile && $statefile ne 'tcp')  {
4298             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4299             warn $@ if $@;
4300         }
4301
4302         if ($migratedfrom) {
4303
4304             eval {
4305                 set_migration_caps($vmid);
4306             };
4307             warn $@ if $@;
4308
4309             if ($spice_port) {
4310                 print "spice listens on port $spice_port\n";
4311                 if ($spice_ticket) {
4312                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4313                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4314                 }
4315             }
4316
4317         } else {
4318
4319             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4320                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4321                     if $conf->{balloon};
4322             }
4323
4324             foreach my $opt (keys %$conf) {
4325                 next if $opt !~  m/^net\d+$/;
4326                 my $nicconf = parse_net($conf->{$opt});
4327                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4328             }
4329         }
4330
4331         vm_mon_cmd_nocheck($vmid, 'qom-set',
4332                     path => "machine/peripheral/balloon0",
4333                     property => "guest-stats-polling-interval",
4334                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4335
4336     });
4337 }
4338
4339 sub vm_mon_cmd {
4340     my ($vmid, $execute, %params) = @_;
4341
4342     my $cmd = { execute => $execute, arguments => \%params };
4343     vm_qmp_command($vmid, $cmd);
4344 }
4345
4346 sub vm_mon_cmd_nocheck {
4347     my ($vmid, $execute, %params) = @_;
4348
4349     my $cmd = { execute => $execute, arguments => \%params };
4350     vm_qmp_command($vmid, $cmd, 1);
4351 }
4352
4353 sub vm_qmp_command {
4354     my ($vmid, $cmd, $nocheck) = @_;
4355
4356     my $res;
4357
4358     my $timeout;
4359     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4360         $timeout = $cmd->{arguments}->{timeout};
4361         delete $cmd->{arguments}->{timeout};
4362     }
4363
4364     eval {
4365         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4366         my $sname = qmp_socket($vmid);
4367         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4368             my $qmpclient = PVE::QMPClient->new();
4369
4370             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4371         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4372             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4373                 if scalar(%{$cmd->{arguments}});
4374             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4375         } else {
4376             die "unable to open monitor socket\n";
4377         }
4378     };
4379     if (my $err = $@) {
4380         syslog("err", "VM $vmid qmp command failed - $err");
4381         die $err;
4382     }
4383
4384     return $res;
4385 }
4386
4387 sub vm_human_monitor_command {
4388     my ($vmid, $cmdline) = @_;
4389
4390     my $res;
4391
4392     my $cmd = {
4393         execute => 'human-monitor-command',
4394         arguments => { 'command-line' => $cmdline},
4395     };
4396
4397     return vm_qmp_command($vmid, $cmd);
4398 }
4399
4400 sub vm_commandline {
4401     my ($storecfg, $vmid) = @_;
4402
4403     my $conf = load_config($vmid);
4404
4405     my $defaults = load_defaults();
4406
4407     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4408
4409     return join(' ', @$cmd);
4410 }
4411
4412 sub vm_reset {
4413     my ($vmid, $skiplock) = @_;
4414
4415     lock_config($vmid, sub {
4416
4417         my $conf = load_config($vmid);
4418
4419         check_lock($conf) if !$skiplock;
4420
4421         vm_mon_cmd($vmid, "system_reset");
4422     });
4423 }
4424
4425 sub get_vm_volumes {
4426     my ($conf) = @_;
4427
4428     my $vollist = [];
4429     foreach_volid($conf, sub {
4430         my ($volid, $is_cdrom) = @_;
4431
4432         return if $volid =~ m|^/|;
4433
4434         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4435         return if !$sid;
4436
4437         push @$vollist, $volid;
4438     });
4439
4440     return $vollist;
4441 }
4442
4443 sub vm_stop_cleanup {
4444     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4445
4446     eval {
4447
4448         if (!$keepActive) {
4449             my $vollist = get_vm_volumes($conf);
4450             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4451         }
4452
4453         foreach my $ext (qw(mon qmp pid vnc qga)) {
4454             unlink "/var/run/qemu-server/${vmid}.$ext";
4455         }
4456
4457         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4458     };
4459     warn $@ if $@; # avoid errors - just warn
4460 }
4461
4462 # Note: use $nockeck to skip tests if VM configuration file exists.
4463 # We need that when migration VMs to other nodes (files already moved)
4464 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4465 sub vm_stop {
4466     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4467
4468     $force = 1 if !defined($force) && !$shutdown;
4469
4470     if ($migratedfrom){
4471         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4472         kill 15, $pid if $pid;
4473         my $conf = load_config($vmid, $migratedfrom);
4474         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4475         return;
4476     }
4477
4478     lock_config($vmid, sub {
4479
4480         my $pid = check_running($vmid, $nocheck);
4481         return if !$pid;
4482
4483         my $conf;
4484         if (!$nocheck) {
4485             $conf = load_config($vmid);
4486             check_lock($conf) if !$skiplock;
4487             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4488                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4489                 $timeout = $opts->{down} if $opts->{down};
4490             }
4491         }
4492
4493         $timeout = 60 if !defined($timeout);
4494
4495         eval {
4496             if ($shutdown) {
4497                 if (defined($conf) && $conf->{agent}) {
4498                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4499                 } else {
4500                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4501                 }
4502             } else {
4503                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4504             }
4505         };
4506         my $err = $@;
4507
4508         if (!$err) {
4509             my $count = 0;
4510             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4511                 $count++;
4512                 sleep 1;
4513             }
4514
4515             if ($count >= $timeout) {
4516                 if ($force) {
4517                     warn "VM still running - terminating now with SIGTERM\n";
4518                     kill 15, $pid;
4519                 } else {
4520                     die "VM quit/powerdown failed - got timeout\n";
4521                 }
4522             } else {
4523                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4524                 return;
4525             }
4526         } else {
4527             if ($force) {
4528                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4529                 kill 15, $pid;
4530             } else {
4531                 die "VM quit/powerdown failed\n";
4532             }
4533         }
4534
4535         # wait again
4536         $timeout = 10;
4537
4538         my $count = 0;
4539         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4540             $count++;
4541             sleep 1;
4542         }
4543
4544         if ($count >= $timeout) {
4545             warn "VM still running - terminating now with SIGKILL\n";
4546             kill 9, $pid;
4547             sleep 1;
4548         }
4549
4550         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4551    });
4552 }
4553
4554 sub vm_suspend {
4555     my ($vmid, $skiplock) = @_;
4556
4557     lock_config($vmid, sub {
4558
4559         my $conf = load_config($vmid);
4560
4561         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4562
4563         vm_mon_cmd($vmid, "stop");
4564     });
4565 }
4566
4567 sub vm_resume {
4568     my ($vmid, $skiplock) = @_;
4569
4570     lock_config($vmid, sub {
4571
4572         my $conf = load_config($vmid);
4573
4574         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4575
4576         vm_mon_cmd($vmid, "cont");
4577     });
4578 }
4579
4580 sub vm_sendkey {
4581     my ($vmid, $skiplock, $key) = @_;
4582
4583     lock_config($vmid, sub {
4584
4585         my $conf = load_config($vmid);
4586
4587         # there is no qmp command, so we use the human monitor command
4588         vm_human_monitor_command($vmid, "sendkey $key");
4589     });
4590 }
4591
4592 sub vm_destroy {
4593     my ($storecfg, $vmid, $skiplock) = @_;
4594
4595     lock_config($vmid, sub {
4596
4597         my $conf = load_config($vmid);
4598
4599         check_lock($conf) if !$skiplock;
4600
4601         if (!check_running($vmid)) {
4602             destroy_vm($storecfg, $vmid);
4603         } else {
4604             die "VM $vmid is running - destroy failed\n";
4605         }
4606     });
4607 }
4608
4609 # pci helpers
4610
4611 sub file_write {
4612     my ($filename, $buf) = @_;
4613
4614     my $fh = IO::File->new($filename, "w");
4615     return undef if !$fh;
4616
4617     my $res = print $fh $buf;
4618
4619     $fh->close();
4620
4621     return $res;
4622 }
4623
4624 sub pci_device_info {
4625     my ($name) = @_;
4626
4627     my $res;
4628
4629     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4630     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4631
4632     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4633     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4634
4635     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4636     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4637
4638     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4639     return undef if !defined($product) || $product !~ s/^0x//;
4640
4641     $res = {
4642         name => $name,
4643         vendor => $vendor,
4644         product => $product,
4645         domain => $domain,
4646         bus => $bus,
4647         slot => $slot,
4648         func => $func,
4649         irq => $irq,
4650         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4651     };
4652
4653     return $res;
4654 }
4655
4656 sub pci_dev_reset {
4657     my ($dev) = @_;
4658
4659     my $name = $dev->{name};
4660
4661     my $fn = "$pcisysfs/devices/$name/reset";
4662
4663     return file_write($fn, "1");
4664 }
4665
4666 sub pci_dev_bind_to_stub {
4667     my ($dev) = @_;
4668
4669     my $name = $dev->{name};
4670
4671     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4672     return 1 if -d $testdir;
4673
4674     my $data = "$dev->{vendor} $dev->{product}";
4675     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4676
4677     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4678     if (!file_write($fn, $name)) {
4679         return undef if -f $fn;
4680     }
4681
4682     $fn = "$pcisysfs/drivers/pci-stub/bind";
4683     if (! -d $testdir) {
4684         return undef if !file_write($fn, $name);
4685     }
4686
4687     return -d $testdir;
4688 }
4689
4690 sub pci_dev_bind_to_vfio {
4691     my ($dev) = @_;
4692
4693     my $name = $dev->{name};
4694
4695     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4696
4697     if (!-d $vfio_basedir) {
4698         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4699     }
4700     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4701
4702     my $testdir = "$vfio_basedir/$name";
4703     return 1 if -d $testdir;
4704
4705     my $data = "$dev->{vendor} $dev->{product}";
4706     return undef if !file_write("$vfio_basedir/new_id", $data);
4707
4708     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4709     if (!file_write($fn, $name)) {
4710         return undef if -f $fn;
4711     }
4712
4713     $fn = "$vfio_basedir/bind";
4714     if (! -d $testdir) {
4715         return undef if !file_write($fn, $name);
4716     }
4717
4718     return -d $testdir;
4719 }
4720
4721 sub pci_dev_group_bind_to_vfio {
4722     my ($pciid) = @_;
4723
4724     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4725
4726     if (!-d $vfio_basedir) {
4727         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4728     }
4729     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4730
4731     # get IOMMU group devices
4732     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4733       my @devs = grep /^0000:/, readdir($D);
4734     closedir($D);
4735
4736     foreach my $pciid (@devs) {
4737         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4738
4739         # pci bridges, switches or root ports are not supported
4740         # they have a pci_bus subdirectory so skip them
4741         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4742
4743         my $info = pci_device_info($1);
4744         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4745     }
4746
4747     return 1;
4748 }
4749
4750 sub print_pci_addr {
4751     my ($id, $bridges) = @_;
4752
4753     my $res = '';
4754     my $devices = {
4755         piix3 => { bus => 0, addr => 1 },
4756         #addr2 : first videocard
4757         balloon0 => { bus => 0, addr => 3 },
4758         watchdog => { bus => 0, addr => 4 },
4759         scsihw0 => { bus => 0, addr => 5 },
4760         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4761         scsihw1 => { bus => 0, addr => 6 },
4762         ahci0 => { bus => 0, addr => 7 },
4763         qga0 => { bus => 0, addr => 8 },
4764         spice => { bus => 0, addr => 9 },
4765         virtio0 => { bus => 0, addr => 10 },
4766         virtio1 => { bus => 0, addr => 11 },
4767         virtio2 => { bus => 0, addr => 12 },
4768         virtio3 => { bus => 0, addr => 13 },
4769         virtio4 => { bus => 0, addr => 14 },
4770         virtio5 => { bus => 0, addr => 15 },
4771         hostpci0 => { bus => 0, addr => 16 },
4772         hostpci1 => { bus => 0, addr => 17 },
4773         net0 => { bus => 0, addr => 18 },
4774         net1 => { bus => 0, addr => 19 },
4775         net2 => { bus => 0, addr => 20 },
4776         net3 => { bus => 0, addr => 21 },
4777         net4 => { bus => 0, addr => 22 },
4778         net5 => { bus => 0, addr => 23 },
4779         vga1 => { bus => 0, addr => 24 },
4780         vga2 => { bus => 0, addr => 25 },
4781         vga3 => { bus => 0, addr => 26 },
4782         hostpci2 => { bus => 0, addr => 27 },
4783         hostpci3 => { bus => 0, addr => 28 },
4784         #addr29 : usb-host (pve-usb.cfg)
4785         'pci.1' => { bus => 0, addr => 30 },
4786         'pci.2' => { bus => 0, addr => 31 },
4787         'net6' => { bus => 1, addr => 1 },
4788         'net7' => { bus => 1, addr => 2 },
4789         'net8' => { bus => 1, addr => 3 },
4790         'net9' => { bus => 1, addr => 4 },
4791         'net10' => { bus => 1, addr => 5 },
4792         'net11' => { bus => 1, addr => 6 },
4793         'net12' => { bus => 1, addr => 7 },
4794         'net13' => { bus => 1, addr => 8 },
4795         'net14' => { bus => 1, addr => 9 },
4796         'net15' => { bus => 1, addr => 10 },
4797         'net16' => { bus => 1, addr => 11 },
4798         'net17' => { bus => 1, addr => 12 },
4799         'net18' => { bus => 1, addr => 13 },
4800         'net19' => { bus => 1, addr => 14 },
4801         'net20' => { bus => 1, addr => 15 },
4802         'net21' => { bus => 1, addr => 16 },
4803         'net22' => { bus => 1, addr => 17 },
4804         'net23' => { bus => 1, addr => 18 },
4805         'net24' => { bus => 1, addr => 19 },
4806         'net25' => { bus => 1, addr => 20 },
4807         'net26' => { bus => 1, addr => 21 },
4808         'net27' => { bus => 1, addr => 22 },
4809         'net28' => { bus => 1, addr => 23 },
4810         'net29' => { bus => 1, addr => 24 },
4811         'net30' => { bus => 1, addr => 25 },
4812         'net31' => { bus => 1, addr => 26 },
4813         'virtio6' => { bus => 2, addr => 1 },
4814         'virtio7' => { bus => 2, addr => 2 },
4815         'virtio8' => { bus => 2, addr => 3 },
4816         'virtio9' => { bus => 2, addr => 4 },
4817         'virtio10' => { bus => 2, addr => 5 },
4818         'virtio11' => { bus => 2, addr => 6 },
4819         'virtio12' => { bus => 2, addr => 7 },
4820         'virtio13' => { bus => 2, addr => 8 },
4821         'virtio14' => { bus => 2, addr => 9 },
4822         'virtio15' => { bus => 2, addr => 10 },
4823         'virtioscsi0' => { bus => 3, addr => 1 },
4824         'virtioscsi1' => { bus => 3, addr => 2 },
4825         'virtioscsi2' => { bus => 3, addr => 3 },
4826         'virtioscsi3' => { bus => 3, addr => 4 },
4827         'virtioscsi4' => { bus => 3, addr => 5 },
4828         'virtioscsi5' => { bus => 3, addr => 6 },
4829         'virtioscsi6' => { bus => 3, addr => 7 },
4830         'virtioscsi7' => { bus => 3, addr => 8 },
4831         'virtioscsi8' => { bus => 3, addr => 9 },
4832         'virtioscsi9' => { bus => 3, addr => 10 },
4833         'virtioscsi10' => { bus => 3, addr => 11 },
4834         'virtioscsi11' => { bus => 3, addr => 12 },
4835         'virtioscsi12' => { bus => 3, addr => 13 },
4836         'virtioscsi13' => { bus => 3, addr => 14 },
4837         'virtioscsi14' => { bus => 3, addr => 15 },
4838         'virtioscsi15' => { bus => 3, addr => 16 },
4839         'virtioscsi16' => { bus => 3, addr => 17 },
4840         'virtioscsi17' => { bus => 3, addr => 18 },
4841         'virtioscsi18' => { bus => 3, addr => 19 },
4842         'virtioscsi19' => { bus => 3, addr => 20 },
4843         'virtioscsi20' => { bus => 3, addr => 21 },
4844         'virtioscsi21' => { bus => 3, addr => 22 },
4845         'virtioscsi22' => { bus => 3, addr => 23 },
4846         'virtioscsi23' => { bus => 3, addr => 24 },
4847         'virtioscsi24' => { bus => 3, addr => 25 },
4848         'virtioscsi25' => { bus => 3, addr => 26 },
4849         'virtioscsi26' => { bus => 3, addr => 27 },
4850         'virtioscsi27' => { bus => 3, addr => 28 },
4851         'virtioscsi28' => { bus => 3, addr => 29 },
4852         'virtioscsi29' => { bus => 3, addr => 30 },
4853         'virtioscsi30' => { bus => 3, addr => 31 },
4854
4855     };
4856
4857     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4858            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4859            my $bus = $devices->{$id}->{bus};
4860            $res = ",bus=pci.$bus,addr=$addr";
4861            $bridges->{$bus} = 1 if $bridges;
4862     }
4863     return $res;
4864
4865 }
4866
4867 sub print_pcie_addr {
4868     my ($id) = @_;
4869
4870     my $res = '';
4871     my $devices = {
4872         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4873         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4874         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4875         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4876     };
4877
4878     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4879            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4880            my $bus = $devices->{$id}->{bus};
4881            $res = ",bus=$bus,addr=$addr";
4882     }
4883     return $res;
4884
4885 }
4886
4887 # vzdump restore implementaion
4888
4889 sub tar_archive_read_firstfile {
4890     my $archive = shift;
4891
4892     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4893
4894     # try to detect archive type first
4895     my $pid = open (TMP, "tar tf '$archive'|") ||
4896         die "unable to open file '$archive'\n";
4897     my $firstfile = <TMP>;
4898     kill 15, $pid;
4899     close TMP;
4900
4901     die "ERROR: archive contaions no data\n" if !$firstfile;
4902     chomp $firstfile;
4903
4904     return $firstfile;
4905 }
4906
4907 sub tar_restore_cleanup {
4908     my ($storecfg, $statfile) = @_;
4909
4910     print STDERR "starting cleanup\n";
4911
4912     if (my $fd = IO::File->new($statfile, "r")) {
4913         while (defined(my $line = <$fd>)) {
4914             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4915                 my $volid = $2;
4916                 eval {
4917                     if ($volid =~ m|^/|) {
4918                         unlink $volid || die 'unlink failed\n';
4919                     } else {
4920                         PVE::Storage::vdisk_free($storecfg, $volid);
4921                     }
4922                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4923                 };
4924                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4925             } else {
4926                 print STDERR "unable to parse line in statfile - $line";
4927             }
4928         }
4929         $fd->close();
4930     }
4931 }
4932
4933 sub restore_archive {
4934     my ($archive, $vmid, $user, $opts) = @_;
4935
4936     my $format = $opts->{format};
4937     my $comp;
4938
4939     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4940         $format = 'tar' if !$format;
4941         $comp = 'gzip';
4942     } elsif ($archive =~ m/\.tar$/) {
4943         $format = 'tar' if !$format;
4944     } elsif ($archive =~ m/.tar.lzo$/) {
4945         $format = 'tar' if !$format;
4946         $comp = 'lzop';
4947     } elsif ($archive =~ m/\.vma$/) {
4948         $format = 'vma' if !$format;
4949     } elsif ($archive =~ m/\.vma\.gz$/) {
4950         $format = 'vma' if !$format;
4951         $comp = 'gzip';
4952     } elsif ($archive =~ m/\.vma\.lzo$/) {
4953         $format = 'vma' if !$format;
4954         $comp = 'lzop';
4955     } else {
4956         $format = 'vma' if !$format; # default
4957     }
4958
4959     # try to detect archive format
4960     if ($format eq 'tar') {
4961         return restore_tar_archive($archive, $vmid, $user, $opts);
4962     } else {
4963         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4964     }
4965 }
4966
4967 sub restore_update_config_line {
4968     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4969
4970     return if $line =~ m/^\#qmdump\#/;
4971     return if $line =~ m/^\#vzdump\#/;
4972     return if $line =~ m/^lock:/;
4973     return if $line =~ m/^unused\d+:/;
4974     return if $line =~ m/^parent:/;
4975     return if $line =~ m/^template:/; # restored VM is never a template
4976
4977     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4978         # try to convert old 1.X settings
4979         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4980         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4981             my ($model, $macaddr) = split(/\=/, $devconfig);
4982             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4983             my $net = {
4984                 model => $model,
4985                 bridge => "vmbr$ind",
4986                 macaddr => $macaddr,
4987             };
4988             my $netstr = print_net($net);
4989
4990             print $outfd "net$cookie->{netcount}: $netstr\n";
4991             $cookie->{netcount}++;
4992         }
4993     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4994         my ($id, $netstr) = ($1, $2);
4995         my $net = parse_net($netstr);
4996         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4997         $netstr = print_net($net);
4998         print $outfd "$id: $netstr\n";
4999     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5000         my $virtdev = $1;
5001         my $value = $3;
5002         if ($line =~ m/backup=no/) {
5003             print $outfd "#$line";
5004         } elsif ($virtdev && $map->{$virtdev}) {
5005             my $di = parse_drive($virtdev, $value);
5006             delete $di->{format}; # format can change on restore
5007             $di->{file} = $map->{$virtdev};
5008             $value = print_drive($vmid, $di);
5009             print $outfd "$virtdev: $value\n";
5010         } else {
5011             print $outfd $line;
5012         }
5013     } else {
5014         print $outfd $line;
5015     }
5016 }
5017
5018 sub scan_volids {
5019     my ($cfg, $vmid) = @_;
5020
5021     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5022
5023     my $volid_hash = {};
5024     foreach my $storeid (keys %$info) {
5025         foreach my $item (@{$info->{$storeid}}) {
5026             next if !($item->{volid} && $item->{size});
5027             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5028             $volid_hash->{$item->{volid}} = $item;
5029         }
5030     }
5031
5032     return $volid_hash;
5033 }
5034
5035 sub get_used_paths {
5036     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5037
5038     my $used_path = {};
5039
5040     my $scan_config = sub {
5041         my ($cref, $snapname) = @_;
5042
5043         foreach my $key (keys %$cref) {
5044             my $value = $cref->{$key};
5045             if (valid_drivename($key)) {
5046                 next if $skip_drive && $key eq $skip_drive;
5047                 my $drive = parse_drive($key, $value);
5048                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5049                 if ($drive->{file} =~ m!^/!) {
5050                     $used_path->{$drive->{file}}++; # = 1;
5051                 } else {
5052                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5053                     next if !$storeid;
5054                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5055                     next if !$scfg;
5056                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5057                     $used_path->{$path}++; # = 1;
5058                 }
5059             }
5060         }
5061     };
5062
5063     &$scan_config($conf);
5064
5065     undef $skip_drive;
5066
5067     if ($scan_snapshots) {
5068         foreach my $snapname (keys %{$conf->{snapshots}}) {
5069             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5070         }
5071     }
5072
5073     return $used_path;
5074 }
5075
5076 sub update_disksize {
5077     my ($vmid, $conf, $volid_hash) = @_;
5078
5079     my $changes;
5080
5081     my $used = {};
5082
5083     # Note: it is allowed to define multiple storages with same path (alias), so
5084     # we need to check both 'volid' and real 'path' (two different volid can point
5085     # to the same path).
5086
5087     my $usedpath = {};
5088
5089     # update size info
5090     foreach my $opt (keys %$conf) {
5091         if (valid_drivename($opt)) {
5092             my $drive = parse_drive($opt, $conf->{$opt});
5093             my $volid = $drive->{file};
5094             next if !$volid;
5095
5096             $used->{$volid} = 1;
5097             if ($volid_hash->{$volid} &&
5098                 (my $path = $volid_hash->{$volid}->{path})) {
5099                 $usedpath->{$path} = 1;
5100             }
5101
5102             next if drive_is_cdrom($drive);
5103             next if !$volid_hash->{$volid};
5104
5105             $drive->{size} = $volid_hash->{$volid}->{size};
5106             my $new = print_drive($vmid, $drive);
5107             if ($new ne $conf->{$opt}) {
5108                 $changes = 1;
5109                 $conf->{$opt} = $new;
5110             }
5111         }
5112     }
5113
5114     # remove 'unusedX' entry if volume is used
5115     foreach my $opt (keys %$conf) {
5116         next if $opt !~ m/^unused\d+$/;
5117         my $volid = $conf->{$opt};
5118         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5119         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5120             $changes = 1;
5121             delete $conf->{$opt};
5122         }
5123     }
5124
5125     foreach my $volid (sort keys %$volid_hash) {
5126         next if $volid =~ m/vm-$vmid-state-/;
5127         next if $used->{$volid};
5128         my $path = $volid_hash->{$volid}->{path};
5129         next if !$path; # just to be sure
5130         next if $usedpath->{$path};
5131         $changes = 1;
5132         add_unused_volume($conf, $volid);
5133         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5134     }
5135
5136     return $changes;
5137 }
5138
5139 sub rescan {
5140     my ($vmid, $nolock) = @_;
5141
5142     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5143
5144     my $volid_hash = scan_volids($cfg, $vmid);
5145
5146     my $updatefn =  sub {
5147         my ($vmid) = @_;
5148
5149         my $conf = load_config($vmid);
5150
5151         check_lock($conf);
5152
5153         my $vm_volids = {};
5154         foreach my $volid (keys %$volid_hash) {
5155             my $info = $volid_hash->{$volid};
5156             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5157         }
5158
5159         my $changes = update_disksize($vmid, $conf, $vm_volids);
5160
5161         update_config_nolock($vmid, $conf, 1) if $changes;
5162     };
5163
5164     if (defined($vmid)) {
5165         if ($nolock) {
5166             &$updatefn($vmid);
5167         } else {
5168             lock_config($vmid, $updatefn, $vmid);
5169         }
5170     } else {
5171         my $vmlist = config_list();
5172         foreach my $vmid (keys %$vmlist) {
5173             if ($nolock) {
5174                 &$updatefn($vmid);
5175             } else {
5176                 lock_config($vmid, $updatefn, $vmid);
5177             }
5178         }
5179     }
5180 }
5181
5182 sub restore_vma_archive {
5183     my ($archive, $vmid, $user, $opts, $comp) = @_;
5184
5185     my $input = $archive eq '-' ? "<&STDIN" : undef;
5186     my $readfrom = $archive;
5187
5188     my $uncomp = '';
5189     if ($comp) {
5190         $readfrom = '-';
5191         my $qarchive = PVE::Tools::shellquote($archive);
5192         if ($comp eq 'gzip') {
5193             $uncomp = "zcat $qarchive|";
5194         } elsif ($comp eq 'lzop') {
5195             $uncomp = "lzop -d -c $qarchive|";
5196         } else {
5197             die "unknown compression method '$comp'\n";
5198         }
5199
5200     }
5201
5202     my $tmpdir = "/var/tmp/vzdumptmp$$";
5203     rmtree $tmpdir;
5204
5205     # disable interrupts (always do cleanups)
5206     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5207         warn "got interrupt - ignored\n";
5208     };
5209
5210     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5211     POSIX::mkfifo($mapfifo, 0600);
5212     my $fifofh;
5213
5214     my $openfifo = sub {
5215         open($fifofh, '>', $mapfifo) || die $!;
5216     };
5217
5218     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5219
5220     my $oldtimeout;
5221     my $timeout = 5;
5222
5223     my $devinfo = {};
5224
5225     my $rpcenv = PVE::RPCEnvironment::get();
5226
5227     my $conffile = config_file($vmid);
5228     my $tmpfn = "$conffile.$$.tmp";
5229
5230     # Note: $oldconf is undef if VM does not exists
5231     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5232
5233     my $print_devmap = sub {
5234         my $virtdev_hash = {};
5235
5236         my $cfgfn = "$tmpdir/qemu-server.conf";
5237
5238         # we can read the config - that is already extracted
5239         my $fh = IO::File->new($cfgfn, "r") ||
5240             "unable to read qemu-server.conf - $!\n";
5241
5242         while (defined(my $line = <$fh>)) {
5243             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5244                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5245                 die "archive does not contain data for drive '$virtdev'\n"
5246                     if !$devinfo->{$devname};
5247                 if (defined($opts->{storage})) {
5248                     $storeid = $opts->{storage} || 'local';
5249                 } elsif (!$storeid) {
5250                     $storeid = 'local';
5251                 }
5252                 $format = 'raw' if !$format;
5253                 $devinfo->{$devname}->{devname} = $devname;
5254                 $devinfo->{$devname}->{virtdev} = $virtdev;
5255                 $devinfo->{$devname}->{format} = $format;
5256                 $devinfo->{$devname}->{storeid} = $storeid;
5257
5258                 # check permission on storage
5259                 my $pool = $opts->{pool}; # todo: do we need that?
5260                 if ($user ne 'root@pam') {
5261                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5262                 }
5263
5264                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5265             }
5266         }
5267
5268         foreach my $devname (keys %$devinfo) {
5269             die "found no device mapping information for device '$devname'\n"
5270                 if !$devinfo->{$devname}->{virtdev};
5271         }
5272
5273         my $cfg = cfs_read_file('storage.cfg');
5274
5275         # create empty/temp config
5276         if ($oldconf) {
5277             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5278             foreach_drive($oldconf, sub {
5279                 my ($ds, $drive) = @_;
5280
5281                 return if drive_is_cdrom($drive);
5282
5283                 my $volid = $drive->{file};
5284
5285                 return if !$volid || $volid =~ m|^/|;
5286
5287                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5288                 return if !$path || !$owner || ($owner != $vmid);
5289
5290                 # Note: only delete disk we want to restore
5291                 # other volumes will become unused
5292                 if ($virtdev_hash->{$ds}) {
5293                     PVE::Storage::vdisk_free($cfg, $volid);
5294                 }
5295             });
5296         }
5297
5298         my $map = {};
5299         foreach my $virtdev (sort keys %$virtdev_hash) {
5300             my $d = $virtdev_hash->{$virtdev};
5301             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5302             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5303
5304             # test if requested format is supported
5305             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5306             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5307             $d->{format} = $defFormat if !$supported;
5308
5309             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5310                                                   $d->{format}, undef, $alloc_size);
5311             print STDERR "new volume ID is '$volid'\n";
5312             $d->{volid} = $volid;
5313             my $path = PVE::Storage::path($cfg, $volid);
5314
5315             my $write_zeros = 1;
5316             # fixme: what other storages types initialize volumes with zero?
5317             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5318                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5319                 $write_zeros = 0;
5320             }
5321
5322             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5323
5324             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5325             $map->{$virtdev} = $volid;
5326         }
5327
5328         $fh->seek(0, 0) || die "seek failed - $!\n";
5329
5330         my $outfd = new IO::File ($tmpfn, "w") ||
5331             die "unable to write config for VM $vmid\n";
5332
5333         my $cookie = { netcount => 0 };
5334         while (defined(my $line = <$fh>)) {
5335             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5336         }
5337
5338         $fh->close();
5339         $outfd->close();
5340     };
5341
5342     eval {
5343         # enable interrupts
5344         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5345             die "interrupted by signal\n";
5346         };
5347         local $SIG{ALRM} = sub { die "got timeout\n"; };
5348
5349         $oldtimeout = alarm($timeout);
5350
5351         my $parser = sub {
5352             my $line = shift;
5353
5354             print "$line\n";
5355
5356             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5357                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5358                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5359             } elsif ($line =~ m/^CTIME: /) {
5360                 # we correctly received the vma config, so we can disable
5361                 # the timeout now for disk allocation (set to 10 minutes, so
5362                 # that we always timeout if something goes wrong)
5363                 alarm(600);
5364                 &$print_devmap();
5365                 print $fifofh "done\n";
5366                 my $tmp = $oldtimeout || 0;
5367                 $oldtimeout = undef;
5368                 alarm($tmp);
5369                 close($fifofh);
5370             }
5371         };
5372
5373         print "restore vma archive: $cmd\n";
5374         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5375     };
5376     my $err = $@;
5377
5378     alarm($oldtimeout) if $oldtimeout;
5379
5380     unlink $mapfifo;
5381
5382     if ($err) {
5383         rmtree $tmpdir;
5384         unlink $tmpfn;
5385
5386         my $cfg = cfs_read_file('storage.cfg');
5387         foreach my $devname (keys %$devinfo) {
5388             my $volid = $devinfo->{$devname}->{volid};
5389             next if !$volid;
5390             eval {
5391                 if ($volid =~ m|^/|) {
5392                     unlink $volid || die 'unlink failed\n';
5393                 } else {
5394                     PVE::Storage::vdisk_free($cfg, $volid);
5395                 }
5396                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5397             };
5398             print STDERR "unable to cleanup '$volid' - $@" if $@;
5399         }
5400         die $err;
5401     }
5402
5403     rmtree $tmpdir;
5404
5405     rename($tmpfn, $conffile) ||
5406         die "unable to commit configuration file '$conffile'\n";
5407
5408     PVE::Cluster::cfs_update(); # make sure we read new file
5409
5410     eval { rescan($vmid, 1); };
5411     warn $@ if $@;
5412 }
5413
5414 sub restore_tar_archive {
5415     my ($archive, $vmid, $user, $opts) = @_;
5416
5417     if ($archive ne '-') {
5418         my $firstfile = tar_archive_read_firstfile($archive);
5419         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5420             if $firstfile ne 'qemu-server.conf';
5421     }
5422
5423     my $storecfg = cfs_read_file('storage.cfg');
5424
5425     # destroy existing data - keep empty config
5426     my $vmcfgfn = config_file($vmid);
5427     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5428
5429     my $tocmd = "/usr/lib/qemu-server/qmextract";
5430
5431     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5432     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5433     $tocmd .= ' --prealloc' if $opts->{prealloc};
5434     $tocmd .= ' --info' if $opts->{info};
5435
5436     # tar option "xf" does not autodetect compression when read from STDIN,
5437     # so we pipe to zcat
5438     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5439         PVE::Tools::shellquote("--to-command=$tocmd");
5440
5441     my $tmpdir = "/var/tmp/vzdumptmp$$";
5442     mkpath $tmpdir;
5443
5444     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5445     local $ENV{VZDUMP_VMID} = $vmid;
5446     local $ENV{VZDUMP_USER} = $user;
5447
5448     my $conffile = config_file($vmid);
5449     my $tmpfn = "$conffile.$$.tmp";
5450
5451     # disable interrupts (always do cleanups)
5452     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5453         print STDERR "got interrupt - ignored\n";
5454     };
5455
5456     eval {
5457         # enable interrupts
5458         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5459             die "interrupted by signal\n";
5460         };
5461
5462         if ($archive eq '-') {
5463             print "extracting archive from STDIN\n";
5464             run_command($cmd, input => "<&STDIN");
5465         } else {
5466             print "extracting archive '$archive'\n";
5467             run_command($cmd);
5468         }
5469
5470         return if $opts->{info};
5471
5472         # read new mapping
5473         my $map = {};
5474         my $statfile = "$tmpdir/qmrestore.stat";
5475         if (my $fd = IO::File->new($statfile, "r")) {
5476             while (defined (my $line = <$fd>)) {
5477                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5478                     $map->{$1} = $2 if $1;
5479                 } else {
5480                     print STDERR "unable to parse line in statfile - $line\n";
5481                 }
5482             }
5483             $fd->close();
5484         }
5485
5486         my $confsrc = "$tmpdir/qemu-server.conf";
5487
5488         my $srcfd = new IO::File($confsrc, "r") ||
5489             die "unable to open file '$confsrc'\n";
5490
5491         my $outfd = new IO::File ($tmpfn, "w") ||
5492             die "unable to write config for VM $vmid\n";
5493
5494         my $cookie = { netcount => 0 };
5495         while (defined (my $line = <$srcfd>)) {
5496             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5497         }
5498
5499         $srcfd->close();
5500         $outfd->close();
5501     };
5502     my $err = $@;
5503
5504     if ($err) {
5505
5506         unlink $tmpfn;
5507
5508         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5509
5510         die $err;
5511     }
5512
5513     rmtree $tmpdir;
5514
5515     rename $tmpfn, $conffile ||
5516         die "unable to commit configuration file '$conffile'\n";
5517
5518     PVE::Cluster::cfs_update(); # make sure we read new file
5519
5520     eval { rescan($vmid, 1); };
5521     warn $@ if $@;
5522 };
5523
5524
5525 # Internal snapshots
5526
5527 # NOTE: Snapshot create/delete involves several non-atomic
5528 # action, and can take a long time.
5529 # So we try to avoid locking the file and use 'lock' variable
5530 # inside the config file instead.
5531
5532 my $snapshot_copy_config = sub {
5533     my ($source, $dest) = @_;
5534
5535     foreach my $k (keys %$source) {
5536         next if $k eq 'snapshots';
5537         next if $k eq 'snapstate';
5538         next if $k eq 'snaptime';
5539         next if $k eq 'vmstate';
5540         next if $k eq 'lock';
5541         next if $k eq 'digest';
5542         next if $k eq 'description';
5543         next if $k =~ m/^unused\d+$/;
5544
5545         $dest->{$k} = $source->{$k};
5546     }
5547 };
5548
5549 my $snapshot_apply_config = sub {
5550     my ($conf, $snap) = @_;
5551
5552     # copy snapshot list
5553     my $newconf = {
5554         snapshots => $conf->{snapshots},
5555     };
5556
5557     # keep description and list of unused disks
5558     foreach my $k (keys %$conf) {
5559         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5560         $newconf->{$k} = $conf->{$k};
5561     }
5562
5563     &$snapshot_copy_config($snap, $newconf);
5564
5565     return $newconf;
5566 };
5567
5568 sub foreach_writable_storage {
5569     my ($conf, $func) = @_;
5570
5571     my $sidhash = {};
5572
5573     foreach my $ds (keys %$conf) {
5574         next if !valid_drivename($ds);
5575
5576         my $drive = parse_drive($ds, $conf->{$ds});
5577         next if !$drive;
5578         next if drive_is_cdrom($drive);
5579
5580         my $volid = $drive->{file};
5581
5582         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5583         $sidhash->{$sid} = $sid if $sid;
5584     }
5585
5586     foreach my $sid (sort keys %$sidhash) {
5587         &$func($sid);
5588     }
5589 }
5590
5591 my $alloc_vmstate_volid = sub {
5592     my ($storecfg, $vmid, $conf, $snapname) = @_;
5593
5594     # Note: we try to be smart when selecting a $target storage
5595
5596     my $target;
5597
5598     # search shared storage first
5599     foreach_writable_storage($conf, sub {
5600         my ($sid) = @_;
5601         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5602         return if !$scfg->{shared};
5603
5604         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5605     });
5606
5607     if (!$target) {
5608         # now search local storage
5609         foreach_writable_storage($conf, sub {
5610             my ($sid) = @_;
5611             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5612             return if $scfg->{shared};
5613
5614             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5615         });
5616     }
5617
5618     $target = 'local' if !$target;
5619
5620     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5621     # we abort live save after $conf->{memory}, so we need at max twice that space
5622     my $size = $conf->{memory}*2 + $driver_state_size;
5623
5624     my $name = "vm-$vmid-state-$snapname";
5625     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5626     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5627     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5628
5629     return $volid;
5630 };
5631
5632 my $snapshot_prepare = sub {
5633     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5634
5635     my $snap;
5636
5637     my $updatefn =  sub {
5638
5639         my $conf = load_config($vmid);
5640
5641         die "you can't take a snapshot if it's a template\n"
5642             if is_template($conf);
5643
5644         check_lock($conf);
5645
5646         $conf->{lock} = 'snapshot';
5647
5648         die "snapshot name '$snapname' already used\n"
5649             if defined($conf->{snapshots}->{$snapname});
5650
5651         my $storecfg = PVE::Storage::config();
5652         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5653
5654         $snap = $conf->{snapshots}->{$snapname} = {};
5655
5656         if ($save_vmstate && check_running($vmid)) {
5657             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5658         }
5659
5660         &$snapshot_copy_config($conf, $snap);
5661
5662         $snap->{snapstate} = "prepare";
5663         $snap->{snaptime} = time();
5664         $snap->{description} = $comment if $comment;
5665
5666         # always overwrite machine if we save vmstate. This makes sure we
5667         # can restore it later using correct machine type
5668         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5669
5670         update_config_nolock($vmid, $conf, 1);
5671     };
5672
5673     lock_config($vmid, $updatefn);
5674
5675     return $snap;
5676 };
5677
5678 my $snapshot_commit = sub {
5679     my ($vmid, $snapname) = @_;
5680
5681     my $updatefn = sub {
5682
5683         my $conf = load_config($vmid);
5684
5685         die "missing snapshot lock\n"
5686             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5687
5688         my $has_machine_config = defined($conf->{machine});
5689
5690         my $snap = $conf->{snapshots}->{$snapname};
5691
5692         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5693
5694         die "wrong snapshot state\n"
5695             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5696
5697         delete $snap->{snapstate};
5698         delete $conf->{lock};
5699
5700         my $newconf = &$snapshot_apply_config($conf, $snap);
5701
5702         delete $newconf->{machine} if !$has_machine_config;
5703
5704         $newconf->{parent} = $snapname;
5705
5706         update_config_nolock($vmid, $newconf, 1);
5707     };
5708
5709     lock_config($vmid, $updatefn);
5710 };
5711
5712 sub snapshot_rollback {
5713     my ($vmid, $snapname) = @_;
5714
5715     my $prepare = 1;
5716
5717     my $storecfg = PVE::Storage::config();
5718
5719     my $conf = load_config($vmid);
5720
5721     my $get_snapshot_config = sub {
5722
5723         die "you can't rollback if vm is a template\n" if is_template($conf);
5724
5725         my $res = $conf->{snapshots}->{$snapname};
5726
5727         die "snapshot '$snapname' does not exist\n" if !defined($res);
5728
5729         return $res;
5730     };
5731
5732     my $snap = &$get_snapshot_config();
5733
5734     foreach_drive($snap, sub {
5735         my ($ds, $drive) = @_;
5736
5737         return if drive_is_cdrom($drive);
5738
5739         my $volid = $drive->{file};
5740
5741         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5742     });
5743
5744     my $updatefn = sub {
5745
5746         $conf = load_config($vmid);
5747
5748         $snap = &$get_snapshot_config();
5749
5750         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5751             if $snap->{snapstate};
5752
5753         if ($prepare) {
5754             check_lock($conf);
5755             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5756         }
5757
5758         die "unable to rollback vm $vmid: vm is running\n"
5759             if check_running($vmid);
5760
5761         if ($prepare) {
5762             $conf->{lock} = 'rollback';
5763         } else {
5764             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5765             delete $conf->{lock};
5766         }
5767
5768         my $forcemachine;
5769
5770         if (!$prepare) {
5771             my $has_machine_config = defined($conf->{machine});
5772
5773             # copy snapshot config to current config
5774             $conf = &$snapshot_apply_config($conf, $snap);
5775             $conf->{parent} = $snapname;
5776
5777             # Note: old code did not store 'machine', so we try to be smart
5778             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5779             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5780             # we remove the 'machine' configuration if not explicitly specified
5781             # in the original config.
5782             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5783         }
5784
5785         update_config_nolock($vmid, $conf, 1);
5786
5787         if (!$prepare && $snap->{vmstate}) {
5788             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5789             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5790         }
5791     };
5792
5793     lock_config($vmid, $updatefn);
5794
5795     foreach_drive($snap, sub {
5796         my ($ds, $drive) = @_;
5797
5798         return if drive_is_cdrom($drive);
5799
5800         my $volid = $drive->{file};
5801         my $device = "drive-$ds";
5802
5803         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5804     });
5805
5806     $prepare = 0;
5807     lock_config($vmid, $updatefn);
5808 }
5809
5810 my $savevm_wait = sub {
5811     my ($vmid) = @_;
5812
5813     for(;;) {
5814         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5815         if (!$stat->{status}) {
5816             die "savevm not active\n";
5817         } elsif ($stat->{status} eq 'active') {
5818             sleep(1);
5819             next;
5820         } elsif ($stat->{status} eq 'completed') {
5821             last;
5822         } else {
5823             die "query-savevm returned status '$stat->{status}'\n";
5824         }
5825     }
5826 };
5827
5828 sub do_snapshots_with_qemu {
5829     my ($storecfg, $volid) = @_;
5830
5831     my $storage_name = PVE::Storage::parse_volume_id($volid);
5832
5833     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5834         return 1;
5835     }
5836
5837     if ($volid =~ m/\.(qcow2|qed)$/){
5838         return 1;
5839     }
5840
5841     return undef;
5842 }
5843
5844 sub snapshot_create {
5845     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5846
5847     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5848
5849     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5850
5851     my $config = load_config($vmid);
5852
5853     my $running = check_running($vmid);
5854
5855     my $freezefs = $running && $config->{agent};
5856     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5857
5858     my $drivehash = {};
5859
5860     if ($freezefs) {
5861         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5862         warn "guest-fsfreeze-freeze problems - $@" if $@;
5863     }
5864
5865     eval {
5866         # create internal snapshots of all drives
5867
5868         my $storecfg = PVE::Storage::config();
5869
5870         if ($running) {
5871             if ($snap->{vmstate}) {
5872                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5873                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5874                 &$savevm_wait($vmid);
5875             } else {
5876                 vm_mon_cmd($vmid, "savevm-start");
5877             }
5878         };
5879
5880         foreach_drive($snap, sub {
5881             my ($ds, $drive) = @_;
5882
5883             return if drive_is_cdrom($drive);
5884
5885             my $volid = $drive->{file};
5886             my $device = "drive-$ds";
5887
5888             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5889             $drivehash->{$ds} = 1;
5890        });
5891     };
5892     my $err = $@;
5893
5894     if ($running) {
5895         eval { vm_mon_cmd($vmid, "savevm-end")  };
5896         warn $@ if $@;
5897
5898         if ($freezefs) {
5899             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5900             warn "guest-fsfreeze-thaw problems - $@" if $@;
5901         }
5902
5903         # savevm-end is async, we need to wait
5904         for (;;) {
5905             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5906             if (!$stat->{bytes}) {
5907                 last;
5908             } else {
5909                 print "savevm not yet finished\n";
5910                 sleep(1);
5911                 next;
5912             }
5913         }
5914     }
5915
5916     if ($err) {
5917         warn "snapshot create failed: starting cleanup\n";
5918         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5919         warn $@ if $@;
5920         die $err;
5921     }
5922
5923     &$snapshot_commit($vmid, $snapname);
5924 }
5925
5926 # Note: $drivehash is only set when called from snapshot_create.
5927 sub snapshot_delete {
5928     my ($vmid, $snapname, $force, $drivehash) = @_;
5929
5930     my $prepare = 1;
5931
5932     my $snap;
5933     my $unused = [];
5934
5935     my $unlink_parent = sub {
5936         my ($confref, $new_parent) = @_;
5937
5938         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5939             if ($new_parent) {
5940                 $confref->{parent} = $new_parent;
5941             } else {
5942                 delete $confref->{parent};
5943             }
5944         }
5945     };
5946
5947     my $updatefn =  sub {
5948         my ($remove_drive) = @_;
5949
5950         my $conf = load_config($vmid);
5951
5952         if (!$drivehash) {
5953             check_lock($conf);
5954             die "you can't delete a snapshot if vm is a template\n"
5955                 if is_template($conf);
5956         }
5957
5958         $snap = $conf->{snapshots}->{$snapname};
5959
5960         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5961
5962         # remove parent refs
5963         if (!$prepare) {
5964             &$unlink_parent($conf, $snap->{parent});
5965             foreach my $sn (keys %{$conf->{snapshots}}) {
5966                 next if $sn eq $snapname;
5967                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5968             }
5969         }
5970
5971         if ($remove_drive) {
5972             if ($remove_drive eq 'vmstate') {
5973                 delete $snap->{$remove_drive};
5974             } else {
5975                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5976                 my $volid = $drive->{file};
5977                 delete $snap->{$remove_drive};
5978                 add_unused_volume($conf, $volid);
5979             }
5980         }
5981
5982         if ($prepare) {
5983             $snap->{snapstate} = 'delete';
5984         } else {
5985             delete $conf->{snapshots}->{$snapname};
5986             delete $conf->{lock} if $drivehash;
5987             foreach my $volid (@$unused) {
5988                 add_unused_volume($conf, $volid);
5989             }
5990         }
5991
5992         update_config_nolock($vmid, $conf, 1);
5993     };
5994
5995     lock_config($vmid, $updatefn);
5996
5997     # now remove vmstate file
5998
5999     my $storecfg = PVE::Storage::config();
6000
6001     if ($snap->{vmstate}) {
6002         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6003         if (my $err = $@) {
6004             die $err if !$force;
6005             warn $err;
6006         }
6007         # save changes (remove vmstate from snapshot)
6008         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6009     };
6010
6011     # now remove all internal snapshots
6012     foreach_drive($snap, sub {
6013         my ($ds, $drive) = @_;
6014
6015         return if drive_is_cdrom($drive);
6016
6017         my $volid = $drive->{file};
6018         my $device = "drive-$ds";
6019
6020         if (!$drivehash || $drivehash->{$ds}) {
6021             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6022             if (my $err = $@) {
6023                 die $err if !$force;
6024                 warn $err;
6025             }
6026         }
6027
6028         # save changes (remove drive fron snapshot)
6029         lock_config($vmid, $updatefn, $ds) if !$force;
6030         push @$unused, $volid;
6031     });
6032
6033     # now cleanup config
6034     $prepare = 0;
6035     lock_config($vmid, $updatefn);
6036 }
6037
6038 sub has_feature {
6039     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6040
6041     my $err;
6042     foreach_drive($conf, sub {
6043         my ($ds, $drive) = @_;
6044
6045         return if drive_is_cdrom($drive);
6046         my $volid = $drive->{file};
6047         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6048     });
6049
6050     return $err ? 0 : 1;
6051 }
6052
6053 sub template_create {
6054     my ($vmid, $conf, $disk) = @_;
6055
6056     my $storecfg = PVE::Storage::config();
6057
6058     foreach_drive($conf, sub {
6059         my ($ds, $drive) = @_;
6060
6061         return if drive_is_cdrom($drive);
6062         return if $disk && $ds ne $disk;
6063
6064         my $volid = $drive->{file};
6065         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6066
6067         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6068         $drive->{file} = $voliddst;
6069         $conf->{$ds} = print_drive($vmid, $drive);
6070         update_config_nolock($vmid, $conf, 1);
6071     });
6072 }
6073
6074 sub is_template {
6075     my ($conf) = @_;
6076
6077     return 1 if defined $conf->{template} && $conf->{template} == 1;
6078 }
6079
6080 sub qemu_img_convert {
6081     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6082
6083     my $storecfg = PVE::Storage::config();
6084     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6085     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6086
6087     if ($src_storeid && $dst_storeid) {
6088         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6089         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6090
6091         my $src_format = qemu_img_format($src_scfg, $src_volname);
6092         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6093
6094         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6095         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6096
6097         my $cmd = [];
6098         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6099         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6100         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6101
6102         my $parser = sub {
6103             my $line = shift;
6104             if($line =~ m/\((\S+)\/100\%\)/){
6105                 my $percent = $1;
6106                 my $transferred = int($size * $percent / 100);
6107                 my $remaining = $size - $transferred;
6108
6109                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6110             }
6111
6112         };
6113
6114         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6115         my $err = $@;
6116         die "copy failed: $err" if $err;
6117     }
6118 }
6119
6120 sub qemu_img_format {
6121     my ($scfg, $volname) = @_;
6122
6123     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6124         return $1;
6125     } else {
6126         return "raw";
6127     }
6128 }
6129
6130 sub qemu_drive_mirror {
6131     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6132
6133     my $storecfg = PVE::Storage::config();
6134     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6135
6136     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6137
6138     my $format = qemu_img_format($dst_scfg, $dst_volname);
6139
6140     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6141
6142     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6143     $opts->{format} = $format if $format;
6144
6145     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6146
6147     eval {
6148     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6149         while (1) {
6150             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6151             my $stat = @$stats[0];
6152             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6153             die "error job is not mirroring" if $stat->{type} ne "mirror";
6154
6155             my $busy = $stat->{busy};
6156             my $ready = $stat->{ready};
6157
6158             if (my $total = $stat->{len}) {
6159                 my $transferred = $stat->{offset} || 0;
6160                 my $remaining = $total - $transferred;
6161                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6162
6163                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6164             }
6165
6166
6167             if ($stat->{ready} eq 'true') {
6168
6169                 last if $vmiddst != $vmid;
6170
6171                 # try to switch the disk if source and destination are on the same guest
6172                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6173                 last if !$@;
6174                 die $@ if $@ !~ m/cannot be completed/;
6175             }
6176             sleep 1;
6177         }
6178
6179
6180     };
6181     my $err = $@;
6182
6183     my $cancel_job = sub {
6184         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6185         while (1) {
6186             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6187             my $stat = @$stats[0];
6188             last if !$stat;
6189             sleep 1;
6190         }
6191     };
6192
6193     if ($err) {
6194         eval { &$cancel_job(); };
6195         die "mirroring error: $err";
6196     }
6197
6198     if ($vmiddst != $vmid) {
6199         # if we clone a disk for a new target vm, we don't switch the disk
6200         &$cancel_job(); # so we call block-job-cancel
6201     }
6202 }
6203
6204 sub clone_disk {
6205     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6206         $newvmid, $storage, $format, $full, $newvollist) = @_;
6207
6208     my $newvolid;
6209
6210     if (!$full) {
6211         print "create linked clone of drive $drivename ($drive->{file})\n";
6212         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6213         push @$newvollist, $newvolid;
6214     } else {
6215         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6216         $storeid = $storage if $storage;
6217
6218         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6219         if (!$format) {
6220             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6221             $format = qemu_img_format($scfg, $volname);
6222         }
6223
6224         # test if requested format is supported - else use default
6225         my $supported = grep { $_ eq $format } @$validFormats;
6226         $format = $defFormat if !$supported;
6227
6228         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6229
6230         print "create full clone of drive $drivename ($drive->{file})\n";
6231         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6232         push @$newvollist, $newvolid;
6233
6234         if (!$running || $snapname) {
6235             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6236         } else {
6237             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6238         }
6239     }
6240
6241     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6242
6243     my $disk = $drive;
6244     $disk->{format} = undef;
6245     $disk->{file} = $newvolid;
6246     $disk->{size} = $size;
6247
6248     return $disk;
6249 }
6250
6251 # this only works if VM is running
6252 sub get_current_qemu_machine {
6253     my ($vmid) = @_;
6254
6255     my $cmd = { execute => 'query-machines', arguments => {} };
6256     my $res = vm_qmp_command($vmid, $cmd);
6257
6258     my ($current, $default);
6259     foreach my $e (@$res) {
6260         $default = $e->{name} if $e->{'is-default'};
6261         $current = $e->{name} if $e->{'is-current'};
6262     }
6263
6264     # fallback to the default machine if current is not supported by qemu
6265     return $current || $default || 'pc';
6266 }
6267
6268 sub qemu_machine_feature_enabled {
6269     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6270
6271     my $current_major;
6272     my $current_minor;
6273
6274     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6275
6276         $current_major = $3;
6277         $current_minor = $4;
6278
6279     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6280
6281         $current_major = $1;
6282         $current_minor = $2;
6283     }
6284
6285     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6286
6287
6288 }
6289
6290 sub lspci {
6291
6292     my $devices = {};
6293
6294     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6295             my (undef, $id, $function) = @_;
6296             my $res = { id => $id, function => $function};
6297             push @{$devices->{$id}}, $res;
6298     });
6299
6300     return $devices;
6301 }
6302
6303 sub vm_iothreads_list {
6304     my ($vmid) = @_;
6305
6306     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6307
6308     my $iothreads = {};
6309     foreach my $iothread (@$res) {
6310         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6311     }
6312
6313     return $iothreads;
6314 }
6315
6316 sub scsihw_infos {
6317     my ($conf, $drive) = @_;
6318
6319     my $maxdev = 0;
6320
6321     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6322         $maxdev = 7;
6323     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6324         $maxdev = 1;
6325     } else {
6326         $maxdev = 256;
6327     }
6328
6329     my $controller = int($drive->{index} / $maxdev);
6330     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6331
6332     return ($maxdev, $controller, $controller_prefix);
6333 }
6334
6335 1;