PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     return undef if $res->{cache} &&
 917         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 918     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 919     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 920     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 921     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 922     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 923     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 924     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 925     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 926     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 927     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 928     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 929     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 930     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 931     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 932
 933     return undef if $res->{mbps_rd} && $res->{mbps};
 934     return undef if $res->{mbps_wr} && $res->{mbps};
 935
 936     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 937     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 938     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 939     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 942
 943     return undef if $res->{iops_rd} && $res->{iops};
 944     return undef if $res->{iops_wr} && $res->{iops};
 945
 946
 947     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 948     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 949     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 950     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 951     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 952     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 953
 954
 955     if ($res->{size}) {
 956         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 957     }
 958
 959     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 960         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 961         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 962         return undef if $res->{interface} eq 'virtio';
 963     }
 964
 965     # rerror does not work with scsi drives
 966     if ($res->{rerror}) {
 967         return undef if $res->{interface} eq 'scsi';
 968     }
 969
 970     return $res;
 971 }
 972
 973 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 974
 975 sub print_drive {
 976     my ($vmid, $drive) = @_;
 977
 978     my $opts = '';
 979     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 980         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 981     }
 982
 983     if ($drive->{size}) {
 984         $opts .= ",size=" . &$format_size($drive->{size});
 985     }
 986
 987     return "$drive->{file}$opts";
 988 }
 989
 990 sub scsi_inquiry {
 991     my($fh, $noerr) = @_;
 992
 993     my $SG_IO = 0x2285;
 994     my $SG_GET_VERSION_NUM = 0x2282;
 995
 996     my $versionbuf = "\x00" x 8;
 997     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
 998     if (!$ret) {
 999         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1000         return undef;
1001     }
1002     my $version = unpack("I", $versionbuf);
1003     if ($version < 30000) {
1004         die "scsi generic interface too old\n"  if !$noerr;
1005         return undef;
1006     }
1007
1008     my $buf = "\x00" x 36;
1009     my $sensebuf = "\x00" x 8;
1010     my $cmd = pack("C x3 C x1", 0x12, 36);
1011
1012     # see /usr/include/scsi/sg.h
1013     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1014
1015     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1016                       length($sensebuf), 0, length($buf), $buf,
1017                       $cmd, $sensebuf, 6000);
1018
1019     $ret = ioctl($fh, $SG_IO, $packet);
1020     if (!$ret) {
1021         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024
1025     my @res = unpack($sg_io_hdr_t, $packet);
1026     if ($res[17] || $res[18]) {
1027         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my $res = {};
1032     (my $byte0, my $byte1, $res->{vendor},
1033      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1034
1035     $res->{removable} = $byte1 & 128 ? 1 : 0;
1036     $res->{type} = $byte0 & 31;
1037
1038     return $res;
1039 }
1040
1041 sub path_is_scsi {
1042     my ($path) = @_;
1043
1044     my $fh = IO::File->new("+<$path") || return undef;
1045     my $res = scsi_inquiry($fh, 1);
1046     close($fh);
1047
1048     return $res;
1049 }
1050
1051 sub machine_type_is_q35 {
1052     my ($conf) = @_;
1053
1054     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1055 }
1056
1057 sub print_tabletdevice_full {
1058     my ($conf) = @_;
1059
1060     my $q35 = machine_type_is_q35($conf);
1061
1062     # we use uhci for old VMs because tablet driver was buggy in older qemu
1063     my $usbbus = $q35 ? "ehci" : "uhci";
1064
1065     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1066 }
1067
1068 sub print_drivedevice_full {
1069     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1070
1071     my $device = '';
1072     my $maxdev = 0;
1073
1074     if ($drive->{interface} eq 'virtio') {
1075         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1076         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1077         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1078     } elsif ($drive->{interface} eq 'scsi') {
1079
1080         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1081         my $unit = $drive->{index} % $maxdev;
1082         my $devicetype = 'hd';
1083         my $path = '';
1084         if (drive_is_cdrom($drive)) {
1085               $devicetype = 'cd';
1086         } else {
1087               if ($drive->{file} =~ m|^/|) {
1088                   $path = $drive->{file};
1089               } else {
1090                   $path = PVE::Storage::path($storecfg, $drive->{file});
1091               }
1092
1093               if($path =~ m/^iscsi\:\/\//){
1094                   $devicetype = 'generic';
1095               } else {
1096                   if (my $info = path_is_scsi($path)) {
1097                       if ($info->{type} == 0) {
1098                           $devicetype = 'block';
1099                       } elsif ($info->{type} == 1) { # tape
1100                           $devicetype = 'generic';
1101                       }
1102                   }
1103               }
1104          }
1105
1106         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1107             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1108         } else {
1109             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1110         }
1111
1112     } elsif ($drive->{interface} eq 'ide'){
1113         $maxdev = 2;
1114         my $controller = int($drive->{index} / $maxdev);
1115         my $unit = $drive->{index} % $maxdev;
1116         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1117
1118         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1119     } elsif ($drive->{interface} eq 'sata'){
1120         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1121         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1122         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'usb') {
1124         die "implement me";
1125         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1126     } else {
1127         die "unsupported interface type";
1128     }
1129
1130     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1131
1132     return $device;
1133 }
1134
1135 sub get_initiator_name {
1136     my $initiator;
1137
1138     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1139     while (defined(my $line = <$fh>)) {
1140         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1141         $initiator = $1;
1142         last;
1143     }
1144     $fh->close();
1145
1146     return $initiator;
1147 }
1148
1149 sub print_drive_full {
1150     my ($storecfg, $vmid, $drive) = @_;
1151
1152     my $path;
1153     my $volid = $drive->{file};
1154     my $format;
1155
1156     if (drive_is_cdrom($drive)) {
1157         $path = get_iso_path($storecfg, $vmid, $volid);
1158     } else {
1159         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1160         if ($storeid) {
1161             $path = PVE::Storage::path($storecfg, $volid);
1162             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1163             $format = qemu_img_format($scfg, $volname);
1164         } else {
1165             $path = $volid;
1166         }
1167    }
1168
1169     my $opts = '';
1170     foreach my $o (@qemu_drive_options) {
1171         next if $o eq 'bootindex';
1172         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1173     }
1174
1175     $opts .= ",format=$format" if $format && !$drive->{format};
1176
1177     foreach my $o (qw(bps bps_rd bps_wr)) {
1178         my $v = $drive->{"m$o"};
1179         $opts .= ",$o=" . int($v*1024*1024) if $v;
1180     }
1181
1182     my $cache_direct = 0;
1183
1184     if (my $cache = $drive->{cache}) {
1185         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1186     } elsif (!drive_is_cdrom($drive)) {
1187         $opts .= ",cache=none";
1188         $cache_direct = 1;
1189     }
1190
1191     # aio native works only with O_DIRECT
1192     if (!$drive->{aio}) {
1193         if($cache_direct) {
1194             $opts .= ",aio=native";
1195         } else {
1196             $opts .= ",aio=threads";
1197         }
1198     }
1199
1200     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1201     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1202
1203     my $pathinfo = $path ? "file=$path," : '';
1204
1205     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1206 }
1207
1208 sub print_netdevice_full {
1209     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1210
1211     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1212
1213     my $device = $net->{model};
1214     if ($net->{model} eq 'virtio') {
1215          $device = 'virtio-net-pci';
1216      };
1217
1218     my $pciaddr = print_pci_addr("$netid", $bridges);
1219     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1220     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1221         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1222         my $vectors = $net->{queues} * 2 + 2;
1223         $tmpstr .= ",vectors=$vectors,mq=on";
1224     }
1225     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1226     return $tmpstr;
1227 }
1228
1229 sub print_netdev_full {
1230     my ($vmid, $conf, $net, $netid) = @_;
1231
1232     my $i = '';
1233     if ($netid =~ m/^net(\d+)$/) {
1234         $i = int($1);
1235     }
1236
1237     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1238
1239     my $ifname = "tap${vmid}i$i";
1240
1241     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1242     die "interface name '$ifname' is too long (max 15 character)\n"
1243         if length($ifname) >= 16;
1244
1245     my $vhostparam = '';
1246     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1247
1248     my $vmname = $conf->{name} || "vm$vmid";
1249
1250     my $netdev = "";
1251
1252     if ($net->{bridge}) {
1253         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1254     } else {
1255         $netdev = "type=user,id=$netid,hostname=$vmname";
1256     }
1257
1258     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1259
1260     return $netdev;
1261 }
1262
1263 sub drive_is_cdrom {
1264     my ($drive) = @_;
1265
1266     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1267
1268 }
1269
1270 sub parse_numa {
1271     my ($data) = @_;
1272
1273     my $res = {};
1274
1275     foreach my $kvp (split(/,/, $data)) {
1276
1277         if ($kvp =~ m/^memory=(\S+)$/) {
1278             $res->{memory} = $1;
1279         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1280             $res->{policy} = $1;
1281         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1282             $res->{cpus}->{start} = $1;
1283             $res->{cpus}->{end} = $3;
1284         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1285             $res->{hostnodes}->{start} = $1;
1286             $res->{hostnodes}->{end} = $3;
1287         } else {
1288             return undef;
1289         }
1290     }
1291
1292     return $res;
1293 }
1294
1295 sub parse_hostpci {
1296     my ($value) = @_;
1297
1298     return undef if !$value;
1299
1300
1301     my @list = split(/,/, $value);
1302     my $found;
1303
1304     my $res = {};
1305     foreach my $kv (@list) {
1306
1307         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1308             $found = 1;
1309             if(defined($4)){
1310                 push @{$res->{pciid}}, { id => $2 , function => $4};
1311
1312             }else{
1313                 my $pcidevices = lspci($2);
1314                 $res->{pciid} = $pcidevices->{$2};
1315             }
1316         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1317             $res->{driver} = $1;
1318         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1319             $res->{rombar} = $1;
1320         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1321             $res->{'x-vga'} = $1;
1322         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1323             $res->{pcie} = 1 if $1 == 1;
1324         } else {
1325             warn "unknown hostpci setting '$kv'\n";
1326         }
1327     }
1328
1329     return undef if !$found;
1330
1331     return $res;
1332 }
1333
1334 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1335 sub parse_net {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1343             my $model = lc($1);
1344             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1345             $res->{model} = $model;
1346             $res->{macaddr} = $mac;
1347         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1348             $res->{bridge} = $1;
1349         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1350             $res->{queues} = $1;
1351         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1352             $res->{rate} = $1;
1353         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1354             $res->{tag} = $1;
1355         } elsif ($kvp =~ m/^firewall=([01])$/) {
1356             $res->{firewall} = $1;
1357         } elsif ($kvp =~ m/^link_down=([01])$/) {
1358             $res->{link_down} = $1;
1359         } else {
1360             return undef;
1361         }
1362
1363     }
1364
1365     return undef if !$res->{model};
1366
1367     return $res;
1368 }
1369
1370 sub print_net {
1371     my $net = shift;
1372
1373     my $res = "$net->{model}";
1374     $res .= "=$net->{macaddr}" if $net->{macaddr};
1375     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1376     $res .= ",rate=$net->{rate}" if $net->{rate};
1377     $res .= ",tag=$net->{tag}" if $net->{tag};
1378     $res .= ",firewall=1" if $net->{firewall};
1379     $res .= ",link_down=1" if $net->{link_down};
1380     $res .= ",queues=$net->{queues}" if $net->{queues};
1381
1382     return $res;
1383 }
1384
1385 sub add_random_macs {
1386     my ($settings) = @_;
1387
1388     foreach my $opt (keys %$settings) {
1389         next if $opt !~ m/^net(\d+)$/;
1390         my $net = parse_net($settings->{$opt});
1391         next if !$net;
1392         $settings->{$opt} = print_net($net);
1393     }
1394 }
1395
1396 sub add_unused_volume {
1397     my ($config, $volid) = @_;
1398
1399     my $key;
1400     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1401         my $test = "unused$ind";
1402         if (my $vid = $config->{$test}) {
1403             return if $vid eq $volid; # do not add duplicates
1404         } else {
1405             $key = $test;
1406         }
1407     }
1408
1409     die "To many unused volume - please delete them first.\n" if !$key;
1410
1411     $config->{$key} = $volid;
1412
1413     return $key;
1414 }
1415
1416 sub vm_is_volid_owner {
1417     my ($storecfg, $vmid, $volid) = @_;
1418
1419     if ($volid !~  m|^/|) {
1420         my ($path, $owner);
1421         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1422         if ($owner && ($owner == $vmid)) {
1423             return 1;
1424         }
1425     }
1426
1427     return undef;
1428 }
1429
1430 sub vmconfig_delete_pending_option {
1431     my ($conf, $key) = @_;
1432
1433     delete $conf->{pending}->{$key};
1434     my $pending_delete_hash = { $key => 1 };
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1439 }
1440
1441 sub vmconfig_undelete_pending_option {
1442     my ($conf, $key) = @_;
1443
1444     my $pending_delete_hash = {};
1445     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1446         $pending_delete_hash->{$opt} = 1;
1447     }
1448     delete $pending_delete_hash->{$key};
1449
1450     my @keylist = keys %$pending_delete_hash;
1451     if (scalar(@keylist)) {
1452         $conf->{pending}->{delete} = join(',', @keylist);
1453     } else {
1454         delete $conf->{pending}->{delete};
1455     }
1456 }
1457
1458 sub vmconfig_register_unused_drive {
1459     my ($storecfg, $vmid, $conf, $drive) = @_;
1460
1461     if (!drive_is_cdrom($drive)) {
1462         my $volid = $drive->{file};
1463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1464             add_unused_volume($conf, $volid, $vmid);
1465         }
1466     }
1467 }
1468
1469 sub vmconfig_cleanup_pending {
1470     my ($conf) = @_;
1471
1472     # remove pending changes when nothing changed
1473     my $changes;
1474     foreach my $opt (keys %{$conf->{pending}}) {
1475         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1476             $changes = 1;
1477             delete $conf->{pending}->{$opt};
1478         }
1479     }
1480
1481     # remove delete if option is not set
1482     my $pending_delete_hash = {};
1483     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1484         if (defined($conf->{$opt})) {
1485             $pending_delete_hash->{$opt} = 1;
1486         } else {
1487             $changes = 1;
1488         }
1489     }
1490
1491     my @keylist = keys %$pending_delete_hash;
1492     if (scalar(@keylist)) {
1493         $conf->{pending}->{delete} = join(',', @keylist);
1494     } else {
1495         delete $conf->{pending}->{delete};
1496     }
1497
1498     return $changes;
1499 }
1500
1501 my $valid_smbios1_options = {
1502     manufacturer => '\S+',
1503     product => '\S+',
1504     version => '\S+',
1505     serial => '\S+',
1506     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1507     sku => '\S+',
1508     family => '\S+',
1509 };
1510
1511 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1512 sub parse_smbios1 {
1513     my ($data) = @_;
1514
1515     my $res = {};
1516
1517     foreach my $kvp (split(/,/, $data)) {
1518         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1519         my ($k, $v) = split(/=/, $kvp);
1520         return undef if !defined($k) || !defined($v);
1521         return undef if !$valid_smbios1_options->{$k};
1522         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1523         $res->{$k} = $v;
1524     }
1525
1526     return $res;
1527 }
1528
1529 sub print_smbios1 {
1530     my ($smbios1) = @_;
1531
1532     my $data = '';
1533     foreach my $k (keys %$smbios1) {
1534         next if !defined($smbios1->{$k});
1535         next if !$valid_smbios1_options->{$k};
1536         $data .= ',' if $data;
1537         $data .= "$k=$smbios1->{$k}";
1538     }
1539     return $data;
1540 }
1541
1542 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1543 sub verify_smbios1 {
1544     my ($value, $noerr) = @_;
1545
1546     return $value if parse_smbios1($value);
1547
1548     return undef if $noerr;
1549
1550     die "unable to parse smbios (type 1) options\n";
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1554 sub verify_bootdisk {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if valid_drivename($value);
1558
1559     return undef if $noerr;
1560
1561     die "invalid boot disk '$value'\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1565 sub verify_numa {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_numa($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse numa options\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1576 sub verify_net {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_net($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse network options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1587 sub verify_drive {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_drive(undef, $value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse drive options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1598 sub verify_hostpci {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_hostpci($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse pci id\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1609 sub verify_watchdog {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_watchdog($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse watchdog options\n";
1617 }
1618
1619 sub parse_watchdog {
1620     my ($value) = @_;
1621
1622     return undef if !$value;
1623
1624     my $res = {};
1625
1626     foreach my $p (split(/,/, $value)) {
1627         next if $p =~ m/^\s*$/;
1628
1629         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1630             $res->{model} = $2;
1631         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1632             $res->{action} = $2;
1633         } else {
1634             return undef;
1635         }
1636     }
1637
1638     return $res;
1639 }
1640
1641 sub parse_usb_device {
1642     my ($value) = @_;
1643
1644     return undef if !$value;
1645
1646     my @dl = split(/,/, $value);
1647     my $found;
1648
1649     my $res = {};
1650     foreach my $v (@dl) {
1651         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1652             $found = 1;
1653             $res->{vendorid} = $2;
1654             $res->{productid} = $4;
1655         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1656             $found = 1;
1657             $res->{hostbus} = $1;
1658             $res->{hostport} = $2;
1659         } elsif ($v =~ m/^spice$/) {
1660             $found = 1;
1661             $res->{spice} = 1;
1662         } else {
1663             return undef;
1664         }
1665     }
1666     return undef if !$found;
1667
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1672 sub verify_usb_device {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_usb_device($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse usb device\n";
1680 }
1681
1682 # add JSON properties for create and set function
1683 sub json_config_properties {
1684     my $prop = shift;
1685
1686     foreach my $opt (keys %$confdesc) {
1687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1688         $prop->{$opt} = $confdesc->{$opt};
1689     }
1690
1691     return $prop;
1692 }
1693
1694 sub check_type {
1695     my ($key, $value) = @_;
1696
1697     die "unknown setting '$key'\n" if !$confdesc->{$key};
1698
1699     my $type = $confdesc->{$key}->{type};
1700
1701     if (!defined($value)) {
1702         die "got undefined value\n";
1703     }
1704
1705     if ($value =~ m/[\n\r]/) {
1706         die "property contains a line feed\n";
1707     }
1708
1709     if ($type eq 'boolean') {
1710         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1711         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1712         die "type check ('boolean') failed - got '$value'\n";
1713     } elsif ($type eq 'integer') {
1714         return int($1) if $value =~ m/^(\d+)$/;
1715         die "type check ('integer') failed - got '$value'\n";
1716     } elsif ($type eq 'number') {
1717         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1718         die "type check ('number') failed - got '$value'\n";
1719     } elsif ($type eq 'string') {
1720         if (my $fmt = $confdesc->{$key}->{format}) {
1721             if ($fmt eq 'pve-qm-drive') {
1722                 # special case - we need to pass $key to parse_drive()
1723                 my $drive = parse_drive($key, $value);
1724                 return $value if $drive;
1725                 die "unable to parse drive options\n";
1726             }
1727             PVE::JSONSchema::check_format($fmt, $value);
1728             return $value;
1729         }
1730         $value =~ s/^\"(.*)\"$/$1/;
1731         return $value;
1732     } else {
1733         die "internal error"
1734     }
1735 }
1736
1737 sub lock_config_full {
1738     my ($vmid, $timeout, $code, @param) = @_;
1739
1740     my $filename = config_file_lock($vmid);
1741
1742     my $res = lock_file($filename, $timeout, $code, @param);
1743
1744     die $@ if $@;
1745
1746     return $res;
1747 }
1748
1749 sub lock_config_mode {
1750     my ($vmid, $timeout, $shared, $code, @param) = @_;
1751
1752     my $filename = config_file_lock($vmid);
1753
1754     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1755
1756     die $@ if $@;
1757
1758     return $res;
1759 }
1760
1761 sub lock_config {
1762     my ($vmid, $code, @param) = @_;
1763
1764     return lock_config_full($vmid, 10, $code, @param);
1765 }
1766
1767 sub cfs_config_path {
1768     my ($vmid, $node) = @_;
1769
1770     $node = $nodename if !$node;
1771     return "nodes/$node/qemu-server/$vmid.conf";
1772 }
1773
1774 sub check_iommu_support{
1775     #fixme : need to check IOMMU support
1776     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1777
1778     my $iommu=1;
1779     return $iommu;
1780
1781 }
1782
1783 sub config_file {
1784     my ($vmid, $node) = @_;
1785
1786     my $cfspath = cfs_config_path($vmid, $node);
1787     return "/etc/pve/$cfspath";
1788 }
1789
1790 sub config_file_lock {
1791     my ($vmid) = @_;
1792
1793     return "$lock_dir/lock-$vmid.conf";
1794 }
1795
1796 sub touch_config {
1797     my ($vmid) = @_;
1798
1799     my $conf = config_file($vmid);
1800     utime undef, undef, $conf;
1801 }
1802
1803 sub destroy_vm {
1804     my ($storecfg, $vmid, $keep_empty_config) = @_;
1805
1806     my $conffile = config_file($vmid);
1807
1808     my $conf = load_config($vmid);
1809
1810     check_lock($conf);
1811
1812     # only remove disks owned by this VM
1813     foreach_drive($conf, sub {
1814         my ($ds, $drive) = @_;
1815
1816         return if drive_is_cdrom($drive);
1817
1818         my $volid = $drive->{file};
1819
1820         return if !$volid || $volid =~ m|^/|;
1821
1822         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1823         return if !$path || !$owner || ($owner != $vmid);
1824
1825         PVE::Storage::vdisk_free($storecfg, $volid);
1826     });
1827
1828     if ($keep_empty_config) {
1829         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1830     } else {
1831         unlink $conffile;
1832     }
1833
1834     # also remove unused disk
1835     eval {
1836         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1837
1838         eval {
1839             PVE::Storage::foreach_volid($dl, sub {
1840                 my ($volid, $sid, $volname, $d) = @_;
1841                 PVE::Storage::vdisk_free($storecfg, $volid);
1842             });
1843         };
1844         warn $@ if $@;
1845
1846     };
1847     warn $@ if $@;
1848 }
1849
1850 sub load_config {
1851     my ($vmid, $node) = @_;
1852
1853     my $cfspath = cfs_config_path($vmid, $node);
1854
1855     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1856
1857     die "no such VM ('$vmid')\n" if !defined($conf);
1858
1859     return $conf;
1860 }
1861
1862 sub parse_vm_config {
1863     my ($filename, $raw) = @_;
1864
1865     return undef if !defined($raw);
1866
1867     my $res = {
1868         digest => Digest::SHA::sha1_hex($raw),
1869         snapshots => {},
1870         pending => {},
1871     };
1872
1873     $filename =~ m|/qemu-server/(\d+)\.conf$|
1874         || die "got strange filename '$filename'";
1875
1876     my $vmid = $1;
1877
1878     my $conf = $res;
1879     my $descr = '';
1880     my $section = '';
1881
1882     my @lines = split(/\n/, $raw);
1883     foreach my $line (@lines) {
1884         next if $line =~ m/^\s*$/;
1885
1886         if ($line =~ m/^\[PENDING\]\s*$/i) {
1887             $section = 'pending';
1888             $conf->{description} = $descr if $descr;
1889             $descr = '';
1890             $conf = $res->{$section} = {};
1891             next;
1892
1893         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1894             $section = $1;
1895             $conf->{description} = $descr if $descr;
1896             $descr = '';
1897             $conf = $res->{snapshots}->{$section} = {};
1898             next;
1899         }
1900
1901         if ($line =~ m/^\#(.*)\s*$/) {
1902             $descr .= PVE::Tools::decode_text($1) . "\n";
1903             next;
1904         }
1905
1906         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1907             $descr .= PVE::Tools::decode_text($2);
1908         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1909             $conf->{snapstate} = $1;
1910         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1911             my $key = $1;
1912             my $value = $2;
1913             $conf->{$key} = $value;
1914         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1915             my $value = $1;
1916             if ($section eq 'pending') {
1917                 $conf->{delete} = $value; # we parse this later
1918             } else {
1919                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1920             }
1921         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1922             my $key = $1;
1923             my $value = $2;
1924             eval { $value = check_type($key, $value); };
1925             if ($@) {
1926                 warn "vm $vmid - unable to parse value of '$key' - $@";
1927             } else {
1928                 my $fmt = $confdesc->{$key}->{format};
1929                 if ($fmt && $fmt eq 'pve-qm-drive') {
1930                     my $v = parse_drive($key, $value);
1931                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1932                         $v->{file} = $volid;
1933                         $value = print_drive($vmid, $v);
1934                     } else {
1935                         warn "vm $vmid - unable to parse value of '$key'\n";
1936                         next;
1937                     }
1938                 }
1939
1940                 if ($key eq 'cdrom') {
1941                     $conf->{ide2} = $value;
1942                 } else {
1943                     $conf->{$key} = $value;
1944                 }
1945             }
1946         }
1947     }
1948
1949     $conf->{description} = $descr if $descr;
1950
1951     delete $res->{snapstate}; # just to be sure
1952
1953     return $res;
1954 }
1955
1956 sub write_vm_config {
1957     my ($filename, $conf) = @_;
1958
1959     delete $conf->{snapstate}; # just to be sure
1960
1961     if ($conf->{cdrom}) {
1962         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1963         $conf->{ide2} = $conf->{cdrom};
1964         delete $conf->{cdrom};
1965     }
1966
1967     # we do not use 'smp' any longer
1968     if ($conf->{sockets}) {
1969         delete $conf->{smp};
1970     } elsif ($conf->{smp}) {
1971         $conf->{sockets} = $conf->{smp};
1972         delete $conf->{cores};
1973         delete $conf->{smp};
1974     }
1975
1976     my $used_volids = {};
1977
1978     my $cleanup_config = sub {
1979         my ($cref, $pending, $snapname) = @_;
1980
1981         foreach my $key (keys %$cref) {
1982             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1983                 $key eq 'snapstate' || $key eq 'pending';
1984             my $value = $cref->{$key};
1985             if ($key eq 'delete') {
1986                 die "propertry 'delete' is only allowed in [PENDING]\n"
1987                     if !$pending;
1988                 # fixme: check syntax?
1989                 next;
1990             }
1991             eval { $value = check_type($key, $value); };
1992             die "unable to parse value of '$key' - $@" if $@;
1993
1994             $cref->{$key} = $value;
1995
1996             if (!$snapname && valid_drivename($key)) {
1997                 my $drive = parse_drive($key, $value);
1998                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1999             }
2000         }
2001     };
2002
2003     &$cleanup_config($conf);
2004
2005     &$cleanup_config($conf->{pending}, 1);
2006
2007     foreach my $snapname (keys %{$conf->{snapshots}}) {
2008         die "internal error" if $snapname eq 'pending';
2009         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2010     }
2011
2012     # remove 'unusedX' settings if we re-add a volume
2013     foreach my $key (keys %$conf) {
2014         my $value = $conf->{$key};
2015         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2016             delete $conf->{$key};
2017         }
2018     }
2019
2020     my $generate_raw_config = sub {
2021         my ($conf) = @_;
2022
2023         my $raw = '';
2024
2025         # add description as comment to top of file
2026         my $descr = $conf->{description} || '';
2027         foreach my $cl (split(/\n/, $descr)) {
2028             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2029         }
2030
2031         foreach my $key (sort keys %$conf) {
2032             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2033             $raw .= "$key: $conf->{$key}\n";
2034         }
2035         return $raw;
2036     };
2037
2038     my $raw = &$generate_raw_config($conf);
2039
2040     if (scalar(keys %{$conf->{pending}})){
2041         $raw .= "\n[PENDING]\n";
2042         $raw .= &$generate_raw_config($conf->{pending});
2043     }
2044
2045     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2046         $raw .= "\n[$snapname]\n";
2047         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2048     }
2049
2050     return $raw;
2051 }
2052
2053 sub update_config_nolock {
2054     my ($vmid, $conf, $skiplock) = @_;
2055
2056     check_lock($conf) if !$skiplock;
2057
2058     my $cfspath = cfs_config_path($vmid);
2059
2060     PVE::Cluster::cfs_write_file($cfspath, $conf);
2061 }
2062
2063 sub update_config {
2064     my ($vmid, $conf, $skiplock) = @_;
2065
2066     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2067 }
2068
2069 sub load_defaults {
2070
2071     my $res = {};
2072
2073     # we use static defaults from our JSON schema configuration
2074     foreach my $key (keys %$confdesc) {
2075         if (defined(my $default = $confdesc->{$key}->{default})) {
2076             $res->{$key} = $default;
2077         }
2078     }
2079
2080     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2081     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2082
2083     return $res;
2084 }
2085
2086 sub config_list {
2087     my $vmlist = PVE::Cluster::get_vmlist();
2088     my $res = {};
2089     return $res if !$vmlist || !$vmlist->{ids};
2090     my $ids = $vmlist->{ids};
2091
2092     foreach my $vmid (keys %$ids) {
2093         my $d = $ids->{$vmid};
2094         next if !$d->{node} || $d->{node} ne $nodename;
2095         next if !$d->{type} || $d->{type} ne 'qemu';
2096         $res->{$vmid}->{exists} = 1;
2097     }
2098     return $res;
2099 }
2100
2101 # test if VM uses local resources (to prevent migration)
2102 sub check_local_resources {
2103     my ($conf, $noerr) = @_;
2104
2105     my $loc_res = 0;
2106
2107     $loc_res = 1 if $conf->{hostusb}; # old syntax
2108     $loc_res = 1 if $conf->{hostpci}; # old syntax
2109
2110     foreach my $k (keys %$conf) {
2111         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2112         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2113     }
2114
2115     die "VM uses local resources\n" if $loc_res && !$noerr;
2116
2117     return $loc_res;
2118 }
2119
2120 # check if used storages are available on all nodes (use by migrate)
2121 sub check_storage_availability {
2122     my ($storecfg, $conf, $node) = @_;
2123
2124     foreach_drive($conf, sub {
2125         my ($ds, $drive) = @_;
2126
2127         my $volid = $drive->{file};
2128         return if !$volid;
2129
2130         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2131         return if !$sid;
2132
2133         # check if storage is available on both nodes
2134         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2135         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2136    });
2137 }
2138
2139 # list nodes where all VM images are available (used by has_feature API)
2140 sub shared_nodes {
2141     my ($conf, $storecfg) = @_;
2142
2143     my $nodelist = PVE::Cluster::get_nodelist();
2144     my $nodehash = { map { $_ => 1 } @$nodelist };
2145     my $nodename = PVE::INotify::nodename();
2146
2147     foreach_drive($conf, sub {
2148         my ($ds, $drive) = @_;
2149
2150         my $volid = $drive->{file};
2151         return if !$volid;
2152
2153         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2154         if ($storeid) {
2155             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2156             if ($scfg->{disable}) {
2157                 $nodehash = {};
2158             } elsif (my $avail = $scfg->{nodes}) {
2159                 foreach my $node (keys %$nodehash) {
2160                     delete $nodehash->{$node} if !$avail->{$node};
2161                 }
2162             } elsif (!$scfg->{shared}) {
2163                 foreach my $node (keys %$nodehash) {
2164                     delete $nodehash->{$node} if $node ne $nodename
2165                 }
2166             }
2167         }
2168     });
2169
2170     return $nodehash
2171 }
2172
2173 sub check_lock {
2174     my ($conf) = @_;
2175
2176     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2177 }
2178
2179 sub check_cmdline {
2180     my ($pidfile, $pid) = @_;
2181
2182     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2183     if (defined($fh)) {
2184         my $line = <$fh>;
2185         $fh->close;
2186         return undef if !$line;
2187         my @param = split(/\0/, $line);
2188
2189         my $cmd = $param[0];
2190         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2191
2192         for (my $i = 0; $i < scalar (@param); $i++) {
2193             my $p = $param[$i];
2194             next if !$p;
2195             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2196                 my $p = $param[$i+1];
2197                 return 1 if $p && ($p eq $pidfile);
2198                 return undef;
2199             }
2200         }
2201     }
2202     return undef;
2203 }
2204
2205 sub check_running {
2206     my ($vmid, $nocheck, $node) = @_;
2207
2208     my $filename = config_file($vmid, $node);
2209
2210     die "unable to find configuration file for VM $vmid - no such machine\n"
2211         if !$nocheck && ! -f $filename;
2212
2213     my $pidfile = pidfile_name($vmid);
2214
2215     if (my $fd = IO::File->new("<$pidfile")) {
2216         my $st = stat($fd);
2217         my $line = <$fd>;
2218         close($fd);
2219
2220         my $mtime = $st->mtime;
2221         if ($mtime > time()) {
2222             warn "file '$filename' modified in future\n";
2223         }
2224
2225         if ($line =~ m/^(\d+)$/) {
2226             my $pid = $1;
2227             if (check_cmdline($pidfile, $pid)) {
2228                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2229                     return $pid;
2230                 }
2231             }
2232         }
2233     }
2234
2235     return undef;
2236 }
2237
2238 sub vzlist {
2239
2240     my $vzlist = config_list();
2241
2242     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2243
2244     while (defined(my $de = $fd->read)) {
2245         next if $de !~ m/^(\d+)\.pid$/;
2246         my $vmid = $1;
2247         next if !defined($vzlist->{$vmid});
2248         if (my $pid = check_running($vmid)) {
2249             $vzlist->{$vmid}->{pid} = $pid;
2250         }
2251     }
2252
2253     return $vzlist;
2254 }
2255
2256 sub disksize {
2257     my ($storecfg, $conf) = @_;
2258
2259     my $bootdisk = $conf->{bootdisk};
2260     return undef if !$bootdisk;
2261     return undef if !valid_drivename($bootdisk);
2262
2263     return undef if !$conf->{$bootdisk};
2264
2265     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2266     return undef if !defined($drive);
2267
2268     return undef if drive_is_cdrom($drive);
2269
2270     my $volid = $drive->{file};
2271     return undef if !$volid;
2272
2273     return $drive->{size};
2274 }
2275
2276 my $last_proc_pid_stat;
2277
2278 # get VM status information
2279 # This must be fast and should not block ($full == false)
2280 # We only query KVM using QMP if $full == true (this can be slow)
2281 sub vmstatus {
2282     my ($opt_vmid, $full) = @_;
2283
2284     my $res = {};
2285
2286     my $storecfg = PVE::Storage::config();
2287
2288     my $list = vzlist();
2289     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2290
2291     my $cpucount = $cpuinfo->{cpus} || 1;
2292
2293     foreach my $vmid (keys %$list) {
2294         next if $opt_vmid && ($vmid ne $opt_vmid);
2295
2296         my $cfspath = cfs_config_path($vmid);
2297         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2298
2299         my $d = {};
2300         $d->{pid} = $list->{$vmid}->{pid};
2301
2302         # fixme: better status?
2303         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2304
2305         my $size = disksize($storecfg, $conf);
2306         if (defined($size)) {
2307             $d->{disk} = 0; # no info available
2308             $d->{maxdisk} = $size;
2309         } else {
2310             $d->{disk} = 0;
2311             $d->{maxdisk} = 0;
2312         }
2313
2314         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2315         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2316         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2317
2318         $d->{name} = $conf->{name} || "VM $vmid";
2319         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2320
2321         if ($conf->{balloon}) {
2322             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2323             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2324         }
2325
2326         $d->{uptime} = 0;
2327         $d->{cpu} = 0;
2328         $d->{mem} = 0;
2329
2330         $d->{netout} = 0;
2331         $d->{netin} = 0;
2332
2333         $d->{diskread} = 0;
2334         $d->{diskwrite} = 0;
2335
2336         $d->{template} = is_template($conf);
2337
2338         $res->{$vmid} = $d;
2339     }
2340
2341     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2342     foreach my $dev (keys %$netdev) {
2343         next if $dev !~ m/^tap([1-9]\d*)i/;
2344         my $vmid = $1;
2345         my $d = $res->{$vmid};
2346         next if !$d;
2347
2348         $d->{netout} += $netdev->{$dev}->{receive};
2349         $d->{netin} += $netdev->{$dev}->{transmit};
2350
2351         if ($full) {
2352             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2353             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2354         }
2355
2356     }
2357
2358     my $ctime = gettimeofday;
2359
2360     foreach my $vmid (keys %$list) {
2361
2362         my $d = $res->{$vmid};
2363         my $pid = $d->{pid};
2364         next if !$pid;
2365
2366         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2367         next if !$pstat; # not running
2368
2369         my $used = $pstat->{utime} + $pstat->{stime};
2370
2371         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2372
2373         if ($pstat->{vsize}) {
2374             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2375         }
2376
2377         my $old = $last_proc_pid_stat->{$pid};
2378         if (!$old) {
2379             $last_proc_pid_stat->{$pid} = {
2380                 time => $ctime,
2381                 used => $used,
2382                 cpu => 0,
2383             };
2384             next;
2385         }
2386
2387         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2388
2389         if ($dtime > 1000) {
2390             my $dutime = $used -  $old->{used};
2391
2392             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2393             $last_proc_pid_stat->{$pid} = {
2394                 time => $ctime,
2395                 used => $used,
2396                 cpu => $d->{cpu},
2397             };
2398         } else {
2399             $d->{cpu} = $old->{cpu};
2400         }
2401     }
2402
2403     return $res if !$full;
2404
2405     my $qmpclient = PVE::QMPClient->new();
2406
2407     my $ballooncb = sub {
2408         my ($vmid, $resp) = @_;
2409
2410         my $info = $resp->{'return'};
2411         return if !$info->{max_mem};
2412
2413         my $d = $res->{$vmid};
2414
2415         # use memory assigned to VM
2416         $d->{maxmem} = $info->{max_mem};
2417         $d->{balloon} = $info->{actual};
2418
2419         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2420             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2421             $d->{freemem} = $info->{free_mem};
2422         }
2423
2424         $d->{ballooninfo} = $info;
2425     };
2426
2427     my $blockstatscb = sub {
2428         my ($vmid, $resp) = @_;
2429         my $data = $resp->{'return'} || [];
2430         my $totalrdbytes = 0;
2431         my $totalwrbytes = 0;
2432
2433         for my $blockstat (@$data) {
2434             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2435             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2436
2437             $blockstat->{device} =~ s/drive-//;
2438             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2439         }
2440         $res->{$vmid}->{diskread} = $totalrdbytes;
2441         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2442     };
2443
2444     my $statuscb = sub {
2445         my ($vmid, $resp) = @_;
2446
2447         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2448         # this fails if ballon driver is not loaded, so this must be
2449         # the last commnand (following command are aborted if this fails).
2450         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2451
2452         my $status = 'unknown';
2453         if (!defined($status = $resp->{'return'}->{status})) {
2454             warn "unable to get VM status\n";
2455             return;
2456         }
2457
2458         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2459     };
2460
2461     foreach my $vmid (keys %$list) {
2462         next if $opt_vmid && ($vmid ne $opt_vmid);
2463         next if !$res->{$vmid}->{pid}; # not running
2464         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2465     }
2466
2467     $qmpclient->queue_execute(undef, 1);
2468
2469     foreach my $vmid (keys %$list) {
2470         next if $opt_vmid && ($vmid ne $opt_vmid);
2471         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2472     }
2473
2474     return $res;
2475 }
2476
2477 sub foreach_dimm {
2478     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2479
2480     my $dimm_id = 0;
2481     my $current_size = 1024;
2482     my $dimm_size = 512;
2483     return if $current_size == $memory;
2484
2485     for (my $j = 0; $j < 8; $j++) {
2486         for (my $i = 0; $i < 32; $i++) {
2487             my $name = "dimm${dimm_id}";
2488             $dimm_id++;
2489             my $numanode = $i % $sockets;
2490             $current_size += $dimm_size;
2491             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2492             return  $current_size if $current_size >= $memory;
2493         }
2494         $dimm_size *= 2;
2495     }
2496 }
2497
2498 sub foreach_reverse_dimm {
2499     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2500
2501     my $dimm_id = 253;
2502     my $current_size = 4177920;
2503     my $dimm_size = 65536;
2504     return if $current_size == $memory;
2505
2506     for (my $j = 0; $j < 8; $j++) {
2507         for (my $i = 0; $i < 32; $i++) {
2508             my $name = "dimm${dimm_id}";
2509             $dimm_id--;
2510             my $numanode = $i % $sockets;
2511             $current_size -= $dimm_size;
2512             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2513             return  $current_size if $current_size <= $memory;
2514         }
2515         $dimm_size /= 2;
2516     }
2517 }
2518
2519 sub foreach_drive {
2520     my ($conf, $func) = @_;
2521
2522     foreach my $ds (keys %$conf) {
2523         next if !valid_drivename($ds);
2524
2525         my $drive = parse_drive($ds, $conf->{$ds});
2526         next if !$drive;
2527
2528         &$func($ds, $drive);
2529     }
2530 }
2531
2532 sub foreach_volid {
2533     my ($conf, $func) = @_;
2534
2535     my $volhash = {};
2536
2537     my $test_volid = sub {
2538         my ($volid, $is_cdrom) = @_;
2539
2540         return if !$volid;
2541
2542         $volhash->{$volid} = $is_cdrom || 0;
2543     };
2544
2545     foreach_drive($conf, sub {
2546         my ($ds, $drive) = @_;
2547         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2548     });
2549
2550     foreach my $snapname (keys %{$conf->{snapshots}}) {
2551         my $snap = $conf->{snapshots}->{$snapname};
2552         &$test_volid($snap->{vmstate}, 0);
2553         foreach_drive($snap, sub {
2554             my ($ds, $drive) = @_;
2555             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2556         });
2557     }
2558
2559     foreach my $volid (keys %$volhash) {
2560         &$func($volid, $volhash->{$volid});
2561     }
2562 }
2563
2564 sub vga_conf_has_spice {
2565     my ($vga) = @_;
2566
2567     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2568
2569     return $1 || 1;
2570 }
2571
2572 sub config_to_command {
2573     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2574
2575     my $cmd = [];
2576     my $globalFlags = [];
2577     my $machineFlags = [];
2578     my $rtcFlags = [];
2579     my $cpuFlags = [];
2580     my $devices = [];
2581     my $pciaddr = '';
2582     my $bridges = {};
2583     my $kvmver = kvm_user_version();
2584     my $vernum = 0; # unknown
2585     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2586         $vernum = $1*1000000+$2*1000;
2587     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2588         $vernum = $1*1000000+$2*1000+$3;
2589     }
2590
2591     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2592
2593     my $have_ovz = -f '/proc/vz/vestat';
2594
2595     my $q35 = machine_type_is_q35($conf);
2596     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2597     my $machine_type = $forcemachine || $conf->{machine};
2598
2599     my $cpuunits = defined($conf->{cpuunits}) ?
2600             $conf->{cpuunits} : $defaults->{cpuunits};
2601
2602     push @$cmd, '/usr/bin/systemd-run';
2603     push @$cmd, '--scope';
2604     push @$cmd, '--slice', "qemu";
2605     push @$cmd, '--unit', $vmid;
2606     push @$cmd, '-p', "CPUShares=$cpuunits";
2607     if ($conf->{cpulimit}) {
2608         my $cpulimit = int($conf->{cpulimit} * 100);
2609         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2610     }
2611
2612     push @$cmd, '/usr/bin/kvm';
2613
2614     push @$cmd, '-id', $vmid;
2615
2616     my $use_virtio = 0;
2617
2618     my $qmpsocket = qmp_socket($vmid);
2619     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2620     push @$cmd, '-mon', "chardev=qmp,mode=control";
2621
2622     my $socket = vnc_socket($vmid);
2623     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2624
2625     push @$cmd, '-pidfile' , pidfile_name($vmid);
2626
2627     push @$cmd, '-daemonize';
2628
2629     if ($conf->{smbios1}) {
2630         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2631     }
2632
2633     if ($q35) {
2634         # the q35 chipset support native usb2, so we enable usb controller
2635         # by default for this machine type
2636         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2637     } else {
2638         $pciaddr = print_pci_addr("piix3", $bridges);
2639         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2640
2641         my $use_usb2 = 0;
2642         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2643             next if !$conf->{"usb$i"};
2644             $use_usb2 = 1;
2645         }
2646         # include usb device config
2647         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2648     }
2649
2650     my $vga = $conf->{vga};
2651
2652     my $qxlnum = vga_conf_has_spice($vga);
2653     $vga = 'qxl' if $qxlnum;
2654
2655     if (!$vga) {
2656         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2657                                 $conf->{ostype} eq 'win7' ||
2658                                 $conf->{ostype} eq 'w2k8')) {
2659             $vga = 'std';
2660         } else {
2661             $vga = 'cirrus';
2662         }
2663     }
2664
2665     # enable absolute mouse coordinates (needed by vnc)
2666     my $tablet;
2667     if (defined($conf->{tablet})) {
2668         $tablet = $conf->{tablet};
2669     } else {
2670         $tablet = $defaults->{tablet};
2671         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2672         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2673     }
2674
2675     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2676
2677     # host pci devices
2678     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2679         my $d = parse_hostpci($conf->{"hostpci$i"});
2680         next if !$d;
2681
2682         my $pcie = $d->{pcie};
2683         if($pcie){
2684             die "q35 machine model is not enabled" if !$q35;
2685             $pciaddr = print_pcie_addr("hostpci$i");
2686         }else{
2687             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2688         }
2689
2690         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2691         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2692         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2693         if ($xvga && $xvga ne '') {
2694             push @$cpuFlags, 'kvm=off';
2695             $vga = 'none';
2696         }
2697         $driver = "vfio-pci" if $xvga ne '';
2698         my $pcidevices = $d->{pciid};
2699         my $multifunction = 1 if @$pcidevices > 1;
2700
2701         my $j=0;
2702         foreach my $pcidevice (@$pcidevices) {
2703
2704             my $id = "hostpci$i";
2705             $id .= ".$j" if $multifunction;
2706             my $addr = $pciaddr;
2707             $addr .= ".$j" if $multifunction;
2708             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2709
2710             if($j == 0){
2711                 $devicestr .= "$rombar$xvga";
2712                 $devicestr .= ",multifunction=on" if $multifunction;
2713             }
2714
2715             push @$devices, '-device', $devicestr;
2716             $j++;
2717         }
2718     }
2719
2720     # usb devices
2721     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2722         my $d = parse_usb_device($conf->{"usb$i"});
2723         next if !$d;
2724         if ($d->{vendorid} && $d->{productid}) {
2725             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2726         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2727             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2728         } elsif ($d->{spice}) {
2729             # usb redir support for spice
2730             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2731             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2732         }
2733     }
2734
2735     # serial devices
2736     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2737         if (my $path = $conf->{"serial$i"}) {
2738             if ($path eq 'socket') {
2739                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2740                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2741                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2742             } else {
2743                 die "no such serial device\n" if ! -c $path;
2744                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2745                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2746             }
2747         }
2748     }
2749
2750     # parallel devices
2751     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2752         if (my $path = $conf->{"parallel$i"}) {
2753             die "no such parallel device\n" if ! -c $path;
2754             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2755             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2756             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2757         }
2758     }
2759
2760     my $vmname = $conf->{name} || "vm$vmid";
2761
2762     push @$cmd, '-name', $vmname;
2763
2764     my $sockets = 1;
2765     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2766     $sockets = $conf->{sockets} if  $conf->{sockets};
2767
2768     my $cores = $conf->{cores} || 1;
2769
2770     my $maxcpus = $sockets * $cores;
2771
2772     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2773
2774     my $allowed_vcpus = $cpuinfo->{cpus};
2775
2776     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2777         if ($allowed_vcpus < $maxcpus);
2778
2779     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2780
2781     push @$cmd, '-nodefaults';
2782
2783     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2784
2785     my $bootindex_hash = {};
2786     my $i = 1;
2787     foreach my $o (split(//, $bootorder)) {
2788         $bootindex_hash->{$o} = $i*100;
2789         $i++;
2790     }
2791
2792     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2793
2794     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2795
2796     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2797
2798     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2799
2800     # time drift fix
2801     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2802
2803     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2804     my $useLocaltime = $conf->{localtime};
2805
2806     if (my $ost = $conf->{ostype}) {
2807         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2808
2809         if ($ost =~ m/^w/) { # windows
2810             $useLocaltime = 1 if !defined($conf->{localtime});
2811
2812             # use time drift fix when acpi is enabled
2813             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2814                 $tdf = 1 if !defined($conf->{tdf});
2815             }
2816         }
2817
2818         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2819             $ost eq 'wvista') {
2820             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2821             push @$cmd, '-no-hpet';
2822             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2823                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2824                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2825                 push @$cpuFlags , 'hv_time' if !$nokvm;
2826
2827             } else {
2828                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2829             }
2830         }
2831
2832         if ($ost eq 'win7' || $ost eq 'win8') {
2833             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2834         }
2835     }
2836
2837     push @$rtcFlags, 'driftfix=slew' if $tdf;
2838
2839     if ($nokvm) {
2840         push @$machineFlags, 'accel=tcg';
2841     } else {
2842         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2843     }
2844
2845     if ($machine_type) {
2846         push @$machineFlags, "type=${machine_type}";
2847     }
2848
2849     if ($conf->{startdate}) {
2850         push @$rtcFlags, "base=$conf->{startdate}";
2851     } elsif ($useLocaltime) {
2852         push @$rtcFlags, 'base=localtime';
2853     }
2854
2855     my $cpu = $nokvm ? "qemu64" : "kvm64";
2856     $cpu = $conf->{cpu} if $conf->{cpu};
2857
2858     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2859
2860     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2861
2862     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2863
2864     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2865
2866     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2867
2868     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2869
2870         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2871         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2872     }
2873
2874     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2875
2876     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2877
2878     push @$cmd, '-cpu', $cpu;
2879
2880     my $memory = $conf->{memory} || $defaults->{memory};
2881     my $static_memory = 0;
2882     my $dimm_memory = 0;
2883
2884     if ($hotplug_features->{memory}) {
2885         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2886         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2887         $static_memory = $STATICMEM;
2888         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2889         $dimm_memory = $memory - $static_memory;
2890         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2891
2892     } else {
2893
2894         $static_memory = $memory;
2895         push @$cmd, '-m', $static_memory;
2896     }
2897
2898     if ($conf->{numa}) {
2899
2900         my $numa_totalmemory = undef;
2901         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2902             next if !$conf->{"numa$i"};
2903             my $numa = parse_numa($conf->{"numa$i"});
2904             next if !$numa;
2905             # memory
2906             die "missing numa node$i memory value\n" if !$numa->{memory};
2907             my $numa_memory = $numa->{memory};
2908             $numa_totalmemory += $numa_memory;
2909             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2910
2911             # cpus
2912             my $cpus_start = $numa->{cpus}->{start};
2913             die "missing numa node$i cpus\n" if !defined($cpus_start);
2914             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2915             my $cpus = $cpus_start;
2916             if (defined($cpus_end)) {
2917                 $cpus .= "-$cpus_end";
2918                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2919             }
2920
2921             # hostnodes
2922             my $hostnodes_start = $numa->{hostnodes}->{start};
2923             if (defined($hostnodes_start)) {
2924                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2925                 my $hostnodes = $hostnodes_start;
2926                 if (defined($hostnodes_end)) {
2927                     $hostnodes .= "-$hostnodes_end";
2928                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2929                 }
2930
2931                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2932                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2933                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2934                 }
2935
2936                 # policy
2937                 my $policy = $numa->{policy};
2938                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2939                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2940             }
2941
2942             push @$cmd, '-object', $numa_object;
2943             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2944         }
2945
2946         die "total memory for NUMA nodes must be equal to vm static memory\n"
2947             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2948
2949         #if no custom tology, we split memory and cores across numa nodes
2950         if(!$numa_totalmemory) {
2951
2952             my $numa_memory = ($static_memory / $sockets) . "M";
2953
2954             for (my $i = 0; $i < $sockets; $i++)  {
2955
2956                 my $cpustart = ($cores * $i);
2957                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2958                 my $cpus = $cpustart;
2959                 $cpus .= "-$cpuend" if $cpuend;
2960
2961                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2962                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2963             }
2964         }
2965     }
2966
2967     if ($hotplug_features->{memory}) {
2968         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2969             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2970             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2971             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2972
2973             #if dimm_memory is not aligned to dimm map
2974             if($current_size > $memory) {
2975                  $conf->{memory} = $current_size;
2976                  update_config_nolock($vmid, $conf, 1);
2977             }
2978         });
2979     }
2980
2981     push @$cmd, '-S' if $conf->{freeze};
2982
2983     # set keyboard layout
2984     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2985     push @$cmd, '-k', $kb if $kb;
2986
2987     # enable sound
2988     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2989     #push @$cmd, '-soundhw', 'es1370';
2990     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2991
2992     if($conf->{agent}) {
2993         my $qgasocket = qmp_socket($vmid, 1);
2994         my $pciaddr = print_pci_addr("qga0", $bridges);
2995         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2996         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2997         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2998     }
2999
3000     my $spice_port;
3001
3002     if ($qxlnum) {
3003         if ($qxlnum > 1) {
3004             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3005                 for(my $i = 1; $i < $qxlnum; $i++){
3006                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3007                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3008                 }
3009             } else {
3010                 # assume other OS works like Linux
3011                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3012                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3013             }
3014         }
3015
3016         my $pciaddr = print_pci_addr("spice", $bridges);
3017
3018         my $nodename = PVE::INotify::nodename();
3019         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3020         $spice_port = PVE::Tools::next_spice_port($pfamily);
3021
3022         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3023
3024         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3025         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3026         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3027     }
3028
3029     # enable balloon by default, unless explicitly disabled
3030     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3031         $pciaddr = print_pci_addr("balloon0", $bridges);
3032         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3033     }
3034
3035     if ($conf->{watchdog}) {
3036         my $wdopts = parse_watchdog($conf->{watchdog});
3037         $pciaddr = print_pci_addr("watchdog", $bridges);
3038         my $watchdog = $wdopts->{model} || 'i6300esb';
3039         push @$devices, '-device', "$watchdog$pciaddr";
3040         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3041     }
3042
3043     my $vollist = [];
3044     my $scsicontroller = {};
3045     my $ahcicontroller = {};
3046     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3047
3048     # Add iscsi initiator name if available
3049     if (my $initiator = get_initiator_name()) {
3050         push @$devices, '-iscsi', "initiator-name=$initiator";
3051     }
3052
3053     foreach_drive($conf, sub {
3054         my ($ds, $drive) = @_;
3055
3056         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3057             push @$vollist, $drive->{file};
3058         }
3059
3060         $use_virtio = 1 if $ds =~ m/^virtio/;
3061
3062         if (drive_is_cdrom ($drive)) {
3063             if ($bootindex_hash->{d}) {
3064                 $drive->{bootindex} = $bootindex_hash->{d};
3065                 $bootindex_hash->{d} += 1;
3066             }
3067         } else {
3068             if ($bootindex_hash->{c}) {
3069                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3070                 $bootindex_hash->{c} += 1;
3071             }
3072         }
3073
3074         if($drive->{interface} eq 'virtio'){
3075            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3076         }
3077
3078         if ($drive->{interface} eq 'scsi') {
3079
3080             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3081
3082             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3083             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3084
3085             my $iothread = '';
3086             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3087                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3088                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3089             }
3090
3091             my $queues = '';
3092             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3093                 $queues = ",num_queues=$drive->{queues}";
3094             }
3095
3096             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3097             $scsicontroller->{$controller}=1;
3098         }
3099
3100         if ($drive->{interface} eq 'sata') {
3101            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3102            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3103            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3104            $ahcicontroller->{$controller}=1;
3105         }
3106
3107         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3108         push @$devices, '-drive',$drive_cmd;
3109         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3110     });
3111
3112     for (my $i = 0; $i < $MAX_NETS; $i++) {
3113          next if !$conf->{"net$i"};
3114          my $d = parse_net($conf->{"net$i"});
3115          next if !$d;
3116
3117          $use_virtio = 1 if $d->{model} eq 'virtio';
3118
3119          if ($bootindex_hash->{n}) {
3120             $d->{bootindex} = $bootindex_hash->{n};
3121             $bootindex_hash->{n} += 1;
3122          }
3123
3124          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3125          push @$devices, '-netdev', $netdevfull;
3126
3127          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3128          push @$devices, '-device', $netdevicefull;
3129     }
3130
3131     if (!$q35) {
3132         # add pci bridges
3133         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3134            $bridges->{1} = 1;
3135            $bridges->{2} = 1;
3136         }
3137
3138         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3139
3140         while (my ($k, $v) = each %$bridges) {
3141             $pciaddr = print_pci_addr("pci.$k");
3142             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3143         }
3144     }
3145
3146     # add custom args
3147     if ($conf->{args}) {
3148         my $aa = PVE::Tools::split_args($conf->{args});
3149         push @$cmd, @$aa;
3150     }
3151
3152     push @$cmd, @$devices;
3153     push @$cmd, '-rtc', join(',', @$rtcFlags)
3154         if scalar(@$rtcFlags);
3155     push @$cmd, '-machine', join(',', @$machineFlags)
3156         if scalar(@$machineFlags);
3157     push @$cmd, '-global', join(',', @$globalFlags)
3158         if scalar(@$globalFlags);
3159
3160     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3161 }
3162
3163 sub vnc_socket {
3164     my ($vmid) = @_;
3165     return "${var_run_tmpdir}/$vmid.vnc";
3166 }
3167
3168 sub spice_port {
3169     my ($vmid) = @_;
3170
3171     my $res = vm_mon_cmd($vmid, 'query-spice');
3172
3173     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3174 }
3175
3176 sub qmp_socket {
3177     my ($vmid, $qga) = @_;
3178     my $sockettype = $qga ? 'qga' : 'qmp';
3179     return "${var_run_tmpdir}/$vmid.$sockettype";
3180 }
3181
3182 sub pidfile_name {
3183     my ($vmid) = @_;
3184     return "${var_run_tmpdir}/$vmid.pid";
3185 }
3186
3187 sub vm_devices_list {
3188     my ($vmid) = @_;
3189
3190     my $res = vm_mon_cmd($vmid, 'query-pci');
3191     my $devices = {};
3192     foreach my $pcibus (@$res) {
3193         foreach my $device (@{$pcibus->{devices}}) {
3194             next if !$device->{'qdev_id'};
3195             if ($device->{'pci_bridge'}) {
3196                 $devices->{$device->{'qdev_id'}} = 1;
3197                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3198                     next if !$bridge_device->{'qdev_id'};
3199                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3200                     $devices->{$device->{'qdev_id'}}++;
3201                 }
3202             } else {
3203                 $devices->{$device->{'qdev_id'}} = 1;
3204             }
3205         }
3206     }
3207
3208     my $resblock = vm_mon_cmd($vmid, 'query-block');
3209     foreach my $block (@$resblock) {
3210         if($block->{device} =~ m/^drive-(\S+)/){
3211                 $devices->{$1} = 1;
3212         }
3213     }
3214
3215     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3216     foreach my $mice (@$resmice) {
3217         if ($mice->{name} eq 'QEMU HID Tablet') {
3218             $devices->{tablet} = 1;
3219             last;
3220         }
3221     }
3222
3223     return $devices;
3224 }
3225
3226 sub vm_deviceplug {
3227     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3228
3229     my $q35 = machine_type_is_q35($conf);
3230
3231     my $devices_list = vm_devices_list($vmid);
3232     return 1 if defined($devices_list->{$deviceid});
3233
3234     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3235
3236     if ($deviceid eq 'tablet') {
3237
3238         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3239
3240     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3241
3242         qemu_iothread_add($vmid, $deviceid, $device);
3243
3244         qemu_driveadd($storecfg, $vmid, $device);
3245         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3246
3247         qemu_deviceadd($vmid, $devicefull);
3248         eval { qemu_deviceaddverify($vmid, $deviceid); };
3249         if (my $err = $@) {
3250             eval { qemu_drivedel($vmid, $deviceid); };
3251             warn $@ if $@;
3252             die $err;
3253         }
3254
3255     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3256
3257
3258         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3259         my $pciaddr = print_pci_addr($deviceid);
3260         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3261
3262         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3263
3264         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3265             qemu_iothread_add($vmid, $deviceid, $device);
3266             $devicefull .= ",iothread=iothread-$deviceid";
3267         }
3268
3269         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3270             $devicefull .= ",num_queues=$device->{queues}";
3271         }
3272
3273         qemu_deviceadd($vmid, $devicefull);
3274         qemu_deviceaddverify($vmid, $deviceid);
3275
3276     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3277
3278         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3279         qemu_driveadd($storecfg, $vmid, $device);
3280
3281         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3282         eval { qemu_deviceadd($vmid, $devicefull); };
3283         if (my $err = $@) {
3284             eval { qemu_drivedel($vmid, $deviceid); };
3285             warn $@ if $@;
3286             die $err;
3287         }
3288
3289     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3290
3291         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3292         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3293         qemu_deviceadd($vmid, $netdevicefull);
3294         eval { qemu_deviceaddverify($vmid, $deviceid); };
3295         if (my $err = $@) {
3296             eval { qemu_netdevdel($vmid, $deviceid); };
3297             warn $@ if $@;
3298             die $err;
3299         }
3300
3301     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3302
3303         my $bridgeid = $2;
3304         my $pciaddr = print_pci_addr($deviceid);
3305         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3306
3307         qemu_deviceadd($vmid, $devicefull);
3308         qemu_deviceaddverify($vmid, $deviceid);
3309
3310     } else {
3311         die "can't hotplug device '$deviceid'\n";
3312     }
3313
3314     return 1;
3315 }
3316
3317 # fixme: this should raise exceptions on error!
3318 sub vm_deviceunplug {
3319     my ($vmid, $conf, $deviceid) = @_;
3320
3321     my $devices_list = vm_devices_list($vmid);
3322     return 1 if !defined($devices_list->{$deviceid});
3323
3324     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3325
3326     if ($deviceid eq 'tablet') {
3327
3328         qemu_devicedel($vmid, $deviceid);
3329
3330     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3331
3332         qemu_devicedel($vmid, $deviceid);
3333         qemu_devicedelverify($vmid, $deviceid);
3334         qemu_drivedel($vmid, $deviceid);
3335         qemu_iothread_del($conf, $vmid, $deviceid);
3336
3337     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3338
3339         qemu_devicedel($vmid, $deviceid);
3340         qemu_devicedelverify($vmid, $deviceid);
3341         qemu_iothread_del($conf, $vmid, $deviceid);
3342
3343     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3344
3345         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3346         my $device = parse_drive($deviceid, $conf->{$deviceid});
3347         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3348
3349         qemu_devicedel($vmid, $deviceid);
3350         qemu_drivedel($vmid, $deviceid);
3351         qemu_deletescsihw($conf, $vmid, $deviceid);
3352
3353     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3354
3355         qemu_devicedel($vmid, $deviceid);
3356         qemu_devicedelverify($vmid, $deviceid);
3357         qemu_netdevdel($vmid, $deviceid);
3358
3359     } else {
3360         die "can't unplug device '$deviceid'\n";
3361     }
3362
3363     return 1;
3364 }
3365
3366 sub qemu_deviceadd {
3367     my ($vmid, $devicefull) = @_;
3368
3369     $devicefull = "driver=".$devicefull;
3370     my %options =  split(/[=,]/, $devicefull);
3371
3372     vm_mon_cmd($vmid, "device_add" , %options);
3373 }
3374
3375 sub qemu_devicedel {
3376     my ($vmid, $deviceid) = @_;
3377
3378     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3379 }
3380
3381 sub qemu_iothread_add {
3382     my($vmid, $deviceid, $device) = @_;
3383
3384     if ($device->{iothread}) {
3385         my $iothreads = vm_iothreads_list($vmid);
3386         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3387     }
3388 }
3389
3390 sub qemu_iothread_del {
3391     my($conf, $vmid, $deviceid) = @_;
3392
3393     my $device = parse_drive($deviceid, $conf->{$deviceid});
3394     if ($device->{iothread}) {
3395         my $iothreads = vm_iothreads_list($vmid);
3396         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3397     }
3398 }
3399
3400 sub qemu_objectadd {
3401     my($vmid, $objectid, $qomtype) = @_;
3402
3403     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3404
3405     return 1;
3406 }
3407
3408 sub qemu_objectdel {
3409     my($vmid, $objectid) = @_;
3410
3411     vm_mon_cmd($vmid, "object-del", id => $objectid);
3412
3413     return 1;
3414 }
3415
3416 sub qemu_driveadd {
3417     my ($storecfg, $vmid, $device) = @_;
3418
3419     my $drive = print_drive_full($storecfg, $vmid, $device);
3420     $drive =~ s/\\/\\\\/g;
3421     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3422
3423     # If the command succeeds qemu prints: "OK"
3424     return 1 if $ret =~ m/OK/s;
3425
3426     die "adding drive failed: $ret\n";
3427 }
3428
3429 sub qemu_drivedel {
3430     my($vmid, $deviceid) = @_;
3431
3432     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3433     $ret =~ s/^\s+//;
3434
3435     return 1 if $ret eq "";
3436
3437     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3438     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3439
3440     die "deleting drive $deviceid failed : $ret\n";
3441 }
3442
3443 sub qemu_deviceaddverify {
3444     my ($vmid, $deviceid) = @_;
3445
3446     for (my $i = 0; $i <= 5; $i++) {
3447          my $devices_list = vm_devices_list($vmid);
3448          return 1 if defined($devices_list->{$deviceid});
3449          sleep 1;
3450     }
3451
3452     die "error on hotplug device '$deviceid'\n";
3453 }
3454
3455
3456 sub qemu_devicedelverify {
3457     my ($vmid, $deviceid) = @_;
3458
3459     # need to verify that the device is correctly removed as device_del
3460     # is async and empty return is not reliable
3461
3462     for (my $i = 0; $i <= 5; $i++) {
3463          my $devices_list = vm_devices_list($vmid);
3464          return 1 if !defined($devices_list->{$deviceid});
3465          sleep 1;
3466     }
3467
3468     die "error on hot-unplugging device '$deviceid'\n";
3469 }
3470
3471 sub qemu_findorcreatescsihw {
3472     my ($storecfg, $conf, $vmid, $device) = @_;
3473
3474     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3475
3476     my $scsihwid="$controller_prefix$controller";
3477     my $devices_list = vm_devices_list($vmid);
3478
3479     if(!defined($devices_list->{$scsihwid})) {
3480         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3481     }
3482
3483     return 1;
3484 }
3485
3486 sub qemu_deletescsihw {
3487     my ($conf, $vmid, $opt) = @_;
3488
3489     my $device = parse_drive($opt, $conf->{$opt});
3490
3491     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3492         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3493         return 1;
3494     }
3495
3496     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3497
3498     my $devices_list = vm_devices_list($vmid);
3499     foreach my $opt (keys %{$devices_list}) {
3500         if (PVE::QemuServer::valid_drivename($opt)) {
3501             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3502             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3503                 return 1;
3504             }
3505         }
3506     }
3507
3508     my $scsihwid="scsihw$controller";
3509
3510     vm_deviceunplug($vmid, $conf, $scsihwid);
3511
3512     return 1;
3513 }
3514
3515 sub qemu_add_pci_bridge {
3516     my ($storecfg, $conf, $vmid, $device) = @_;
3517
3518     my $bridges = {};
3519
3520     my $bridgeid;
3521
3522     print_pci_addr($device, $bridges);
3523
3524     while (my ($k, $v) = each %$bridges) {
3525         $bridgeid = $k;
3526     }
3527     return 1 if !defined($bridgeid) || $bridgeid < 1;
3528
3529     my $bridge = "pci.$bridgeid";
3530     my $devices_list = vm_devices_list($vmid);
3531
3532     if (!defined($devices_list->{$bridge})) {
3533         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3534     }
3535
3536     return 1;
3537 }
3538
3539 sub qemu_set_link_status {
3540     my ($vmid, $device, $up) = @_;
3541
3542     vm_mon_cmd($vmid, "set_link", name => $device,
3543                up => $up ? JSON::true : JSON::false);
3544 }
3545
3546 sub qemu_netdevadd {
3547     my ($vmid, $conf, $device, $deviceid) = @_;
3548
3549     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3550     my %options =  split(/[=,]/, $netdev);
3551
3552     vm_mon_cmd($vmid, "netdev_add",  %options);
3553     return 1;
3554 }
3555
3556 sub qemu_netdevdel {
3557     my ($vmid, $deviceid) = @_;
3558
3559     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3560 }
3561
3562 sub qemu_cpu_hotplug {
3563     my ($vmid, $conf, $vcpus) = @_;
3564
3565     my $sockets = 1;
3566     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3567     $sockets = $conf->{sockets} if  $conf->{sockets};
3568     my $cores = $conf->{cores} || 1;
3569     my $maxcpus = $sockets * $cores;
3570
3571     $vcpus = $maxcpus if !$vcpus;
3572
3573     die "you can't add more vcpus than maxcpus\n"
3574         if $vcpus > $maxcpus;
3575
3576     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3577     die "online cpu unplug is not yet possible\n"
3578         if $vcpus < $currentvcpus;
3579
3580     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3581     die "vcpus in running vm is different than configuration\n"
3582         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3583
3584     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3585         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3586     }
3587 }
3588
3589 sub qemu_memory_hotplug {
3590     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3591
3592     return $value if !check_running($vmid);
3593
3594     my $memory = $conf->{memory} || $defaults->{memory};
3595     $value = $defaults->{memory} if !$value;
3596     return $value if $value == $memory;
3597
3598     my $static_memory = $STATICMEM;
3599     my $dimm_memory = $memory - $static_memory;
3600
3601     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3602     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3603
3604
3605     my $sockets = 1;
3606     $sockets = $conf->{sockets} if $conf->{sockets};
3607
3608     if($value > $memory) {
3609
3610         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3611             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3612
3613                 return if $current_size <= $conf->{memory};
3614
3615                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3616                 if (my $err = $@) {
3617                     eval { qemu_objectdel($vmid, "mem-$name"); };
3618                     die $err;
3619                 }
3620
3621                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3622                 if (my $err = $@) {
3623                     eval { qemu_objectdel($vmid, "mem-$name"); };
3624                     die $err;
3625                 }
3626                 #update conf after each succesful module hotplug
3627                 $conf->{memory} = $current_size;
3628                 update_config_nolock($vmid, $conf, 1);
3629         });
3630
3631     } else {
3632
3633         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3634             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3635
3636                 return if $current_size >= $conf->{memory};
3637                 print "try to unplug memory dimm $name\n";
3638
3639                 my $retry = 0;
3640                 while (1) {
3641                     eval { qemu_devicedel($vmid, $name) };
3642                     sleep 3;
3643                     my $dimm_list = qemu_dimm_list($vmid);
3644                     last if !$dimm_list->{$name};
3645                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3646                     $retry++;
3647                 }
3648
3649                 #update conf after each succesful module unplug
3650                 $conf->{memory} = $current_size;
3651
3652                 eval { qemu_objectdel($vmid, "mem-$name"); };
3653                 update_config_nolock($vmid, $conf, 1);
3654         });
3655     }
3656 }
3657
3658 sub qemu_dimm_list {
3659     my ($vmid) = @_;
3660
3661     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3662     my $dimms = {};
3663
3664     foreach my $dimm (@$dimmarray) {
3665
3666         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3667         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3668         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3669         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3670         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3671     }
3672     return $dimms;
3673 }
3674
3675 sub qemu_block_set_io_throttle {
3676     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3677
3678     return if !check_running($vmid) ;
3679
3680     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3681
3682 }
3683
3684 # old code, only used to shutdown old VM after update
3685 sub __read_avail {
3686     my ($fh, $timeout) = @_;
3687
3688     my $sel = new IO::Select;
3689     $sel->add($fh);
3690
3691     my $res = '';
3692     my $buf;
3693
3694     my @ready;
3695     while (scalar (@ready = $sel->can_read($timeout))) {
3696         my $count;
3697         if ($count = $fh->sysread($buf, 8192)) {
3698             if ($buf =~ /^(.*)\(qemu\) $/s) {
3699                 $res .= $1;
3700                 last;
3701             } else {
3702                 $res .= $buf;
3703             }
3704         } else {
3705             if (!defined($count)) {
3706                 die "$!\n";
3707             }
3708             last;
3709         }
3710     }
3711
3712     die "monitor read timeout\n" if !scalar(@ready);
3713
3714     return $res;
3715 }
3716
3717 # old code, only used to shutdown old VM after update
3718 sub vm_monitor_command {
3719     my ($vmid, $cmdstr, $nocheck) = @_;
3720
3721     my $res;
3722
3723     eval {
3724         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3725
3726         my $sname = "${var_run_tmpdir}/$vmid.mon";
3727
3728         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3729             die "unable to connect to VM $vmid socket - $!\n";
3730
3731         my $timeout = 3;
3732
3733         # hack: migrate sometime blocks the monitor (when migrate_downtime
3734         # is set)
3735         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3736             $timeout = 60*60; # 1 hour
3737         }
3738
3739         # read banner;
3740         my $data = __read_avail($sock, $timeout);
3741
3742         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3743             die "got unexpected qemu monitor banner\n";
3744         }
3745
3746         my $sel = new IO::Select;
3747         $sel->add($sock);
3748
3749         if (!scalar(my @ready = $sel->can_write($timeout))) {
3750             die "monitor write error - timeout";
3751         }
3752
3753         my $fullcmd = "$cmdstr\r";
3754
3755         # syslog('info', "VM $vmid monitor command: $cmdstr");
3756
3757         my $b;
3758         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3759             die "monitor write error - $!";
3760         }
3761
3762         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3763
3764         $timeout = 20;
3765
3766         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3767             $timeout = 60*60; # 1 hour
3768         } elsif ($cmdstr =~ m/^(eject|change)/) {
3769             $timeout = 60; # note: cdrom mount command is slow
3770         }
3771         if ($res = __read_avail($sock, $timeout)) {
3772
3773             my @lines = split("\r?\n", $res);
3774
3775             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3776
3777             $res = join("\n", @lines);
3778             $res .= "\n";
3779         }
3780     };
3781
3782     my $err = $@;
3783
3784     if ($err) {
3785         syslog("err", "VM $vmid monitor command failed - $err");
3786         die $err;
3787     }
3788
3789     return $res;
3790 }
3791
3792 sub qemu_block_resize {
3793     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3794
3795     my $running = check_running($vmid);
3796
3797     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3798
3799     return if !$running;
3800
3801     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3802
3803 }
3804
3805 sub qemu_volume_snapshot {
3806     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3807
3808     my $running = check_running($vmid);
3809
3810     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3811         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3812     } else {
3813         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3814     }
3815 }
3816
3817 sub qemu_volume_snapshot_delete {
3818     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3819
3820     my $running = check_running($vmid);
3821
3822     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3823
3824     return if !$running;
3825
3826     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3827 }
3828
3829 sub set_migration_caps {
3830     my ($vmid) = @_;
3831
3832     my $cap_ref = [];
3833
3834     my $enabled_cap = {
3835         "auto-converge" => 1,
3836         "xbzrle" => 0,
3837         "x-rdma-pin-all" => 0,
3838         "zero-blocks" => 0,
3839     };
3840
3841     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3842
3843     for my $supported_capability (@$supported_capabilities) {
3844         push @$cap_ref, {
3845             capability => $supported_capability->{capability},
3846             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3847         };
3848     }
3849
3850     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3851 }
3852
3853 my $fast_plug_option = {
3854     'lock' => 1,
3855     'name' => 1,
3856     'onboot' => 1,
3857     'shares' => 1,
3858     'startup' => 1,
3859 };
3860
3861 # hotplug changes in [PENDING]
3862 # $selection hash can be used to only apply specified options, for
3863 # example: { cores => 1 } (only apply changed 'cores')
3864 # $errors ref is used to return error messages
3865 sub vmconfig_hotplug_pending {
3866     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3867
3868     my $defaults = load_defaults();
3869
3870     # commit values which do not have any impact on running VM first
3871     # Note: those option cannot raise errors, we we do not care about
3872     # $selection and always apply them.
3873
3874     my $add_error = sub {
3875         my ($opt, $msg) = @_;
3876         $errors->{$opt} = "hotplug problem - $msg";
3877     };
3878
3879     my $changes = 0;
3880     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3881         if ($fast_plug_option->{$opt}) {
3882             $conf->{$opt} = $conf->{pending}->{$opt};
3883             delete $conf->{pending}->{$opt};
3884             $changes = 1;
3885         }
3886     }
3887
3888     if ($changes) {
3889         update_config_nolock($vmid, $conf, 1);
3890         $conf = load_config($vmid); # update/reload
3891     }
3892
3893     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3894
3895     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3896     foreach my $opt (@delete) {
3897         next if $selection && !$selection->{$opt};
3898         eval {
3899             if ($opt eq 'hotplug') {
3900                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3901             } elsif ($opt eq 'tablet') {
3902                 die "skip\n" if !$hotplug_features->{usb};
3903                 if ($defaults->{tablet}) {
3904                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3905                 } else {
3906                     vm_deviceunplug($vmid, $conf, $opt);
3907                 }
3908             } elsif ($opt eq 'vcpus') {
3909                 die "skip\n" if !$hotplug_features->{cpu};
3910                 qemu_cpu_hotplug($vmid, $conf, undef);
3911             } elsif ($opt eq 'balloon') {
3912                 # enable balloon device is not hotpluggable
3913                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3914             } elsif ($fast_plug_option->{$opt}) {
3915                 # do nothing
3916             } elsif ($opt =~ m/^net(\d+)$/) {
3917                 die "skip\n" if !$hotplug_features->{network};
3918                 vm_deviceunplug($vmid, $conf, $opt);
3919             } elsif (valid_drivename($opt)) {
3920                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3921                 vm_deviceunplug($vmid, $conf, $opt);
3922                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3923             } elsif ($opt =~ m/^memory$/) {
3924                 die "skip\n" if !$hotplug_features->{memory};
3925                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3926             } elsif ($opt eq 'cpuunits') {
3927                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3928             } elsif ($opt eq 'cpulimit') {
3929                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3930             } else {
3931                 die "skip\n";
3932             }
3933         };
3934         if (my $err = $@) {
3935             &$add_error($opt, $err) if $err ne "skip\n";
3936         } else {
3937             # save new config if hotplug was successful
3938             delete $conf->{$opt};
3939             vmconfig_undelete_pending_option($conf, $opt);
3940             update_config_nolock($vmid, $conf, 1);
3941             $conf = load_config($vmid); # update/reload
3942         }
3943     }
3944
3945     foreach my $opt (keys %{$conf->{pending}}) {
3946         next if $selection && !$selection->{$opt};
3947         my $value = $conf->{pending}->{$opt};
3948         eval {
3949             if ($opt eq 'hotplug') {
3950                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3951             } elsif ($opt eq 'tablet') {
3952                 die "skip\n" if !$hotplug_features->{usb};
3953                 if ($value == 1) {
3954                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3955                 } elsif ($value == 0) {
3956                     vm_deviceunplug($vmid, $conf, $opt);
3957                 }
3958             } elsif ($opt eq 'vcpus') {
3959                 die "skip\n" if !$hotplug_features->{cpu};
3960                 qemu_cpu_hotplug($vmid, $conf, $value);
3961             } elsif ($opt eq 'balloon') {
3962                 # enable/disable balloning device is not hotpluggable
3963                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3964                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3965                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3966
3967                 # allow manual ballooning if shares is set to zero
3968                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3969                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3970                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3971                 }
3972             } elsif ($opt =~ m/^net(\d+)$/) {
3973                 # some changes can be done without hotplug
3974                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3975                                     $vmid, $opt, $value);
3976             } elsif (valid_drivename($opt)) {
3977                 # some changes can be done without hotplug
3978                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3979                                      $vmid, $opt, $value, 1);
3980             } elsif ($opt =~ m/^memory$/) { #dimms
3981                 die "skip\n" if !$hotplug_features->{memory};
3982                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3983             } elsif ($opt eq 'cpuunits') {
3984                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3985             } elsif ($opt eq 'cpulimit') {
3986                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3987                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3988             } else {
3989                 die "skip\n";  # skip non-hot-pluggable options
3990             }
3991         };
3992         if (my $err = $@) {
3993             &$add_error($opt, $err) if $err ne "skip\n";
3994         } else {
3995             # save new config if hotplug was successful
3996             $conf->{$opt} = $value;
3997             delete $conf->{pending}->{$opt};
3998             update_config_nolock($vmid, $conf, 1);
3999             $conf = load_config($vmid); # update/reload
4000         }
4001     }
4002 }
4003
4004 sub vmconfig_apply_pending {
4005     my ($vmid, $conf, $storecfg) = @_;
4006
4007     # cold plug
4008
4009     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
4010     foreach my $opt (@delete) { # delete
4011         die "internal error" if $opt =~ m/^unused/;
4012         $conf = load_config($vmid); # update/reload
4013         if (!defined($conf->{$opt})) {
4014             vmconfig_undelete_pending_option($conf, $opt);
4015             update_config_nolock($vmid, $conf, 1);
4016         } elsif (valid_drivename($opt)) {
4017             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
4018             vmconfig_undelete_pending_option($conf, $opt);
4019             delete $conf->{$opt};
4020             update_config_nolock($vmid, $conf, 1);
4021         } else {
4022             vmconfig_undelete_pending_option($conf, $opt);
4023             delete $conf->{$opt};
4024             update_config_nolock($vmid, $conf, 1);
4025         }
4026     }
4027
4028     $conf = load_config($vmid); # update/reload
4029
4030     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4031         $conf = load_config($vmid); # update/reload
4032
4033         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4034             # skip if nothing changed
4035         } elsif (valid_drivename($opt)) {
4036             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4037                 if defined($conf->{$opt});
4038             $conf->{$opt} = $conf->{pending}->{$opt};
4039         } else {
4040             $conf->{$opt} = $conf->{pending}->{$opt};
4041         }
4042
4043         delete $conf->{pending}->{$opt};
4044         update_config_nolock($vmid, $conf, 1);
4045     }
4046 }
4047
4048 my $safe_num_ne = sub {
4049     my ($a, $b) = @_;
4050
4051     return 0 if !defined($a) && !defined($b);
4052     return 1 if !defined($a);
4053     return 1 if !defined($b);
4054
4055     return $a != $b;
4056 };
4057
4058 my $safe_string_ne = sub {
4059     my ($a, $b) = @_;
4060
4061     return 0 if !defined($a) && !defined($b);
4062     return 1 if !defined($a);
4063     return 1 if !defined($b);
4064
4065     return $a ne $b;
4066 };
4067
4068 sub vmconfig_update_net {
4069     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4070
4071     my $newnet = parse_net($value);
4072
4073     if ($conf->{$opt}) {
4074         my $oldnet = parse_net($conf->{$opt});
4075
4076         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4077             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4078             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4079             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4080
4081             # for non online change, we try to hot-unplug
4082             die "skip\n" if !$hotplug;
4083             vm_deviceunplug($vmid, $conf, $opt);
4084         } else {
4085
4086             die "internal error" if $opt !~ m/net(\d+)/;
4087             my $iface = "tap${vmid}i$1";
4088
4089             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4090                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4091             }
4092
4093             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4094                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4095                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4096                 PVE::Network::tap_unplug($iface);
4097                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4098             }
4099
4100             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4101                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4102             }
4103
4104             return 1;
4105         }
4106     }
4107
4108     if ($hotplug) {
4109         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4110     } else {
4111         die "skip\n";
4112     }
4113 }
4114
4115 sub vmconfig_update_disk {
4116     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4117
4118     # fixme: do we need force?
4119
4120     my $drive = parse_drive($opt, $value);
4121
4122     if ($conf->{$opt}) {
4123
4124         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4125
4126             my $media = $drive->{media} || 'disk';
4127             my $oldmedia = $old_drive->{media} || 'disk';
4128             die "unable to change media type\n" if $media ne $oldmedia;
4129
4130             if (!drive_is_cdrom($old_drive)) {
4131
4132                 if ($drive->{file} ne $old_drive->{file}) {
4133
4134                     die "skip\n" if !$hotplug;
4135
4136                     # unplug and register as unused
4137                     vm_deviceunplug($vmid, $conf, $opt);
4138                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4139
4140                 } else {
4141                     # update existing disk
4142
4143                     # skip non hotpluggable value
4144                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4145                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4146                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4147                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4148                         die "skip\n";
4149                     }
4150
4151                     # apply throttle
4152                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4153                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4154                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4155                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4156                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4157                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4158                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4159                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4160                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4161                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4162                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4163                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4164
4165                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4166                                                    ($drive->{mbps} || 0)*1024*1024,
4167                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4168                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4169                                                    $drive->{iops} || 0,
4170                                                    $drive->{iops_rd} || 0,
4171                                                    $drive->{iops_wr} || 0,
4172                                                    ($drive->{mbps_max} || 0)*1024*1024,
4173                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4174                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4175                                                    $drive->{iops_max} || 0,
4176                                                    $drive->{iops_rd_max} || 0,
4177                                                    $drive->{iops_wr_max} || 0);
4178
4179                     }
4180
4181                     return 1;
4182                 }
4183
4184             } else { # cdrom
4185
4186                 if ($drive->{file} eq 'none') {
4187                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4188                 } else {
4189                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4190                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4191                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4192                 }
4193
4194                 return 1;
4195             }
4196         }
4197     }
4198
4199     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4200     # hotplug new disks
4201     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4202 }
4203
4204 sub vm_start {
4205     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4206
4207     lock_config($vmid, sub {
4208         my $conf = load_config($vmid, $migratedfrom);
4209
4210         die "you can't start a vm if it's a template\n" if is_template($conf);
4211
4212         check_lock($conf) if !$skiplock;
4213
4214         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4215
4216         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4217             vmconfig_apply_pending($vmid, $conf, $storecfg);
4218             $conf = load_config($vmid); # update/reload
4219         }
4220
4221         my $defaults = load_defaults();
4222
4223         # set environment variable useful inside network script
4224         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4225
4226         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4227
4228         my $migrate_port = 0;
4229         my $migrate_uri;
4230         if ($statefile) {
4231             if ($statefile eq 'tcp') {
4232                 my $localip = "localhost";
4233                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4234                 my $nodename = PVE::INotify::nodename();
4235                 if ($datacenterconf->{migration_unsecure}) {
4236                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4237                 }
4238                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4239                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4240                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4241                 push @$cmd, '-incoming', $migrate_uri;
4242                 push @$cmd, '-S';
4243             } else {
4244                 push @$cmd, '-loadstate', $statefile;
4245             }
4246         } elsif ($paused) {
4247             push @$cmd, '-S';
4248         }
4249
4250         # host pci devices
4251         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4252           my $d = parse_hostpci($conf->{"hostpci$i"});
4253           next if !$d;
4254           my $pcidevices = $d->{pciid};
4255           foreach my $pcidevice (@$pcidevices) {
4256                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4257
4258                 my $info = pci_device_info("0000:$pciid");
4259                 die "IOMMU not present\n" if !check_iommu_support();
4260                 die "no pci device info for device '$pciid'\n" if !$info;
4261
4262                 if ($d->{driver} && $d->{driver} eq "vfio") {
4263                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4264                 } else {
4265                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4266                 }
4267
4268                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4269           }
4270         }
4271
4272         PVE::Storage::activate_volumes($storecfg, $vollist);
4273
4274         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4275                     umask => 0077); };
4276         my $err = $@;
4277         die "start failed: $err" if $err;
4278
4279         print "migration listens on $migrate_uri\n" if $migrate_uri;
4280
4281         if ($statefile && $statefile ne 'tcp')  {
4282             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4283             warn $@ if $@;
4284         }
4285
4286         if ($migratedfrom) {
4287
4288             eval {
4289                 set_migration_caps($vmid);
4290             };
4291             warn $@ if $@;
4292
4293             if ($spice_port) {
4294                 print "spice listens on port $spice_port\n";
4295                 if ($spice_ticket) {
4296                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4297                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4298                 }
4299             }
4300
4301         } else {
4302
4303             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4304                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4305                     if $conf->{balloon};
4306             }
4307
4308             foreach my $opt (keys %$conf) {
4309                 next if $opt !~  m/^net\d+$/;
4310                 my $nicconf = parse_net($conf->{$opt});
4311                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4312             }
4313         }
4314
4315         vm_mon_cmd_nocheck($vmid, 'qom-set',
4316                     path => "machine/peripheral/balloon0",
4317                     property => "guest-stats-polling-interval",
4318                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4319
4320     });
4321 }
4322
4323 sub vm_mon_cmd {
4324     my ($vmid, $execute, %params) = @_;
4325
4326     my $cmd = { execute => $execute, arguments => \%params };
4327     vm_qmp_command($vmid, $cmd);
4328 }
4329
4330 sub vm_mon_cmd_nocheck {
4331     my ($vmid, $execute, %params) = @_;
4332
4333     my $cmd = { execute => $execute, arguments => \%params };
4334     vm_qmp_command($vmid, $cmd, 1);
4335 }
4336
4337 sub vm_qmp_command {
4338     my ($vmid, $cmd, $nocheck) = @_;
4339
4340     my $res;
4341
4342     my $timeout;
4343     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4344         $timeout = $cmd->{arguments}->{timeout};
4345         delete $cmd->{arguments}->{timeout};
4346     }
4347
4348     eval {
4349         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4350         my $sname = qmp_socket($vmid);
4351         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4352             my $qmpclient = PVE::QMPClient->new();
4353
4354             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4355         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4356             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4357                 if scalar(%{$cmd->{arguments}});
4358             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4359         } else {
4360             die "unable to open monitor socket\n";
4361         }
4362     };
4363     if (my $err = $@) {
4364         syslog("err", "VM $vmid qmp command failed - $err");
4365         die $err;
4366     }
4367
4368     return $res;
4369 }
4370
4371 sub vm_human_monitor_command {
4372     my ($vmid, $cmdline) = @_;
4373
4374     my $res;
4375
4376     my $cmd = {
4377         execute => 'human-monitor-command',
4378         arguments => { 'command-line' => $cmdline},
4379     };
4380
4381     return vm_qmp_command($vmid, $cmd);
4382 }
4383
4384 sub vm_commandline {
4385     my ($storecfg, $vmid) = @_;
4386
4387     my $conf = load_config($vmid);
4388
4389     my $defaults = load_defaults();
4390
4391     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4392
4393     return join(' ', @$cmd);
4394 }
4395
4396 sub vm_reset {
4397     my ($vmid, $skiplock) = @_;
4398
4399     lock_config($vmid, sub {
4400
4401         my $conf = load_config($vmid);
4402
4403         check_lock($conf) if !$skiplock;
4404
4405         vm_mon_cmd($vmid, "system_reset");
4406     });
4407 }
4408
4409 sub get_vm_volumes {
4410     my ($conf) = @_;
4411
4412     my $vollist = [];
4413     foreach_volid($conf, sub {
4414         my ($volid, $is_cdrom) = @_;
4415
4416         return if $volid =~ m|^/|;
4417
4418         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4419         return if !$sid;
4420
4421         push @$vollist, $volid;
4422     });
4423
4424     return $vollist;
4425 }
4426
4427 sub vm_stop_cleanup {
4428     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4429
4430     eval {
4431
4432         if (!$keepActive) {
4433             my $vollist = get_vm_volumes($conf);
4434             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4435         }
4436
4437         foreach my $ext (qw(mon qmp pid vnc qga)) {
4438             unlink "/var/run/qemu-server/${vmid}.$ext";
4439         }
4440
4441         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4442     };
4443     warn $@ if $@; # avoid errors - just warn
4444 }
4445
4446 # Note: use $nockeck to skip tests if VM configuration file exists.
4447 # We need that when migration VMs to other nodes (files already moved)
4448 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4449 sub vm_stop {
4450     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4451
4452     $force = 1 if !defined($force) && !$shutdown;
4453
4454     if ($migratedfrom){
4455         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4456         kill 15, $pid if $pid;
4457         my $conf = load_config($vmid, $migratedfrom);
4458         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4459         return;
4460     }
4461
4462     lock_config($vmid, sub {
4463
4464         my $pid = check_running($vmid, $nocheck);
4465         return if !$pid;
4466
4467         my $conf;
4468         if (!$nocheck) {
4469             $conf = load_config($vmid);
4470             check_lock($conf) if !$skiplock;
4471             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4472                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4473                 $timeout = $opts->{down} if $opts->{down};
4474             }
4475         }
4476
4477         $timeout = 60 if !defined($timeout);
4478
4479         eval {
4480             if ($shutdown) {
4481                 if (defined($conf) && $conf->{agent}) {
4482                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4483                 } else {
4484                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4485                 }
4486             } else {
4487                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4488             }
4489         };
4490         my $err = $@;
4491
4492         if (!$err) {
4493             my $count = 0;
4494             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4495                 $count++;
4496                 sleep 1;
4497             }
4498
4499             if ($count >= $timeout) {
4500                 if ($force) {
4501                     warn "VM still running - terminating now with SIGTERM\n";
4502                     kill 15, $pid;
4503                 } else {
4504                     die "VM quit/powerdown failed - got timeout\n";
4505                 }
4506             } else {
4507                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4508                 return;
4509             }
4510         } else {
4511             if ($force) {
4512                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4513                 kill 15, $pid;
4514             } else {
4515                 die "VM quit/powerdown failed\n";
4516             }
4517         }
4518
4519         # wait again
4520         $timeout = 10;
4521
4522         my $count = 0;
4523         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4524             $count++;
4525             sleep 1;
4526         }
4527
4528         if ($count >= $timeout) {
4529             warn "VM still running - terminating now with SIGKILL\n";
4530             kill 9, $pid;
4531             sleep 1;
4532         }
4533
4534         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4535    });
4536 }
4537
4538 sub vm_suspend {
4539     my ($vmid, $skiplock) = @_;
4540
4541     lock_config($vmid, sub {
4542
4543         my $conf = load_config($vmid);
4544
4545         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4546
4547         vm_mon_cmd($vmid, "stop");
4548     });
4549 }
4550
4551 sub vm_resume {
4552     my ($vmid, $skiplock) = @_;
4553
4554     lock_config($vmid, sub {
4555
4556         my $conf = load_config($vmid);
4557
4558         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4559
4560         vm_mon_cmd($vmid, "cont");
4561     });
4562 }
4563
4564 sub vm_sendkey {
4565     my ($vmid, $skiplock, $key) = @_;
4566
4567     lock_config($vmid, sub {
4568
4569         my $conf = load_config($vmid);
4570
4571         # there is no qmp command, so we use the human monitor command
4572         vm_human_monitor_command($vmid, "sendkey $key");
4573     });
4574 }
4575
4576 sub vm_destroy {
4577     my ($storecfg, $vmid, $skiplock) = @_;
4578
4579     lock_config($vmid, sub {
4580
4581         my $conf = load_config($vmid);
4582
4583         check_lock($conf) if !$skiplock;
4584
4585         if (!check_running($vmid)) {
4586             destroy_vm($storecfg, $vmid);
4587         } else {
4588             die "VM $vmid is running - destroy failed\n";
4589         }
4590     });
4591 }
4592
4593 # pci helpers
4594
4595 sub file_write {
4596     my ($filename, $buf) = @_;
4597
4598     my $fh = IO::File->new($filename, "w");
4599     return undef if !$fh;
4600
4601     my $res = print $fh $buf;
4602
4603     $fh->close();
4604
4605     return $res;
4606 }
4607
4608 sub pci_device_info {
4609     my ($name) = @_;
4610
4611     my $res;
4612
4613     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4614     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4615
4616     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4617     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4618
4619     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4620     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4621
4622     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4623     return undef if !defined($product) || $product !~ s/^0x//;
4624
4625     $res = {
4626         name => $name,
4627         vendor => $vendor,
4628         product => $product,
4629         domain => $domain,
4630         bus => $bus,
4631         slot => $slot,
4632         func => $func,
4633         irq => $irq,
4634         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4635     };
4636
4637     return $res;
4638 }
4639
4640 sub pci_dev_reset {
4641     my ($dev) = @_;
4642
4643     my $name = $dev->{name};
4644
4645     my $fn = "$pcisysfs/devices/$name/reset";
4646
4647     return file_write($fn, "1");
4648 }
4649
4650 sub pci_dev_bind_to_stub {
4651     my ($dev) = @_;
4652
4653     my $name = $dev->{name};
4654
4655     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4656     return 1 if -d $testdir;
4657
4658     my $data = "$dev->{vendor} $dev->{product}";
4659     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4660
4661     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4662     if (!file_write($fn, $name)) {
4663         return undef if -f $fn;
4664     }
4665
4666     $fn = "$pcisysfs/drivers/pci-stub/bind";
4667     if (! -d $testdir) {
4668         return undef if !file_write($fn, $name);
4669     }
4670
4671     return -d $testdir;
4672 }
4673
4674 sub pci_dev_bind_to_vfio {
4675     my ($dev) = @_;
4676
4677     my $name = $dev->{name};
4678
4679     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4680
4681     if (!-d $vfio_basedir) {
4682         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4683     }
4684     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4685
4686     my $testdir = "$vfio_basedir/$name";
4687     return 1 if -d $testdir;
4688
4689     my $data = "$dev->{vendor} $dev->{product}";
4690     return undef if !file_write("$vfio_basedir/new_id", $data);
4691
4692     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4693     if (!file_write($fn, $name)) {
4694         return undef if -f $fn;
4695     }
4696
4697     $fn = "$vfio_basedir/bind";
4698     if (! -d $testdir) {
4699         return undef if !file_write($fn, $name);
4700     }
4701
4702     return -d $testdir;
4703 }
4704
4705 sub pci_dev_group_bind_to_vfio {
4706     my ($pciid) = @_;
4707
4708     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4709
4710     if (!-d $vfio_basedir) {
4711         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4712     }
4713     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4714
4715     # get IOMMU group devices
4716     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4717       my @devs = grep /^0000:/, readdir($D);
4718     closedir($D);
4719
4720     foreach my $pciid (@devs) {
4721         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4722
4723         # pci bridges, switches or root ports are not supported
4724         # they have a pci_bus subdirectory so skip them
4725         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4726
4727         my $info = pci_device_info($1);
4728         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4729     }
4730
4731     return 1;
4732 }
4733
4734 sub print_pci_addr {
4735     my ($id, $bridges) = @_;
4736
4737     my $res = '';
4738     my $devices = {
4739         piix3 => { bus => 0, addr => 1 },
4740         #addr2 : first videocard
4741         balloon0 => { bus => 0, addr => 3 },
4742         watchdog => { bus => 0, addr => 4 },
4743         scsihw0 => { bus => 0, addr => 5 },
4744         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4745         scsihw1 => { bus => 0, addr => 6 },
4746         ahci0 => { bus => 0, addr => 7 },
4747         qga0 => { bus => 0, addr => 8 },
4748         spice => { bus => 0, addr => 9 },
4749         virtio0 => { bus => 0, addr => 10 },
4750         virtio1 => { bus => 0, addr => 11 },
4751         virtio2 => { bus => 0, addr => 12 },
4752         virtio3 => { bus => 0, addr => 13 },
4753         virtio4 => { bus => 0, addr => 14 },
4754         virtio5 => { bus => 0, addr => 15 },
4755         hostpci0 => { bus => 0, addr => 16 },
4756         hostpci1 => { bus => 0, addr => 17 },
4757         net0 => { bus => 0, addr => 18 },
4758         net1 => { bus => 0, addr => 19 },
4759         net2 => { bus => 0, addr => 20 },
4760         net3 => { bus => 0, addr => 21 },
4761         net4 => { bus => 0, addr => 22 },
4762         net5 => { bus => 0, addr => 23 },
4763         vga1 => { bus => 0, addr => 24 },
4764         vga2 => { bus => 0, addr => 25 },
4765         vga3 => { bus => 0, addr => 26 },
4766         hostpci2 => { bus => 0, addr => 27 },
4767         hostpci3 => { bus => 0, addr => 28 },
4768         #addr29 : usb-host (pve-usb.cfg)
4769         'pci.1' => { bus => 0, addr => 30 },
4770         'pci.2' => { bus => 0, addr => 31 },
4771         'net6' => { bus => 1, addr => 1 },
4772         'net7' => { bus => 1, addr => 2 },
4773         'net8' => { bus => 1, addr => 3 },
4774         'net9' => { bus => 1, addr => 4 },
4775         'net10' => { bus => 1, addr => 5 },
4776         'net11' => { bus => 1, addr => 6 },
4777         'net12' => { bus => 1, addr => 7 },
4778         'net13' => { bus => 1, addr => 8 },
4779         'net14' => { bus => 1, addr => 9 },
4780         'net15' => { bus => 1, addr => 10 },
4781         'net16' => { bus => 1, addr => 11 },
4782         'net17' => { bus => 1, addr => 12 },
4783         'net18' => { bus => 1, addr => 13 },
4784         'net19' => { bus => 1, addr => 14 },
4785         'net20' => { bus => 1, addr => 15 },
4786         'net21' => { bus => 1, addr => 16 },
4787         'net22' => { bus => 1, addr => 17 },
4788         'net23' => { bus => 1, addr => 18 },
4789         'net24' => { bus => 1, addr => 19 },
4790         'net25' => { bus => 1, addr => 20 },
4791         'net26' => { bus => 1, addr => 21 },
4792         'net27' => { bus => 1, addr => 22 },
4793         'net28' => { bus => 1, addr => 23 },
4794         'net29' => { bus => 1, addr => 24 },
4795         'net30' => { bus => 1, addr => 25 },
4796         'net31' => { bus => 1, addr => 26 },
4797         'virtio6' => { bus => 2, addr => 1 },
4798         'virtio7' => { bus => 2, addr => 2 },
4799         'virtio8' => { bus => 2, addr => 3 },
4800         'virtio9' => { bus => 2, addr => 4 },
4801         'virtio10' => { bus => 2, addr => 5 },
4802         'virtio11' => { bus => 2, addr => 6 },
4803         'virtio12' => { bus => 2, addr => 7 },
4804         'virtio13' => { bus => 2, addr => 8 },
4805         'virtio14' => { bus => 2, addr => 9 },
4806         'virtio15' => { bus => 2, addr => 10 },
4807         'virtioscsi0' => { bus => 3, addr => 1 },
4808         'virtioscsi1' => { bus => 3, addr => 2 },
4809         'virtioscsi2' => { bus => 3, addr => 3 },
4810         'virtioscsi3' => { bus => 3, addr => 4 },
4811         'virtioscsi4' => { bus => 3, addr => 5 },
4812         'virtioscsi5' => { bus => 3, addr => 6 },
4813         'virtioscsi6' => { bus => 3, addr => 7 },
4814         'virtioscsi7' => { bus => 3, addr => 8 },
4815         'virtioscsi8' => { bus => 3, addr => 9 },
4816         'virtioscsi9' => { bus => 3, addr => 10 },
4817         'virtioscsi10' => { bus => 3, addr => 11 },
4818         'virtioscsi11' => { bus => 3, addr => 12 },
4819         'virtioscsi12' => { bus => 3, addr => 13 },
4820         'virtioscsi13' => { bus => 3, addr => 14 },
4821         'virtioscsi14' => { bus => 3, addr => 15 },
4822         'virtioscsi15' => { bus => 3, addr => 16 },
4823         'virtioscsi16' => { bus => 3, addr => 17 },
4824         'virtioscsi17' => { bus => 3, addr => 18 },
4825         'virtioscsi18' => { bus => 3, addr => 19 },
4826         'virtioscsi19' => { bus => 3, addr => 20 },
4827         'virtioscsi20' => { bus => 3, addr => 21 },
4828         'virtioscsi21' => { bus => 3, addr => 22 },
4829         'virtioscsi22' => { bus => 3, addr => 23 },
4830         'virtioscsi23' => { bus => 3, addr => 24 },
4831         'virtioscsi24' => { bus => 3, addr => 25 },
4832         'virtioscsi25' => { bus => 3, addr => 26 },
4833         'virtioscsi26' => { bus => 3, addr => 27 },
4834         'virtioscsi27' => { bus => 3, addr => 28 },
4835         'virtioscsi28' => { bus => 3, addr => 29 },
4836         'virtioscsi29' => { bus => 3, addr => 30 },
4837         'virtioscsi30' => { bus => 3, addr => 31 },
4838
4839     };
4840
4841     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4842            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4843            my $bus = $devices->{$id}->{bus};
4844            $res = ",bus=pci.$bus,addr=$addr";
4845            $bridges->{$bus} = 1 if $bridges;
4846     }
4847     return $res;
4848
4849 }
4850
4851 sub print_pcie_addr {
4852     my ($id) = @_;
4853
4854     my $res = '';
4855     my $devices = {
4856         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4857         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4858         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4859         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4860     };
4861
4862     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4863            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4864            my $bus = $devices->{$id}->{bus};
4865            $res = ",bus=$bus,addr=$addr";
4866     }
4867     return $res;
4868
4869 }
4870
4871 # vzdump restore implementaion
4872
4873 sub tar_archive_read_firstfile {
4874     my $archive = shift;
4875
4876     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4877
4878     # try to detect archive type first
4879     my $pid = open (TMP, "tar tf '$archive'|") ||
4880         die "unable to open file '$archive'\n";
4881     my $firstfile = <TMP>;
4882     kill 15, $pid;
4883     close TMP;
4884
4885     die "ERROR: archive contaions no data\n" if !$firstfile;
4886     chomp $firstfile;
4887
4888     return $firstfile;
4889 }
4890
4891 sub tar_restore_cleanup {
4892     my ($storecfg, $statfile) = @_;
4893
4894     print STDERR "starting cleanup\n";
4895
4896     if (my $fd = IO::File->new($statfile, "r")) {
4897         while (defined(my $line = <$fd>)) {
4898             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4899                 my $volid = $2;
4900                 eval {
4901                     if ($volid =~ m|^/|) {
4902                         unlink $volid || die 'unlink failed\n';
4903                     } else {
4904                         PVE::Storage::vdisk_free($storecfg, $volid);
4905                     }
4906                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4907                 };
4908                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4909             } else {
4910                 print STDERR "unable to parse line in statfile - $line";
4911             }
4912         }
4913         $fd->close();
4914     }
4915 }
4916
4917 sub restore_archive {
4918     my ($archive, $vmid, $user, $opts) = @_;
4919
4920     my $format = $opts->{format};
4921     my $comp;
4922
4923     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4924         $format = 'tar' if !$format;
4925         $comp = 'gzip';
4926     } elsif ($archive =~ m/\.tar$/) {
4927         $format = 'tar' if !$format;
4928     } elsif ($archive =~ m/.tar.lzo$/) {
4929         $format = 'tar' if !$format;
4930         $comp = 'lzop';
4931     } elsif ($archive =~ m/\.vma$/) {
4932         $format = 'vma' if !$format;
4933     } elsif ($archive =~ m/\.vma\.gz$/) {
4934         $format = 'vma' if !$format;
4935         $comp = 'gzip';
4936     } elsif ($archive =~ m/\.vma\.lzo$/) {
4937         $format = 'vma' if !$format;
4938         $comp = 'lzop';
4939     } else {
4940         $format = 'vma' if !$format; # default
4941     }
4942
4943     # try to detect archive format
4944     if ($format eq 'tar') {
4945         return restore_tar_archive($archive, $vmid, $user, $opts);
4946     } else {
4947         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4948     }
4949 }
4950
4951 sub restore_update_config_line {
4952     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4953
4954     return if $line =~ m/^\#qmdump\#/;
4955     return if $line =~ m/^\#vzdump\#/;
4956     return if $line =~ m/^lock:/;
4957     return if $line =~ m/^unused\d+:/;
4958     return if $line =~ m/^parent:/;
4959     return if $line =~ m/^template:/; # restored VM is never a template
4960
4961     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4962         # try to convert old 1.X settings
4963         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4964         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4965             my ($model, $macaddr) = split(/\=/, $devconfig);
4966             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4967             my $net = {
4968                 model => $model,
4969                 bridge => "vmbr$ind",
4970                 macaddr => $macaddr,
4971             };
4972             my $netstr = print_net($net);
4973
4974             print $outfd "net$cookie->{netcount}: $netstr\n";
4975             $cookie->{netcount}++;
4976         }
4977     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4978         my ($id, $netstr) = ($1, $2);
4979         my $net = parse_net($netstr);
4980         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4981         $netstr = print_net($net);
4982         print $outfd "$id: $netstr\n";
4983     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4984         my $virtdev = $1;
4985         my $value = $3;
4986         if ($line =~ m/backup=no/) {
4987             print $outfd "#$line";
4988         } elsif ($virtdev && $map->{$virtdev}) {
4989             my $di = parse_drive($virtdev, $value);
4990             delete $di->{format}; # format can change on restore
4991             $di->{file} = $map->{$virtdev};
4992             $value = print_drive($vmid, $di);
4993             print $outfd "$virtdev: $value\n";
4994         } else {
4995             print $outfd $line;
4996         }
4997     } else {
4998         print $outfd $line;
4999     }
5000 }
5001
5002 sub scan_volids {
5003     my ($cfg, $vmid) = @_;
5004
5005     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5006
5007     my $volid_hash = {};
5008     foreach my $storeid (keys %$info) {
5009         foreach my $item (@{$info->{$storeid}}) {
5010             next if !($item->{volid} && $item->{size});
5011             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5012             $volid_hash->{$item->{volid}} = $item;
5013         }
5014     }
5015
5016     return $volid_hash;
5017 }
5018
5019 sub get_used_paths {
5020     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5021
5022     my $used_path = {};
5023
5024     my $scan_config = sub {
5025         my ($cref, $snapname) = @_;
5026
5027         foreach my $key (keys %$cref) {
5028             my $value = $cref->{$key};
5029             if (valid_drivename($key)) {
5030                 next if $skip_drive && $key eq $skip_drive;
5031                 my $drive = parse_drive($key, $value);
5032                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5033                 if ($drive->{file} =~ m!^/!) {
5034                     $used_path->{$drive->{file}}++; # = 1;
5035                 } else {
5036                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5037                     next if !$storeid;
5038                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5039                     next if !$scfg;
5040                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5041                     $used_path->{$path}++; # = 1;
5042                 }
5043             }
5044         }
5045     };
5046
5047     &$scan_config($conf);
5048
5049     undef $skip_drive;
5050
5051     if ($scan_snapshots) {
5052         foreach my $snapname (keys %{$conf->{snapshots}}) {
5053             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5054         }
5055     }
5056
5057     return $used_path;
5058 }
5059
5060 sub update_disksize {
5061     my ($vmid, $conf, $volid_hash) = @_;
5062
5063     my $changes;
5064
5065     my $used = {};
5066
5067     # Note: it is allowed to define multiple storages with same path (alias), so
5068     # we need to check both 'volid' and real 'path' (two different volid can point
5069     # to the same path).
5070
5071     my $usedpath = {};
5072
5073     # update size info
5074     foreach my $opt (keys %$conf) {
5075         if (valid_drivename($opt)) {
5076             my $drive = parse_drive($opt, $conf->{$opt});
5077             my $volid = $drive->{file};
5078             next if !$volid;
5079
5080             $used->{$volid} = 1;
5081             if ($volid_hash->{$volid} &&
5082                 (my $path = $volid_hash->{$volid}->{path})) {
5083                 $usedpath->{$path} = 1;
5084             }
5085
5086             next if drive_is_cdrom($drive);
5087             next if !$volid_hash->{$volid};
5088
5089             $drive->{size} = $volid_hash->{$volid}->{size};
5090             my $new = print_drive($vmid, $drive);
5091             if ($new ne $conf->{$opt}) {
5092                 $changes = 1;
5093                 $conf->{$opt} = $new;
5094             }
5095         }
5096     }
5097
5098     # remove 'unusedX' entry if volume is used
5099     foreach my $opt (keys %$conf) {
5100         next if $opt !~ m/^unused\d+$/;
5101         my $volid = $conf->{$opt};
5102         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5103         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5104             $changes = 1;
5105             delete $conf->{$opt};
5106         }
5107     }
5108
5109     foreach my $volid (sort keys %$volid_hash) {
5110         next if $volid =~ m/vm-$vmid-state-/;
5111         next if $used->{$volid};
5112         my $path = $volid_hash->{$volid}->{path};
5113         next if !$path; # just to be sure
5114         next if $usedpath->{$path};
5115         $changes = 1;
5116         add_unused_volume($conf, $volid);
5117         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5118     }
5119
5120     return $changes;
5121 }
5122
5123 sub rescan {
5124     my ($vmid, $nolock) = @_;
5125
5126     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5127
5128     my $volid_hash = scan_volids($cfg, $vmid);
5129
5130     my $updatefn =  sub {
5131         my ($vmid) = @_;
5132
5133         my $conf = load_config($vmid);
5134
5135         check_lock($conf);
5136
5137         my $vm_volids = {};
5138         foreach my $volid (keys %$volid_hash) {
5139             my $info = $volid_hash->{$volid};
5140             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5141         }
5142
5143         my $changes = update_disksize($vmid, $conf, $vm_volids);
5144
5145         update_config_nolock($vmid, $conf, 1) if $changes;
5146     };
5147
5148     if (defined($vmid)) {
5149         if ($nolock) {
5150             &$updatefn($vmid);
5151         } else {
5152             lock_config($vmid, $updatefn, $vmid);
5153         }
5154     } else {
5155         my $vmlist = config_list();
5156         foreach my $vmid (keys %$vmlist) {
5157             if ($nolock) {
5158                 &$updatefn($vmid);
5159             } else {
5160                 lock_config($vmid, $updatefn, $vmid);
5161             }
5162         }
5163     }
5164 }
5165
5166 sub restore_vma_archive {
5167     my ($archive, $vmid, $user, $opts, $comp) = @_;
5168
5169     my $input = $archive eq '-' ? "<&STDIN" : undef;
5170     my $readfrom = $archive;
5171
5172     my $uncomp = '';
5173     if ($comp) {
5174         $readfrom = '-';
5175         my $qarchive = PVE::Tools::shellquote($archive);
5176         if ($comp eq 'gzip') {
5177             $uncomp = "zcat $qarchive|";
5178         } elsif ($comp eq 'lzop') {
5179             $uncomp = "lzop -d -c $qarchive|";
5180         } else {
5181             die "unknown compression method '$comp'\n";
5182         }
5183
5184     }
5185
5186     my $tmpdir = "/var/tmp/vzdumptmp$$";
5187     rmtree $tmpdir;
5188
5189     # disable interrupts (always do cleanups)
5190     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5191         warn "got interrupt - ignored\n";
5192     };
5193
5194     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5195     POSIX::mkfifo($mapfifo, 0600);
5196     my $fifofh;
5197
5198     my $openfifo = sub {
5199         open($fifofh, '>', $mapfifo) || die $!;
5200     };
5201
5202     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5203
5204     my $oldtimeout;
5205     my $timeout = 5;
5206
5207     my $devinfo = {};
5208
5209     my $rpcenv = PVE::RPCEnvironment::get();
5210
5211     my $conffile = config_file($vmid);
5212     my $tmpfn = "$conffile.$$.tmp";
5213
5214     # Note: $oldconf is undef if VM does not exists
5215     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5216
5217     my $print_devmap = sub {
5218         my $virtdev_hash = {};
5219
5220         my $cfgfn = "$tmpdir/qemu-server.conf";
5221
5222         # we can read the config - that is already extracted
5223         my $fh = IO::File->new($cfgfn, "r") ||
5224             "unable to read qemu-server.conf - $!\n";
5225
5226         while (defined(my $line = <$fh>)) {
5227             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5228                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5229                 die "archive does not contain data for drive '$virtdev'\n"
5230                     if !$devinfo->{$devname};
5231                 if (defined($opts->{storage})) {
5232                     $storeid = $opts->{storage} || 'local';
5233                 } elsif (!$storeid) {
5234                     $storeid = 'local';
5235                 }
5236                 $format = 'raw' if !$format;
5237                 $devinfo->{$devname}->{devname} = $devname;
5238                 $devinfo->{$devname}->{virtdev} = $virtdev;
5239                 $devinfo->{$devname}->{format} = $format;
5240                 $devinfo->{$devname}->{storeid} = $storeid;
5241
5242                 # check permission on storage
5243                 my $pool = $opts->{pool}; # todo: do we need that?
5244                 if ($user ne 'root@pam') {
5245                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5246                 }
5247
5248                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5249             }
5250         }
5251
5252         foreach my $devname (keys %$devinfo) {
5253             die "found no device mapping information for device '$devname'\n"
5254                 if !$devinfo->{$devname}->{virtdev};
5255         }
5256
5257         my $cfg = cfs_read_file('storage.cfg');
5258
5259         # create empty/temp config
5260         if ($oldconf) {
5261             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5262             foreach_drive($oldconf, sub {
5263                 my ($ds, $drive) = @_;
5264
5265                 return if drive_is_cdrom($drive);
5266
5267                 my $volid = $drive->{file};
5268
5269                 return if !$volid || $volid =~ m|^/|;
5270
5271                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5272                 return if !$path || !$owner || ($owner != $vmid);
5273
5274                 # Note: only delete disk we want to restore
5275                 # other volumes will become unused
5276                 if ($virtdev_hash->{$ds}) {
5277                     PVE::Storage::vdisk_free($cfg, $volid);
5278                 }
5279             });
5280         }
5281
5282         my $map = {};
5283         foreach my $virtdev (sort keys %$virtdev_hash) {
5284             my $d = $virtdev_hash->{$virtdev};
5285             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5286             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5287
5288             # test if requested format is supported
5289             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5290             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5291             $d->{format} = $defFormat if !$supported;
5292
5293             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5294                                                   $d->{format}, undef, $alloc_size);
5295             print STDERR "new volume ID is '$volid'\n";
5296             $d->{volid} = $volid;
5297             my $path = PVE::Storage::path($cfg, $volid);
5298
5299             my $write_zeros = 1;
5300             # fixme: what other storages types initialize volumes with zero?
5301             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5302                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5303                 $write_zeros = 0;
5304             }
5305
5306             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5307
5308             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5309             $map->{$virtdev} = $volid;
5310         }
5311
5312         $fh->seek(0, 0) || die "seek failed - $!\n";
5313
5314         my $outfd = new IO::File ($tmpfn, "w") ||
5315             die "unable to write config for VM $vmid\n";
5316
5317         my $cookie = { netcount => 0 };
5318         while (defined(my $line = <$fh>)) {
5319             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5320         }
5321
5322         $fh->close();
5323         $outfd->close();
5324     };
5325
5326     eval {
5327         # enable interrupts
5328         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5329             die "interrupted by signal\n";
5330         };
5331         local $SIG{ALRM} = sub { die "got timeout\n"; };
5332
5333         $oldtimeout = alarm($timeout);
5334
5335         my $parser = sub {
5336             my $line = shift;
5337
5338             print "$line\n";
5339
5340             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5341                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5342                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5343             } elsif ($line =~ m/^CTIME: /) {
5344                 # we correctly received the vma config, so we can disable
5345                 # the timeout now for disk allocation (set to 10 minutes, so
5346                 # that we always timeout if something goes wrong)
5347                 alarm(600);
5348                 &$print_devmap();
5349                 print $fifofh "done\n";
5350                 my $tmp = $oldtimeout || 0;
5351                 $oldtimeout = undef;
5352                 alarm($tmp);
5353                 close($fifofh);
5354             }
5355         };
5356
5357         print "restore vma archive: $cmd\n";
5358         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5359     };
5360     my $err = $@;
5361
5362     alarm($oldtimeout) if $oldtimeout;
5363
5364     unlink $mapfifo;
5365
5366     if ($err) {
5367         rmtree $tmpdir;
5368         unlink $tmpfn;
5369
5370         my $cfg = cfs_read_file('storage.cfg');
5371         foreach my $devname (keys %$devinfo) {
5372             my $volid = $devinfo->{$devname}->{volid};
5373             next if !$volid;
5374             eval {
5375                 if ($volid =~ m|^/|) {
5376                     unlink $volid || die 'unlink failed\n';
5377                 } else {
5378                     PVE::Storage::vdisk_free($cfg, $volid);
5379                 }
5380                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5381             };
5382             print STDERR "unable to cleanup '$volid' - $@" if $@;
5383         }
5384         die $err;
5385     }
5386
5387     rmtree $tmpdir;
5388
5389     rename($tmpfn, $conffile) ||
5390         die "unable to commit configuration file '$conffile'\n";
5391
5392     PVE::Cluster::cfs_update(); # make sure we read new file
5393
5394     eval { rescan($vmid, 1); };
5395     warn $@ if $@;
5396 }
5397
5398 sub restore_tar_archive {
5399     my ($archive, $vmid, $user, $opts) = @_;
5400
5401     if ($archive ne '-') {
5402         my $firstfile = tar_archive_read_firstfile($archive);
5403         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5404             if $firstfile ne 'qemu-server.conf';
5405     }
5406
5407     my $storecfg = cfs_read_file('storage.cfg');
5408
5409     # destroy existing data - keep empty config
5410     my $vmcfgfn = config_file($vmid);
5411     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5412
5413     my $tocmd = "/usr/lib/qemu-server/qmextract";
5414
5415     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5416     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5417     $tocmd .= ' --prealloc' if $opts->{prealloc};
5418     $tocmd .= ' --info' if $opts->{info};
5419
5420     # tar option "xf" does not autodetect compression when read from STDIN,
5421     # so we pipe to zcat
5422     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5423         PVE::Tools::shellquote("--to-command=$tocmd");
5424
5425     my $tmpdir = "/var/tmp/vzdumptmp$$";
5426     mkpath $tmpdir;
5427
5428     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5429     local $ENV{VZDUMP_VMID} = $vmid;
5430     local $ENV{VZDUMP_USER} = $user;
5431
5432     my $conffile = config_file($vmid);
5433     my $tmpfn = "$conffile.$$.tmp";
5434
5435     # disable interrupts (always do cleanups)
5436     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5437         print STDERR "got interrupt - ignored\n";
5438     };
5439
5440     eval {
5441         # enable interrupts
5442         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5443             die "interrupted by signal\n";
5444         };
5445
5446         if ($archive eq '-') {
5447             print "extracting archive from STDIN\n";
5448             run_command($cmd, input => "<&STDIN");
5449         } else {
5450             print "extracting archive '$archive'\n";
5451             run_command($cmd);
5452         }
5453
5454         return if $opts->{info};
5455
5456         # read new mapping
5457         my $map = {};
5458         my $statfile = "$tmpdir/qmrestore.stat";
5459         if (my $fd = IO::File->new($statfile, "r")) {
5460             while (defined (my $line = <$fd>)) {
5461                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5462                     $map->{$1} = $2 if $1;
5463                 } else {
5464                     print STDERR "unable to parse line in statfile - $line\n";
5465                 }
5466             }
5467             $fd->close();
5468         }
5469
5470         my $confsrc = "$tmpdir/qemu-server.conf";
5471
5472         my $srcfd = new IO::File($confsrc, "r") ||
5473             die "unable to open file '$confsrc'\n";
5474
5475         my $outfd = new IO::File ($tmpfn, "w") ||
5476             die "unable to write config for VM $vmid\n";
5477
5478         my $cookie = { netcount => 0 };
5479         while (defined (my $line = <$srcfd>)) {
5480             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5481         }
5482
5483         $srcfd->close();
5484         $outfd->close();
5485     };
5486     my $err = $@;
5487
5488     if ($err) {
5489
5490         unlink $tmpfn;
5491
5492         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5493
5494         die $err;
5495     }
5496
5497     rmtree $tmpdir;
5498
5499     rename $tmpfn, $conffile ||
5500         die "unable to commit configuration file '$conffile'\n";
5501
5502     PVE::Cluster::cfs_update(); # make sure we read new file
5503
5504     eval { rescan($vmid, 1); };
5505     warn $@ if $@;
5506 };
5507
5508
5509 # Internal snapshots
5510
5511 # NOTE: Snapshot create/delete involves several non-atomic
5512 # action, and can take a long time.
5513 # So we try to avoid locking the file and use 'lock' variable
5514 # inside the config file instead.
5515
5516 my $snapshot_copy_config = sub {
5517     my ($source, $dest) = @_;
5518
5519     foreach my $k (keys %$source) {
5520         next if $k eq 'snapshots';
5521         next if $k eq 'snapstate';
5522         next if $k eq 'snaptime';
5523         next if $k eq 'vmstate';
5524         next if $k eq 'lock';
5525         next if $k eq 'digest';
5526         next if $k eq 'description';
5527         next if $k =~ m/^unused\d+$/;
5528
5529         $dest->{$k} = $source->{$k};
5530     }
5531 };
5532
5533 my $snapshot_apply_config = sub {
5534     my ($conf, $snap) = @_;
5535
5536     # copy snapshot list
5537     my $newconf = {
5538         snapshots => $conf->{snapshots},
5539     };
5540
5541     # keep description and list of unused disks
5542     foreach my $k (keys %$conf) {
5543         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5544         $newconf->{$k} = $conf->{$k};
5545     }
5546
5547     &$snapshot_copy_config($snap, $newconf);
5548
5549     return $newconf;
5550 };
5551
5552 sub foreach_writable_storage {
5553     my ($conf, $func) = @_;
5554
5555     my $sidhash = {};
5556
5557     foreach my $ds (keys %$conf) {
5558         next if !valid_drivename($ds);
5559
5560         my $drive = parse_drive($ds, $conf->{$ds});
5561         next if !$drive;
5562         next if drive_is_cdrom($drive);
5563
5564         my $volid = $drive->{file};
5565
5566         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5567         $sidhash->{$sid} = $sid if $sid;
5568     }
5569
5570     foreach my $sid (sort keys %$sidhash) {
5571         &$func($sid);
5572     }
5573 }
5574
5575 my $alloc_vmstate_volid = sub {
5576     my ($storecfg, $vmid, $conf, $snapname) = @_;
5577
5578     # Note: we try to be smart when selecting a $target storage
5579
5580     my $target;
5581
5582     # search shared storage first
5583     foreach_writable_storage($conf, sub {
5584         my ($sid) = @_;
5585         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5586         return if !$scfg->{shared};
5587
5588         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5589     });
5590
5591     if (!$target) {
5592         # now search local storage
5593         foreach_writable_storage($conf, sub {
5594             my ($sid) = @_;
5595             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5596             return if $scfg->{shared};
5597
5598             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5599         });
5600     }
5601
5602     $target = 'local' if !$target;
5603
5604     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5605     # we abort live save after $conf->{memory}, so we need at max twice that space
5606     my $size = $conf->{memory}*2 + $driver_state_size;
5607
5608     my $name = "vm-$vmid-state-$snapname";
5609     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5610     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5611     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5612
5613     return $volid;
5614 };
5615
5616 my $snapshot_prepare = sub {
5617     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5618
5619     my $snap;
5620
5621     my $updatefn =  sub {
5622
5623         my $conf = load_config($vmid);
5624
5625         die "you can't take a snapshot if it's a template\n"
5626             if is_template($conf);
5627
5628         check_lock($conf);
5629
5630         $conf->{lock} = 'snapshot';
5631
5632         die "snapshot name '$snapname' already used\n"
5633             if defined($conf->{snapshots}->{$snapname});
5634
5635         my $storecfg = PVE::Storage::config();
5636         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5637
5638         $snap = $conf->{snapshots}->{$snapname} = {};
5639
5640         if ($save_vmstate && check_running($vmid)) {
5641             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5642         }
5643
5644         &$snapshot_copy_config($conf, $snap);
5645
5646         $snap->{snapstate} = "prepare";
5647         $snap->{snaptime} = time();
5648         $snap->{description} = $comment if $comment;
5649
5650         # always overwrite machine if we save vmstate. This makes sure we
5651         # can restore it later using correct machine type
5652         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5653
5654         update_config_nolock($vmid, $conf, 1);
5655     };
5656
5657     lock_config($vmid, $updatefn);
5658
5659     return $snap;
5660 };
5661
5662 my $snapshot_commit = sub {
5663     my ($vmid, $snapname) = @_;
5664
5665     my $updatefn = sub {
5666
5667         my $conf = load_config($vmid);
5668
5669         die "missing snapshot lock\n"
5670             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5671
5672         my $has_machine_config = defined($conf->{machine});
5673
5674         my $snap = $conf->{snapshots}->{$snapname};
5675
5676         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5677
5678         die "wrong snapshot state\n"
5679             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5680
5681         delete $snap->{snapstate};
5682         delete $conf->{lock};
5683
5684         my $newconf = &$snapshot_apply_config($conf, $snap);
5685
5686         delete $newconf->{machine} if !$has_machine_config;
5687
5688         $newconf->{parent} = $snapname;
5689
5690         update_config_nolock($vmid, $newconf, 1);
5691     };
5692
5693     lock_config($vmid, $updatefn);
5694 };
5695
5696 sub snapshot_rollback {
5697     my ($vmid, $snapname) = @_;
5698
5699     my $prepare = 1;
5700
5701     my $storecfg = PVE::Storage::config();
5702
5703     my $conf = load_config($vmid);
5704
5705     my $get_snapshot_config = sub {
5706
5707         die "you can't rollback if vm is a template\n" if is_template($conf);
5708
5709         my $res = $conf->{snapshots}->{$snapname};
5710
5711         die "snapshot '$snapname' does not exist\n" if !defined($res);
5712
5713         return $res;
5714     };
5715
5716     my $snap = &$get_snapshot_config();
5717
5718     foreach_drive($snap, sub {
5719         my ($ds, $drive) = @_;
5720
5721         return if drive_is_cdrom($drive);
5722
5723         my $volid = $drive->{file};
5724
5725         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5726     });
5727
5728     my $updatefn = sub {
5729
5730         $conf = load_config($vmid);
5731
5732         $snap = &$get_snapshot_config();
5733
5734         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5735             if $snap->{snapstate};
5736
5737         if ($prepare) {
5738             check_lock($conf);
5739             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5740         }
5741
5742         die "unable to rollback vm $vmid: vm is running\n"
5743             if check_running($vmid);
5744
5745         if ($prepare) {
5746             $conf->{lock} = 'rollback';
5747         } else {
5748             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5749             delete $conf->{lock};
5750         }
5751
5752         my $forcemachine;
5753
5754         if (!$prepare) {
5755             my $has_machine_config = defined($conf->{machine});
5756
5757             # copy snapshot config to current config
5758             $conf = &$snapshot_apply_config($conf, $snap);
5759             $conf->{parent} = $snapname;
5760
5761             # Note: old code did not store 'machine', so we try to be smart
5762             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5763             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5764             # we remove the 'machine' configuration if not explicitly specified
5765             # in the original config.
5766             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5767         }
5768
5769         update_config_nolock($vmid, $conf, 1);
5770
5771         if (!$prepare && $snap->{vmstate}) {
5772             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5773             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5774         }
5775     };
5776
5777     lock_config($vmid, $updatefn);
5778
5779     foreach_drive($snap, sub {
5780         my ($ds, $drive) = @_;
5781
5782         return if drive_is_cdrom($drive);
5783
5784         my $volid = $drive->{file};
5785         my $device = "drive-$ds";
5786
5787         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5788     });
5789
5790     $prepare = 0;
5791     lock_config($vmid, $updatefn);
5792 }
5793
5794 my $savevm_wait = sub {
5795     my ($vmid) = @_;
5796
5797     for(;;) {
5798         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5799         if (!$stat->{status}) {
5800             die "savevm not active\n";
5801         } elsif ($stat->{status} eq 'active') {
5802             sleep(1);
5803             next;
5804         } elsif ($stat->{status} eq 'completed') {
5805             last;
5806         } else {
5807             die "query-savevm returned status '$stat->{status}'\n";
5808         }
5809     }
5810 };
5811
5812 sub do_snapshots_with_qemu {
5813     my ($storecfg, $volid) = @_;
5814
5815     my $storage_name = PVE::Storage::parse_volume_id($volid);
5816
5817     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5818         return 1;
5819     }
5820
5821     if ($volid =~ m/\.(qcow2|qed)$/){
5822         return 1;
5823     }
5824
5825     return undef;
5826 }
5827
5828 sub snapshot_create {
5829     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5830
5831     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5832
5833     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5834
5835     my $config = load_config($vmid);
5836
5837     my $running = check_running($vmid);
5838
5839     my $freezefs = $running && $config->{agent};
5840     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5841
5842     my $drivehash = {};
5843
5844     if ($freezefs) {
5845         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5846         warn "guest-fsfreeze-freeze problems - $@" if $@;
5847     }
5848
5849     eval {
5850         # create internal snapshots of all drives
5851
5852         my $storecfg = PVE::Storage::config();
5853
5854         if ($running) {
5855             if ($snap->{vmstate}) {
5856                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5857                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5858                 &$savevm_wait($vmid);
5859             } else {
5860                 vm_mon_cmd($vmid, "savevm-start");
5861             }
5862         };
5863
5864         foreach_drive($snap, sub {
5865             my ($ds, $drive) = @_;
5866
5867             return if drive_is_cdrom($drive);
5868
5869             my $volid = $drive->{file};
5870             my $device = "drive-$ds";
5871
5872             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5873             $drivehash->{$ds} = 1;
5874        });
5875     };
5876     my $err = $@;
5877
5878     if ($running) {
5879         eval { vm_mon_cmd($vmid, "savevm-end")  };
5880         warn $@ if $@;
5881
5882         if ($freezefs) {
5883             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5884             warn "guest-fsfreeze-thaw problems - $@" if $@;
5885         }
5886
5887         # savevm-end is async, we need to wait
5888         for (;;) {
5889             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5890             if (!$stat->{bytes}) {
5891                 last;
5892             } else {
5893                 print "savevm not yet finished\n";
5894                 sleep(1);
5895                 next;
5896             }
5897         }
5898     }
5899
5900     if ($err) {
5901         warn "snapshot create failed: starting cleanup\n";
5902         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5903         warn $@ if $@;
5904         die $err;
5905     }
5906
5907     &$snapshot_commit($vmid, $snapname);
5908 }
5909
5910 # Note: $drivehash is only set when called from snapshot_create.
5911 sub snapshot_delete {
5912     my ($vmid, $snapname, $force, $drivehash) = @_;
5913
5914     my $prepare = 1;
5915
5916     my $snap;
5917     my $unused = [];
5918
5919     my $unlink_parent = sub {
5920         my ($confref, $new_parent) = @_;
5921
5922         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5923             if ($new_parent) {
5924                 $confref->{parent} = $new_parent;
5925             } else {
5926                 delete $confref->{parent};
5927             }
5928         }
5929     };
5930
5931     my $updatefn =  sub {
5932         my ($remove_drive) = @_;
5933
5934         my $conf = load_config($vmid);
5935
5936         if (!$drivehash) {
5937             check_lock($conf);
5938             die "you can't delete a snapshot if vm is a template\n"
5939                 if is_template($conf);
5940         }
5941
5942         $snap = $conf->{snapshots}->{$snapname};
5943
5944         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5945
5946         # remove parent refs
5947         if (!$prepare) {
5948             &$unlink_parent($conf, $snap->{parent});
5949             foreach my $sn (keys %{$conf->{snapshots}}) {
5950                 next if $sn eq $snapname;
5951                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5952             }
5953         }
5954
5955         if ($remove_drive) {
5956             if ($remove_drive eq 'vmstate') {
5957                 delete $snap->{$remove_drive};
5958             } else {
5959                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5960                 my $volid = $drive->{file};
5961                 delete $snap->{$remove_drive};
5962                 add_unused_volume($conf, $volid);
5963             }
5964         }
5965
5966         if ($prepare) {
5967             $snap->{snapstate} = 'delete';
5968         } else {
5969             delete $conf->{snapshots}->{$snapname};
5970             delete $conf->{lock} if $drivehash;
5971             foreach my $volid (@$unused) {
5972                 add_unused_volume($conf, $volid);
5973             }
5974         }
5975
5976         update_config_nolock($vmid, $conf, 1);
5977     };
5978
5979     lock_config($vmid, $updatefn);
5980
5981     # now remove vmstate file
5982
5983     my $storecfg = PVE::Storage::config();
5984
5985     if ($snap->{vmstate}) {
5986         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5987         if (my $err = $@) {
5988             die $err if !$force;
5989             warn $err;
5990         }
5991         # save changes (remove vmstate from snapshot)
5992         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5993     };
5994
5995     # now remove all internal snapshots
5996     foreach_drive($snap, sub {
5997         my ($ds, $drive) = @_;
5998
5999         return if drive_is_cdrom($drive);
6000
6001         my $volid = $drive->{file};
6002         my $device = "drive-$ds";
6003
6004         if (!$drivehash || $drivehash->{$ds}) {
6005             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6006             if (my $err = $@) {
6007                 die $err if !$force;
6008                 warn $err;
6009             }
6010         }
6011
6012         # save changes (remove drive fron snapshot)
6013         lock_config($vmid, $updatefn, $ds) if !$force;
6014         push @$unused, $volid;
6015     });
6016
6017     # now cleanup config
6018     $prepare = 0;
6019     lock_config($vmid, $updatefn);
6020 }
6021
6022 sub has_feature {
6023     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6024
6025     my $err;
6026     foreach_drive($conf, sub {
6027         my ($ds, $drive) = @_;
6028
6029         return if drive_is_cdrom($drive);
6030         my $volid = $drive->{file};
6031         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6032     });
6033
6034     return $err ? 0 : 1;
6035 }
6036
6037 sub template_create {
6038     my ($vmid, $conf, $disk) = @_;
6039
6040     my $storecfg = PVE::Storage::config();
6041
6042     foreach_drive($conf, sub {
6043         my ($ds, $drive) = @_;
6044
6045         return if drive_is_cdrom($drive);
6046         return if $disk && $ds ne $disk;
6047
6048         my $volid = $drive->{file};
6049         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6050
6051         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6052         $drive->{file} = $voliddst;
6053         $conf->{$ds} = print_drive($vmid, $drive);
6054         update_config_nolock($vmid, $conf, 1);
6055     });
6056 }
6057
6058 sub is_template {
6059     my ($conf) = @_;
6060
6061     return 1 if defined $conf->{template} && $conf->{template} == 1;
6062 }
6063
6064 sub qemu_img_convert {
6065     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6066
6067     my $storecfg = PVE::Storage::config();
6068     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6069     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6070
6071     if ($src_storeid && $dst_storeid) {
6072         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6073         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6074
6075         my $src_format = qemu_img_format($src_scfg, $src_volname);
6076         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6077
6078         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6079         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6080
6081         my $cmd = [];
6082         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6083         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6084         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6085
6086         my $parser = sub {
6087             my $line = shift;
6088             if($line =~ m/\((\S+)\/100\%\)/){
6089                 my $percent = $1;
6090                 my $transferred = int($size * $percent / 100);
6091                 my $remaining = $size - $transferred;
6092
6093                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6094             }
6095
6096         };
6097
6098         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6099         my $err = $@;
6100         die "copy failed: $err" if $err;
6101     }
6102 }
6103
6104 sub qemu_img_format {
6105     my ($scfg, $volname) = @_;
6106
6107     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6108         return $1;
6109     } else {
6110         return "raw";
6111     }
6112 }
6113
6114 sub qemu_drive_mirror {
6115     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6116
6117     my $storecfg = PVE::Storage::config();
6118     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6119
6120     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6121
6122     my $format = qemu_img_format($dst_scfg, $dst_volname);
6123
6124     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6125
6126     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6127     $opts->{format} = $format if $format;
6128
6129     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6130
6131     eval {
6132     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6133         while (1) {
6134             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6135             my $stat = @$stats[0];
6136             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6137             die "error job is not mirroring" if $stat->{type} ne "mirror";
6138
6139             my $busy = $stat->{busy};
6140             my $ready = $stat->{ready};
6141
6142             if (my $total = $stat->{len}) {
6143                 my $transferred = $stat->{offset} || 0;
6144                 my $remaining = $total - $transferred;
6145                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6146
6147                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6148             }
6149
6150
6151             if ($stat->{ready} eq 'true') {
6152
6153                 last if $vmiddst != $vmid;
6154
6155                 # try to switch the disk if source and destination are on the same guest
6156                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6157                 last if !$@;
6158                 die $@ if $@ !~ m/cannot be completed/;
6159             }
6160             sleep 1;
6161         }
6162
6163
6164     };
6165     my $err = $@;
6166
6167     my $cancel_job = sub {
6168         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6169         while (1) {
6170             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6171             my $stat = @$stats[0];
6172             last if !$stat;
6173             sleep 1;
6174         }
6175     };
6176
6177     if ($err) {
6178         eval { &$cancel_job(); };
6179         die "mirroring error: $err";
6180     }
6181
6182     if ($vmiddst != $vmid) {
6183         # if we clone a disk for a new target vm, we don't switch the disk
6184         &$cancel_job(); # so we call block-job-cancel
6185     }
6186 }
6187
6188 sub clone_disk {
6189     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6190         $newvmid, $storage, $format, $full, $newvollist) = @_;
6191
6192     my $newvolid;
6193
6194     if (!$full) {
6195         print "create linked clone of drive $drivename ($drive->{file})\n";
6196         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6197         push @$newvollist, $newvolid;
6198     } else {
6199         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6200         $storeid = $storage if $storage;
6201
6202         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6203         if (!$format) {
6204             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6205             $format = qemu_img_format($scfg, $volname);
6206         }
6207
6208         # test if requested format is supported - else use default
6209         my $supported = grep { $_ eq $format } @$validFormats;
6210         $format = $defFormat if !$supported;
6211
6212         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6213
6214         print "create full clone of drive $drivename ($drive->{file})\n";
6215         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6216         push @$newvollist, $newvolid;
6217
6218         if (!$running || $snapname) {
6219             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6220         } else {
6221             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6222         }
6223     }
6224
6225     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6226
6227     my $disk = $drive;
6228     $disk->{format} = undef;
6229     $disk->{file} = $newvolid;
6230     $disk->{size} = $size;
6231
6232     return $disk;
6233 }
6234
6235 # this only works if VM is running
6236 sub get_current_qemu_machine {
6237     my ($vmid) = @_;
6238
6239     my $cmd = { execute => 'query-machines', arguments => {} };
6240     my $res = vm_qmp_command($vmid, $cmd);
6241
6242     my ($current, $default);
6243     foreach my $e (@$res) {
6244         $default = $e->{name} if $e->{'is-default'};
6245         $current = $e->{name} if $e->{'is-current'};
6246     }
6247
6248     # fallback to the default machine if current is not supported by qemu
6249     return $current || $default || 'pc';
6250 }
6251
6252 sub qemu_machine_feature_enabled {
6253     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6254
6255     my $current_major;
6256     my $current_minor;
6257
6258     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6259
6260         $current_major = $3;
6261         $current_minor = $4;
6262
6263     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6264
6265         $current_major = $1;
6266         $current_minor = $2;
6267     }
6268
6269     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6270
6271
6272 }
6273
6274 sub lspci {
6275
6276     my $devices = {};
6277
6278     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6279             my (undef, $id, $function) = @_;
6280             my $res = { id => $id, function => $function};
6281             push @{$devices->{$id}}, $res;
6282     });
6283
6284     return $devices;
6285 }
6286
6287 sub vm_iothreads_list {
6288     my ($vmid) = @_;
6289
6290     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6291
6292     my $iothreads = {};
6293     foreach my $iothread (@$res) {
6294         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6295     }
6296
6297     return $iothreads;
6298 }
6299
6300 sub scsihw_infos {
6301     my ($conf, $drive) = @_;
6302
6303     my $maxdev = 0;
6304
6305     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6306         $maxdev = 7;
6307     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6308         $maxdev = 1;
6309     } else {
6310         $maxdev = 256;
6311     }
6312
6313     my $controller = int($drive->{index} / $maxdev);
6314     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6315
6316     return ($maxdev, $controller, $controller_prefix);
6317 }
6318
6319 1;