PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35 use URI::Escape;
  36
  37 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  38
  39 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  40
  41 # Note about locking: we use flock on the config file protect
  42 # against concurent actions.
  43 # Aditionaly, we have a 'lock' setting in the config file. This
  44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  45 # allowed when such lock is set. But you can ignore this kind of
  46 # lock with the --skiplock flag.
  47
  48 cfs_register_file('/qemu-server/',
  49                   \&parse_vm_config,
  50                   \&write_vm_config);
  51
  52 PVE::JSONSchema::register_standard_option('skiplock', {
  53     description => "Ignore locks - only root is allowed to use this option.",
  54     type => 'boolean',
  55     optional => 1,
  56 });
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 #no warnings 'redefine';
  72
  73 sub cgroups_write {
  74    my ($controller, $vmid, $option, $value) = @_;
  75
  76    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  77    PVE::ProcFSTools::write_proc_entry($path, $value);
  78
  79 }
  80
  81 my $nodename = PVE::INotify::nodename();
  82
  83 mkdir "/etc/pve/nodes/$nodename";
  84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  85 mkdir $confdir;
  86
  87 my $var_run_tmpdir = "/var/run/qemu-server";
  88 mkdir $var_run_tmpdir;
  89
  90 my $lock_dir = "/var/lock/qemu-server";
  91 mkdir $lock_dir;
  92
  93 my $pcisysfs = "/sys/bus/pci";
  94
  95 my $cpudesc = {
  96     cputype => {
  97         description => "Emulated CPU type.",
  98         type => 'string',
  99         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 100         default => 'kvm64',
 101         default_key => 1,
 102     },
 103     hidden => {
 104         description => "Do not identify as a KVM virtual machine.",
 105         type => 'boolean',
 106         optional => 1,
 107         default => 0
 108     },
 109 };
 110
 111 my $confdesc = {
 112     onboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Specifies whether a VM will be started during system bootup.",
 116         default => 0,
 117     },
 118     autostart => {
 119         optional => 1,
 120         type => 'boolean',
 121         description => "Automatic restart after crash (currently ignored).",
 122         default => 0,
 123     },
 124     hotplug => {
 125         optional => 1,
 126         type => 'string', format => 'pve-hotplug-features',
 127         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 128         default => 'network,disk,usb',
 129     },
 130     reboot => {
 131         optional => 1,
 132         type => 'boolean',
 133         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 134         default => 1,
 135     },
 136     lock => {
 137         optional => 1,
 138         type => 'string',
 139         description => "Lock/unlock the VM.",
 140         enum => [qw(migrate backup snapshot rollback)],
 141     },
 142     cpulimit => {
 143         optional => 1,
 144         type => 'number',
 145         description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 146         minimum => 0,
 147         maximum => 128,
 148         default => 0,
 149     },
 150     cpuunits => {
 151         optional => 1,
 152         type => 'integer',
 153         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 154         minimum => 0,
 155         maximum => 500000,
 156         default => 1000,
 157     },
 158     memory => {
 159         optional => 1,
 160         type => 'integer',
 161         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 162         minimum => 16,
 163         default => 512,
 164     },
 165     balloon => {
 166         optional => 1,
 167         type => 'integer',
 168         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 169         minimum => 0,
 170     },
 171     shares => {
 172         optional => 1,
 173         type => 'integer',
 174         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 175         minimum => 0,
 176         maximum => 50000,
 177         default => 1000,
 178     },
 179     keyboard => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 183         enum => PVE::Tools::kvmkeymaplist(),
 184         default => 'en-us',
 185     },
 186     name => {
 187         optional => 1,
 188         type => 'string', format => 'dns-name',
 189         description => "Set a name for the VM. Only used on the configuration web interface.",
 190     },
 191     scsihw => {
 192         optional => 1,
 193         type => 'string',
 194         description => "scsi controller model",
 195         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 196         default => 'lsi',
 197     },
 198     description => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 202     },
 203     ostype => {
 204         optional => 1,
 205         type => 'string',
 206         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 207         description => <<EODESC,
 208 Used to enable special optimization/features for specific
 209 operating systems:
 210
 211 other  => unspecified OS
 212 wxp    => Microsoft Windows XP
 213 w2k    => Microsoft Windows 2000
 214 w2k3   => Microsoft Windows 2003
 215 w2k8   => Microsoft Windows 2008
 216 wvista => Microsoft Windows Vista
 217 win7   => Microsoft Windows 7
 218 win8   => Microsoft Windows 8/2012
 219 l24    => Linux 2.4 Kernel
 220 l26    => Linux 2.6/3.X Kernel
 221 solaris => solaris/opensolaris/openindiania kernel
 222
 223 other|l24|l26|solaris                       ... no special behaviour
 224 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 225 EODESC
 226     },
 227     boot => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 231         pattern => '[acdn]{1,4}',
 232         default => 'cdn',
 233     },
 234     bootdisk => {
 235         optional => 1,
 236         type => 'string', format => 'pve-qm-bootdisk',
 237         description => "Enable booting from specified disk.",
 238         pattern => '(ide|sata|scsi|virtio)\d+',
 239     },
 240     smp => {
 241         optional => 1,
 242         type => 'integer',
 243         description => "The number of CPUs. Please use option -sockets instead.",
 244         minimum => 1,
 245         default => 1,
 246     },
 247     sockets => {
 248         optional => 1,
 249         type => 'integer',
 250         description => "The number of CPU sockets.",
 251         minimum => 1,
 252         default => 1,
 253     },
 254     cores => {
 255         optional => 1,
 256         type => 'integer',
 257         description => "The number of cores per socket.",
 258         minimum => 1,
 259         default => 1,
 260     },
 261     numa => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable NUMA.",
 265         default => 0,
 266     },
 267     vcpus => {
 268         optional => 1,
 269         type => 'integer',
 270         description => "Number of hotplugged vcpus.",
 271         minimum => 1,
 272         default => 0,
 273     },
 274     acpi => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable ACPI.",
 278         default => 1,
 279     },
 280     agent => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Enable/disable Qemu GuestAgent.",
 284         default => 0,
 285     },
 286     kvm => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Enable/disable KVM hardware virtualization.",
 290         default => 1,
 291     },
 292     tdf => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Enable/disable time drift fix.",
 296         default => 0,
 297     },
 298     localtime => {
 299         optional => 1,
 300         type => 'boolean',
 301         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 302     },
 303     freeze => {
 304         optional => 1,
 305         type => 'boolean',
 306         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 307     },
 308     vga => {
 309         optional => 1,
 310         type => 'string',
 311         description => "Select the VGA type. If you want to use high resolution" .
 312             " modes (>= 1280x1024x16) then you should use the options " .
 313             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 314             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 315             "display sever. For win* OS you can select how many independent " .
 316             "displays you want, Linux guests can add displays them self. " .
 317             "You can also run without any graphic card, using a serial device" .
 318             " as terminal.",
 319         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 320     },
 321     watchdog => {
 322         optional => 1,
 323         type => 'string', format => 'pve-qm-watchdog',
 324         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 325         description => "Create a virtual hardware watchdog device. Once enabled" .
 326             " (by a guest action), the watchdog must be periodically polled " .
 327             "by an agent inside the guest or else the watchdog will reset " .
 328             "the guest (or execute the respective action specified)",
 329     },
 330     startdate => {
 331         optional => 1,
 332         type => 'string',
 333         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 334         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 335         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 336         default => 'now',
 337     },
 338     startup =>  get_standard_option('pve-startup-order'),
 339     template => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable Template.",
 343         default => 0,
 344     },
 345     args => {
 346         optional => 1,
 347         type => 'string',
 348         description => <<EODESCR,
 349 NOTE: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 350
 351 args: -no-reboot -no-hpet
 352 EODESCR
 353     },
 354     tablet => {
 355         optional => 1,
 356         type => 'boolean',
 357         default => 1,
 358         description => "Enable/disable the USB tablet device. This device is " .
 359             "usually needed to allow absolute mouse positioning with VNC. " .
 360             "Else the mouse runs out of sync with normal VNC clients. " .
 361             "If you're running lots of console-only guests on one host, " .
 362             "you may consider disabling this to save some context switches. " .
 363             "This is turned off by default if you use spice (-vga=qxl).",
 364     },
 365     migrate_speed => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 369         minimum => 0,
 370         default => 0,
 371     },
 372     migrate_downtime => {
 373         optional => 1,
 374         type => 'number',
 375         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 376         minimum => 0,
 377         default => 0.1,
 378     },
 379     cdrom => {
 380         optional => 1,
 381         type => 'string', format => 'pve-qm-drive',
 382         typetext => 'volume',
 383         description => "This is an alias for option -ide2",
 384     },
 385     cpu => {
 386         optional => 1,
 387         description => "Emulated CPU type.",
 388         type => 'string',
 389         format => $cpudesc,
 390     },
 391     parent => get_standard_option('pve-snapshot-name', {
 392         optional => 1,
 393         description => "Parent snapshot name. This is used internally, and should not be modified.",
 394     }),
 395     snaptime => {
 396         optional => 1,
 397         description => "Timestamp for snapshots.",
 398         type => 'integer',
 399         minimum => 0,
 400     },
 401     vmstate => {
 402         optional => 1,
 403         type => 'string', format => 'pve-volume-id',
 404         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 405     },
 406     machine => {
 407         description => "Specific the Qemu machine type.",
 408         type => 'string',
 409         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 410         maxLength => 40,
 411         optional => 1,
 412     },
 413     smbios1 => {
 414         description => "Specify SMBIOS type 1 fields.",
 415         type => 'string', format => 'pve-qm-smbios1',
 416         maxLength => 256,
 417         optional => 1,
 418     },
 419     protection => {
 420         optional => 1,
 421         type => 'boolean',
 422         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 423         default => 0,
 424     },
 425     bios => {
 426         optional => 1,
 427         type => 'string',
 428         enum => [ qw(seabios ovmf) ],
 429         description => "Select BIOS implementation.",
 430         default => 'seabios',
 431     },
 432 };
 433
 434 # what about other qemu settings ?
 435 #cpu => 'string',
 436 #machine => 'string',
 437 #fda => 'file',
 438 #fdb => 'file',
 439 #mtdblock => 'file',
 440 #sd => 'file',
 441 #pflash => 'file',
 442 #snapshot => 'bool',
 443 #bootp => 'file',
 444 ##tftp => 'dir',
 445 ##smb => 'dir',
 446 #kernel => 'file',
 447 #append => 'string',
 448 #initrd => 'file',
 449 ##soundhw => 'string',
 450
 451 while (my ($k, $v) = each %$confdesc) {
 452     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 453 }
 454
 455 my $MAX_IDE_DISKS = 4;
 456 my $MAX_SCSI_DISKS = 14;
 457 my $MAX_VIRTIO_DISKS = 16;
 458 my $MAX_SATA_DISKS = 6;
 459 my $MAX_USB_DEVICES = 5;
 460 my $MAX_NETS = 32;
 461 my $MAX_UNUSED_DISKS = 8;
 462 my $MAX_HOSTPCI_DEVICES = 4;
 463 my $MAX_SERIAL_PORTS = 4;
 464 my $MAX_PARALLEL_PORTS = 3;
 465 my $MAX_NUMA = 8;
 466 my $MAX_MEM = 4194304;
 467 my $STATICMEM = 1024;
 468
 469 my $numadesc = {
 470     optional => 1,
 471     type => 'string', format => 'pve-qm-numanode',
 472     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 473     description => "numa topology",
 474 };
 475 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 476
 477 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 478     $confdesc->{"numa$i"} = $numadesc;
 479 }
 480
 481 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 482                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 483                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 484 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 485
 486 my $netdesc = {
 487     optional => 1,
 488     type => 'string', format => 'pve-qm-net',
 489     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 490     description => <<EODESCR,
 491 Specify network devices.
 492
 493 MODEL is one of: $nic_model_list_txt
 494
 495 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 496 automatically generated if not specified.
 497
 498 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 499
 500 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 501
 502 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 503
 504 10.0.2.2   Gateway
 505 10.0.2.3   DNS Server
 506 10.0.2.4   SMB Server
 507
 508 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 509
 510 EODESCR
 511 };
 512 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 513
 514 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 515     $confdesc->{"net$i"} = $netdesc;
 516 }
 517
 518 my $drivename_hash;
 519
 520 my %drivedesc_base = (
 521     volume => { alias => 'file' },
 522     file => {
 523         type => 'pve-volume-id',
 524         default_key => 1,
 525         format_description => 'volume',
 526         description => "The drive's backing volume.",
 527     },
 528     media => {
 529         type => 'string',
 530         format_description => 'cdrom|disk',
 531         enum => [qw(cdrom disk)],
 532         description => "The drive's media type.",
 533         default => 'disk',
 534         optional => 1
 535     },
 536     cyls => {
 537         type => 'integer',
 538         format_description => 'count',
 539         description => "Force the drive's physical geometry to have a specific cylinder count.",
 540         optional => 1
 541     },
 542     heads => {
 543         type => 'integer',
 544         format_description => 'count',
 545         description => "Force the drive's physical geometry to have a specific head count.",
 546         optional => 1
 547     },
 548     secs => {
 549         type => 'integer',
 550         format_description => 'count',
 551         description => "Force the drive's physical geometry to have a specific sector count.",
 552         optional => 1
 553     },
 554     trans => {
 555         type => 'string',
 556         format_description => 'none|lba|auto',
 557         enum => [qw(none lba auto)],
 558         description => "Force disk geometry bios translation mode.",
 559         optional => 1,
 560     },
 561     snapshot => {
 562         type => 'boolean',
 563         format_description => 'on|off',
 564         description => "Whether the drive should be included when making snapshots.",
 565         optional => 1,
 566     },
 567     cache => {
 568         type => 'string',
 569         format_description => 'none|writethrough|writeback|unsafe|directsync',
 570         enum => [qw(none writethrough writeback unsafe directsync)],
 571         description => "The drive's cache mode",
 572         optional => 1,
 573     },
 574     format => {
 575         type => 'string',
 576         format_description => 'drive format',
 577         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 578         description => "The drive's backing file's data format.",
 579         optional => 1,
 580     },
 581     size => {
 582         type => 'string',
 583         format => 'disk-size',
 584         description => "Disk size. This is purely informational and has no effect.",
 585         optional => 1,
 586     },
 587     backup => {
 588         type => 'boolean',
 589         format_description => 'on|off',
 590         description => "Whether the drive should be included when making backups.",
 591         optional => 1,
 592     },
 593     werror => {
 594         type => 'string',
 595         format_description => 'enospc|ignore|report|stop',
 596         enum => [qw(enospc ignore report stop)],
 597         description => 'Write error action.',
 598         optional => 1,
 599     },
 600     aio => {
 601         type => 'string',
 602         format_description => 'native|threads',
 603         enum => [qw(native threads)],
 604         description => 'AIO type to use.',
 605         optional => 1,
 606     },
 607     discard => {
 608         type => 'string',
 609         format_description => 'ignore|on',
 610         enum => [qw(ignore on)],
 611         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 612         optional => 1,
 613     },
 614     detect_zeroes => {
 615         type => 'boolean',
 616         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 617         optional => 1,
 618     },
 619     serial => {
 620         type => 'string',
 621         format => 'urlencoded',
 622         format_description => 'serial',
 623         maxLength => 20*3, # *3 since it's %xx url enoded
 624         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 625         optional => 1,
 626     }
 627 );
 628
 629 my %rerror_fmt = (
 630     rerror => {
 631         type => 'string',
 632         format_description => 'ignore|report|stop',
 633         enum => [qw(ignore report stop)],
 634         description => 'Read error action.',
 635         optional => 1,
 636     },
 637 );
 638
 639 my %iothread_fmt = ( iothread => {
 640         type => 'boolean',
 641         format_description => 'off|on',
 642         description => "Whether to use iothreads for this drive",
 643         optional => 1,
 644 });
 645
 646 my %model_fmt = (
 647     model => {
 648         type => 'string',
 649         format => 'urlencoded',
 650         format_description => 'model',
 651         maxLength => 40*3, # *3 since it's %xx url enoded
 652         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 653         optional => 1,
 654     },
 655 );
 656
 657 my %queues_fmt = (
 658     queues => {
 659         type => 'integer',
 660         format_description => 'nbqueues',
 661         description => "Number of queues.",
 662         minimum => 2,
 663         optional => 1
 664     }
 665 );
 666
 667 my $add_throttle_desc = sub {
 668     my ($key, $type, $what, $size, $longsize) = @_;
 669     $drivedesc_base{$key} = {
 670         type => $type,
 671         format_description => $size,
 672         description => "Maximum $what speed in $longsize per second.",
 673         optional => 1,
 674     };
 675 };
 676 # throughput: (leaky bucket)
 677 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 678 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 679 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 680 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 681 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 682 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 683 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 684 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 685 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 686
 687 # pools: (pool of IO before throttling starts taking effect)
 688 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 689 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 690 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 691 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 692 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 693 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 694
 695 my $ide_fmt = {
 696     %drivedesc_base,
 697     %rerror_fmt,
 698     %model_fmt,
 699 };
 700
 701 my $idedesc = {
 702     optional => 1,
 703     type => 'string', format => $ide_fmt,
 704     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 705 };
 706 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 707
 708 my $scsi_fmt = {
 709     %drivedesc_base,
 710     %iothread_fmt,
 711     %queues_fmt,
 712 };
 713 my $scsidesc = {
 714     optional => 1,
 715     type => 'string', format => $scsi_fmt,
 716     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 717 };
 718 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 719
 720 my $sata_fmt = {
 721     %drivedesc_base,
 722     %rerror_fmt,
 723 };
 724 my $satadesc = {
 725     optional => 1,
 726     type => 'string', format => $sata_fmt,
 727     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 728 };
 729 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 730
 731 my $virtio_fmt = {
 732     %drivedesc_base,
 733     %iothread_fmt,
 734     %rerror_fmt,
 735 };
 736 my $virtiodesc = {
 737     optional => 1,
 738     type => 'string', format => $virtio_fmt,
 739     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 740 };
 741 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 742
 743 my $alldrive_fmt = {
 744     %drivedesc_base,
 745     %rerror_fmt,
 746     %iothread_fmt,
 747     %model_fmt,
 748     %queues_fmt,
 749 };
 750
 751 my $usbformat = {
 752     host => {
 753         default_key => 1,
 754         type => 'string', format => 'pve-qm-usb-device',
 755         format_description => 'HOSTUSBDEVICE|spice',
 756         description => 'The Host USB device or port or the value spice',
 757     },
 758     usb3 => {
 759         optional => 1,
 760         type => 'boolean',
 761         format_description => 'yes|no',
 762         description => 'Specifies whether if given host option is a USB3 device or port',
 763     },
 764 };
 765
 766 my $usbdesc = {
 767     optional => 1,
 768     type => 'string', format => $usbformat,
 769     description => <<EODESCR,
 770 Configure an USB device (n is 0 to 4). This can be used to
 771 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 772
 773 'bus-port(.port)*' (decimal numbers) or
 774 'vendor_id:product_id' (hexadeciaml numbers) or
 775 'spice'
 776
 777 You can use the 'lsusb -t' command to list existing usb devices.
 778
 779 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 780
 781 The value 'spice' can be used to add a usb redirection devices for spice.
 782
 783 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 784
 785 EODESCR
 786 };
 787 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 788
 789 my $hostpcidesc = {
 790         optional => 1,
 791         type => 'string', format => 'pve-qm-hostpci',
 792         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 793         description => <<EODESCR,
 794 Map host pci devices. HOSTPCIDEVICE syntax is:
 795
 796 'bus:dev.func' (hexadecimal numbers)
 797
 798 You can us the 'lspci' command to list existing pci devices.
 799
 800 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 801
 802 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 803
 804 Experimental: user reported problems with this option.
 805 EODESCR
 806 };
 807 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 808
 809 my $serialdesc = {
 810         optional => 1,
 811         type => 'string',
 812         pattern => '(/dev/.+|socket)',
 813         description =>  <<EODESCR,
 814 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 815
 816 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 817
 818 Experimental: user reported problems with this option.
 819 EODESCR
 820 };
 821
 822 my $paralleldesc= {
 823         optional => 1,
 824         type => 'string',
 825         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 826         description =>  <<EODESCR,
 827 Map host parallel devices (n is 0 to 2).
 828
 829 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 830
 831 Experimental: user reported problems with this option.
 832 EODESCR
 833 };
 834
 835 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 836     $confdesc->{"parallel$i"} = $paralleldesc;
 837 }
 838
 839 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 840     $confdesc->{"serial$i"} = $serialdesc;
 841 }
 842
 843 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 844     $confdesc->{"hostpci$i"} = $hostpcidesc;
 845 }
 846
 847 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 848     $drivename_hash->{"ide$i"} = 1;
 849     $confdesc->{"ide$i"} = $idedesc;
 850 }
 851
 852 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 853     $drivename_hash->{"sata$i"} = 1;
 854     $confdesc->{"sata$i"} = $satadesc;
 855 }
 856
 857 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 858     $drivename_hash->{"scsi$i"} = 1;
 859     $confdesc->{"scsi$i"} = $scsidesc ;
 860 }
 861
 862 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 863     $drivename_hash->{"virtio$i"} = 1;
 864     $confdesc->{"virtio$i"} = $virtiodesc;
 865 }
 866
 867 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 868     $confdesc->{"usb$i"} = $usbdesc;
 869 }
 870
 871 my $unuseddesc = {
 872     optional => 1,
 873     type => 'string', format => 'pve-volume-id',
 874     description => "Reference to unused volumes.",
 875 };
 876
 877 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 878     $confdesc->{"unused$i"} = $unuseddesc;
 879 }
 880
 881 my $kvm_api_version = 0;
 882
 883 sub kvm_version {
 884
 885     return $kvm_api_version if $kvm_api_version;
 886
 887     my $fh = IO::File->new("</dev/kvm") ||
 888         return 0;
 889
 890     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 891         $kvm_api_version = $v;
 892     }
 893
 894     $fh->close();
 895
 896     return  $kvm_api_version;
 897 }
 898
 899 my $kvm_user_version;
 900
 901 sub kvm_user_version {
 902
 903     return $kvm_user_version if $kvm_user_version;
 904
 905     $kvm_user_version = 'unknown';
 906
 907     my $code = sub {
 908         my $line = shift;
 909         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 910             $kvm_user_version = $2;
 911         }
 912     };
 913
 914     eval { run_command("kvm -version", outfunc => $code); };
 915     warn $@ if $@;
 916
 917     return $kvm_user_version;
 918
 919 }
 920
 921 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 922
 923 sub valid_drive_names {
 924     # order is important - used to autoselect boot disk
 925     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 926             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 927             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 928             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 929 }
 930
 931 sub is_valid_drivename {
 932     my $dev = shift;
 933
 934     return defined($drivename_hash->{$dev});
 935 }
 936
 937 sub option_exists {
 938     my $key = shift;
 939     return defined($confdesc->{$key});
 940 }
 941
 942 sub nic_models {
 943     return $nic_model_list;
 944 }
 945
 946 sub os_list_description {
 947
 948     return {
 949         other => 'Other',
 950         wxp => 'Windows XP',
 951         w2k => 'Windows 2000',
 952         w2k3 =>, 'Windows 2003',
 953         w2k8 => 'Windows 2008',
 954         wvista => 'Windows Vista',
 955         win7 => 'Windows 7',
 956         win8 => 'Windows 8/2012',
 957         l24 => 'Linux 2.4',
 958         l26 => 'Linux 2.6',
 959     };
 960 }
 961
 962 my $cdrom_path;
 963
 964 sub get_cdrom_path {
 965
 966     return  $cdrom_path if $cdrom_path;
 967
 968     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 969     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 970     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 971 }
 972
 973 sub get_iso_path {
 974     my ($storecfg, $vmid, $cdrom) = @_;
 975
 976     if ($cdrom eq 'cdrom') {
 977         return get_cdrom_path();
 978     } elsif ($cdrom eq 'none') {
 979         return '';
 980     } elsif ($cdrom =~ m|^/|) {
 981         return $cdrom;
 982     } else {
 983         return PVE::Storage::path($storecfg, $cdrom);
 984     }
 985 }
 986
 987 # try to convert old style file names to volume IDs
 988 sub filename_to_volume_id {
 989     my ($vmid, $file, $media) = @_;
 990
 991     if (!($file eq 'none' || $file eq 'cdrom' ||
 992           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 993
 994         return undef if $file =~ m|/|;
 995
 996         if ($media && $media eq 'cdrom') {
 997             $file = "local:iso/$file";
 998         } else {
 999             $file = "local:$vmid/$file";
1000         }
1001     }
1002
1003     return $file;
1004 }
1005
1006 sub verify_media_type {
1007     my ($opt, $vtype, $media) = @_;
1008
1009     return if !$media;
1010
1011     my $etype;
1012     if ($media eq 'disk') {
1013         $etype = 'images';
1014     } elsif ($media eq 'cdrom') {
1015         $etype = 'iso';
1016     } else {
1017         die "internal error";
1018     }
1019
1020     return if ($vtype eq $etype);
1021
1022     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1023 }
1024
1025 sub cleanup_drive_path {
1026     my ($opt, $storecfg, $drive) = @_;
1027
1028     # try to convert filesystem paths to volume IDs
1029
1030     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1031         ($drive->{file} !~ m|^/dev/.+|) &&
1032         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1033         ($drive->{file} !~ m/^\d+$/)) {
1034         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1035         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1036         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1037         verify_media_type($opt, $vtype, $drive->{media});
1038         $drive->{file} = $volid;
1039     }
1040
1041     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1042 }
1043
1044 sub parse_hotplug_features {
1045     my ($data) = @_;
1046
1047     my $res = {};
1048
1049     return $res if $data eq '0';
1050
1051     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1052
1053     foreach my $feature (PVE::Tools::split_list($data)) {
1054         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1055             $res->{$1} = 1;
1056         } else {
1057             warn "ignoring unknown hotplug feature '$feature'\n";
1058         }
1059     }
1060     return $res;
1061 }
1062
1063 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1064 sub pve_verify_hotplug_features {
1065     my ($value, $noerr) = @_;
1066
1067     return $value if parse_hotplug_features($value);
1068
1069     return undef if $noerr;
1070
1071     die "unable to parse hotplug option\n";
1072 }
1073
1074 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1075 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1076 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1077 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1078 #        [,iothread=on][,serial=serial][,model=model]
1079
1080 sub parse_drive {
1081     my ($key, $data) = @_;
1082
1083     my ($interface, $index);
1084
1085     if ($key =~ m/^([^\d]+)(\d+)$/) {
1086         $interface = $1;
1087         $index = $2;
1088     } else {
1089         return undef;
1090     }
1091
1092     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1093                                      : $confdesc->{$key}->{format};
1094     if (!$desc) {
1095         warn "invalid drive key: $key\n";
1096         return undef;
1097     }
1098     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1099     return undef if !$res;
1100     $res->{interface} = $interface;
1101     $res->{index} = $index;
1102
1103     my $error = 0;
1104     foreach my $opt (qw(bps bps_rd bps_wr)) {
1105         if (my $bps = defined(delete $res->{$opt})) {
1106             if (defined($res->{"m$opt"})) {
1107                 warn "both $opt and m$opt specified\n";
1108                 ++$error;
1109                 next;
1110             }
1111             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1112         }
1113     }
1114     return undef if $error;
1115
1116     return undef if $res->{mbps_rd} && $res->{mbps};
1117     return undef if $res->{mbps_wr} && $res->{mbps};
1118     return undef if $res->{iops_rd} && $res->{iops};
1119     return undef if $res->{iops_wr} && $res->{iops};
1120
1121     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1122         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1123         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1124         return undef if $res->{interface} eq 'virtio';
1125     }
1126
1127     if (my $size = $res->{size}) {
1128         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1129     }
1130
1131     return $res;
1132 }
1133
1134 sub print_drive {
1135     my ($vmid, $drive) = @_;
1136     my $data = { %$drive };
1137     delete $data->{$_} for qw(index interface);
1138     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1139 }
1140
1141 sub scsi_inquiry {
1142     my($fh, $noerr) = @_;
1143
1144     my $SG_IO = 0x2285;
1145     my $SG_GET_VERSION_NUM = 0x2282;
1146
1147     my $versionbuf = "\x00" x 8;
1148     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1149     if (!$ret) {
1150         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1151         return undef;
1152     }
1153     my $version = unpack("I", $versionbuf);
1154     if ($version < 30000) {
1155         die "scsi generic interface too old\n"  if !$noerr;
1156         return undef;
1157     }
1158
1159     my $buf = "\x00" x 36;
1160     my $sensebuf = "\x00" x 8;
1161     my $cmd = pack("C x3 C x1", 0x12, 36);
1162
1163     # see /usr/include/scsi/sg.h
1164     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1165
1166     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1167                       length($sensebuf), 0, length($buf), $buf,
1168                       $cmd, $sensebuf, 6000);
1169
1170     $ret = ioctl($fh, $SG_IO, $packet);
1171     if (!$ret) {
1172         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1173         return undef;
1174     }
1175
1176     my @res = unpack($sg_io_hdr_t, $packet);
1177     if ($res[17] || $res[18]) {
1178         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1179         return undef;
1180     }
1181
1182     my $res = {};
1183     (my $byte0, my $byte1, $res->{vendor},
1184      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1185
1186     $res->{removable} = $byte1 & 128 ? 1 : 0;
1187     $res->{type} = $byte0 & 31;
1188
1189     return $res;
1190 }
1191
1192 sub path_is_scsi {
1193     my ($path) = @_;
1194
1195     my $fh = IO::File->new("+<$path") || return undef;
1196     my $res = scsi_inquiry($fh, 1);
1197     close($fh);
1198
1199     return $res;
1200 }
1201
1202 sub machine_type_is_q35 {
1203     my ($conf) = @_;
1204
1205     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1206 }
1207
1208 sub print_tabletdevice_full {
1209     my ($conf) = @_;
1210
1211     my $q35 = machine_type_is_q35($conf);
1212
1213     # we use uhci for old VMs because tablet driver was buggy in older qemu
1214     my $usbbus = $q35 ? "ehci" : "uhci";
1215
1216     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1217 }
1218
1219 sub print_drivedevice_full {
1220     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1221
1222     my $device = '';
1223     my $maxdev = 0;
1224
1225     if ($drive->{interface} eq 'virtio') {
1226         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1227         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1228         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1229     } elsif ($drive->{interface} eq 'scsi') {
1230
1231         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1232         my $unit = $drive->{index} % $maxdev;
1233         my $devicetype = 'hd';
1234         my $path = '';
1235         if (drive_is_cdrom($drive)) {
1236             $devicetype = 'cd';
1237         } else {
1238             if ($drive->{file} =~ m|^/|) {
1239                 $path = $drive->{file};
1240                 if (my $info = path_is_scsi($path)) {
1241                     if ($info->{type} == 0) {
1242                         $devicetype = 'block';
1243                     } elsif ($info->{type} == 1) { # tape
1244                         $devicetype = 'generic';
1245                     }
1246                 }
1247             } else {
1248                  $path = PVE::Storage::path($storecfg, $drive->{file});
1249             }
1250
1251             if($path =~ m/^iscsi\:\/\//){
1252                 $devicetype = 'generic';
1253             }
1254         }
1255
1256         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1257             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1258         } else {
1259             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1260         }
1261
1262     } elsif ($drive->{interface} eq 'ide'){
1263         $maxdev = 2;
1264         my $controller = int($drive->{index} / $maxdev);
1265         my $unit = $drive->{index} % $maxdev;
1266         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1267
1268         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1269         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1270             $model = URI::Escape::uri_unescape($model);
1271             $device .= ",model=$model";
1272         }
1273     } elsif ($drive->{interface} eq 'sata'){
1274         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1275         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1276         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1277     } elsif ($drive->{interface} eq 'usb') {
1278         die "implement me";
1279         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1280     } else {
1281         die "unsupported interface type";
1282     }
1283
1284     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1285
1286     return $device;
1287 }
1288
1289 sub get_initiator_name {
1290     my $initiator;
1291
1292     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1293     while (defined(my $line = <$fh>)) {
1294         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1295         $initiator = $1;
1296         last;
1297     }
1298     $fh->close();
1299
1300     return $initiator;
1301 }
1302
1303 sub print_drive_full {
1304     my ($storecfg, $vmid, $drive) = @_;
1305
1306     my $path;
1307     my $volid = $drive->{file};
1308     my $format;
1309
1310     if (drive_is_cdrom($drive)) {
1311         $path = get_iso_path($storecfg, $vmid, $volid);
1312     } else {
1313         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1314         if ($storeid) {
1315             $path = PVE::Storage::path($storecfg, $volid);
1316             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1317             $format = qemu_img_format($scfg, $volname);
1318         } else {
1319             $path = $volid;
1320             $format = "raw";
1321         }
1322    }
1323
1324     my $opts = '';
1325     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1326     foreach my $o (@qemu_drive_options) {
1327         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1328     }
1329     if (my $serial = $drive->{serial}) {
1330         $serial = URI::Escape::uri_unescape($serial);
1331         $opts .= ",serial=$serial";
1332     }
1333
1334     $opts .= ",format=$format" if $format && !$drive->{format};
1335
1336     foreach my $o (qw(bps bps_rd bps_wr)) {
1337         my $v = $drive->{"m$o"};
1338         $opts .= ",$o=" . int($v*1024*1024) if $v;
1339     }
1340
1341     my $cache_direct = 0;
1342
1343     if (my $cache = $drive->{cache}) {
1344         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1345     } elsif (!drive_is_cdrom($drive)) {
1346         $opts .= ",cache=none";
1347         $cache_direct = 1;
1348     }
1349
1350     # aio native works only with O_DIRECT
1351     if (!$drive->{aio}) {
1352         if($cache_direct) {
1353             $opts .= ",aio=native";
1354         } else {
1355             $opts .= ",aio=threads";
1356         }
1357     }
1358
1359     if (!drive_is_cdrom($drive)) {
1360         my $detectzeroes;
1361         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1362             $detectzeroes = 'off';
1363         } elsif ($drive->{discard}) {
1364             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1365         } else {
1366             # This used to be our default with discard not being specified:
1367             $detectzeroes = 'on';
1368         }
1369         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1370     }
1371
1372     my $pathinfo = $path ? "file=$path," : '';
1373
1374     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1375 }
1376
1377 sub print_netdevice_full {
1378     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1379
1380     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1381
1382     my $device = $net->{model};
1383     if ($net->{model} eq 'virtio') {
1384          $device = 'virtio-net-pci';
1385      };
1386
1387     my $pciaddr = print_pci_addr("$netid", $bridges);
1388     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1389     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1390         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1391         my $vectors = $net->{queues} * 2 + 2;
1392         $tmpstr .= ",vectors=$vectors,mq=on";
1393     }
1394     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1395
1396     if ($use_old_bios_files) {
1397         my $romfile;
1398         if ($device eq 'virtio-net-pci') {
1399             $romfile = 'pxe-virtio.rom';
1400         } elsif ($device eq 'e1000') {
1401             $romfile = 'pxe-e1000.rom';
1402         } elsif ($device eq 'ne2k') {
1403             $romfile = 'pxe-ne2k_pci.rom';
1404         } elsif ($device eq 'pcnet') {
1405             $romfile = 'pxe-pcnet.rom';
1406         } elsif ($device eq 'rtl8139') {
1407             $romfile = 'pxe-rtl8139.rom';
1408         }
1409         $tmpstr .= ",romfile=$romfile" if $romfile;
1410     }
1411
1412     return $tmpstr;
1413 }
1414
1415 sub print_netdev_full {
1416     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1417
1418     my $i = '';
1419     if ($netid =~ m/^net(\d+)$/) {
1420         $i = int($1);
1421     }
1422
1423     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1424
1425     my $ifname = "tap${vmid}i$i";
1426
1427     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1428     die "interface name '$ifname' is too long (max 15 character)\n"
1429         if length($ifname) >= 16;
1430
1431     my $vhostparam = '';
1432     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1433
1434     my $vmname = $conf->{name} || "vm$vmid";
1435
1436     my $netdev = "";
1437     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1438
1439     if ($net->{bridge}) {
1440         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1441     } else {
1442         $netdev = "type=user,id=$netid,hostname=$vmname";
1443     }
1444
1445     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1446
1447     return $netdev;
1448 }
1449
1450 sub drive_is_cdrom {
1451     my ($drive) = @_;
1452
1453     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1454
1455 }
1456
1457 sub parse_numa {
1458     my ($data) = @_;
1459
1460     my $res = {};
1461
1462     foreach my $kvp (split(/,/, $data)) {
1463
1464         if ($kvp =~ m/^memory=(\S+)$/) {
1465             $res->{memory} = $1;
1466         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1467             $res->{policy} = $1;
1468         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1469             $res->{cpus}->{start} = $1;
1470             $res->{cpus}->{end} = $3;
1471         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1472             $res->{hostnodes}->{start} = $1;
1473             $res->{hostnodes}->{end} = $3;
1474         } else {
1475             return undef;
1476         }
1477     }
1478
1479     return $res;
1480 }
1481
1482 sub parse_hostpci {
1483     my ($value) = @_;
1484
1485     return undef if !$value;
1486
1487
1488     my @list = split(/,/, $value);
1489     my $found;
1490
1491     my $res = {};
1492     foreach my $kv (@list) {
1493
1494         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1495             $found = 1;
1496             if(defined($4)){
1497                 push @{$res->{pciid}}, { id => $2 , function => $4};
1498
1499             }else{
1500                 my $pcidevices = lspci($2);
1501                 $res->{pciid} = $pcidevices->{$2};
1502             }
1503         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1504             $res->{rombar} = $1;
1505         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1506             $res->{'x-vga'} = $1;
1507         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1508             $res->{pcie} = 1 if $1 == 1;
1509         } else {
1510             warn "unknown hostpci setting '$kv'\n";
1511         }
1512     }
1513
1514     return undef if !$found;
1515
1516     return $res;
1517 }
1518
1519 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1520 sub parse_net {
1521     my ($data) = @_;
1522
1523     my $res = {};
1524
1525     foreach my $kvp (split(/,/, $data)) {
1526
1527         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1528             my $model = lc($1);
1529             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1530             $res->{model} = $model;
1531             $res->{macaddr} = $mac;
1532         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1533             $res->{bridge} = $1;
1534         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1535             $res->{queues} = $1;
1536         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1537             $res->{rate} = $1;
1538         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1539             $res->{tag} = $1;
1540         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1541             $res->{trunks} = $1;
1542         } elsif ($kvp =~ m/^firewall=([01])$/) {
1543             $res->{firewall} = $1;
1544         } elsif ($kvp =~ m/^link_down=([01])$/) {
1545             $res->{link_down} = $1;
1546         } else {
1547             return undef;
1548         }
1549
1550     }
1551
1552     return undef if !$res->{model};
1553
1554     return $res;
1555 }
1556
1557 sub print_net {
1558     my $net = shift;
1559
1560     my $res = "$net->{model}";
1561     $res .= "=$net->{macaddr}" if $net->{macaddr};
1562     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1563     $res .= ",rate=$net->{rate}" if $net->{rate};
1564     $res .= ",tag=$net->{tag}" if $net->{tag};
1565     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1566     $res .= ",firewall=1" if $net->{firewall};
1567     $res .= ",link_down=1" if $net->{link_down};
1568     $res .= ",queues=$net->{queues}" if $net->{queues};
1569
1570     return $res;
1571 }
1572
1573 sub add_random_macs {
1574     my ($settings) = @_;
1575
1576     foreach my $opt (keys %$settings) {
1577         next if $opt !~ m/^net(\d+)$/;
1578         my $net = parse_net($settings->{$opt});
1579         next if !$net;
1580         $settings->{$opt} = print_net($net);
1581     }
1582 }
1583
1584 sub vm_is_volid_owner {
1585     my ($storecfg, $vmid, $volid) = @_;
1586
1587     if ($volid !~  m|^/|) {
1588         my ($path, $owner);
1589         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1590         if ($owner && ($owner == $vmid)) {
1591             return 1;
1592         }
1593     }
1594
1595     return undef;
1596 }
1597
1598 sub split_flagged_list {
1599     my $text = shift || '';
1600     $text =~ s/[,;]/ /g;
1601     $text =~ s/^\s+//;
1602     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1603 }
1604
1605 sub join_flagged_list {
1606     my ($how, $lst) = @_;
1607     join $how, map { $lst->{$_} . $_ } keys %$lst;
1608 }
1609
1610 sub vmconfig_delete_pending_option {
1611     my ($conf, $key, $force) = @_;
1612
1613     delete $conf->{pending}->{$key};
1614     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1615     $pending_delete_hash->{$key} = $force ? '!' : '';
1616     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1617 }
1618
1619 sub vmconfig_undelete_pending_option {
1620     my ($conf, $key) = @_;
1621
1622     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1623     delete $pending_delete_hash->{$key};
1624
1625     if (%$pending_delete_hash) {
1626         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1627     } else {
1628         delete $conf->{pending}->{delete};
1629     }
1630 }
1631
1632 sub vmconfig_register_unused_drive {
1633     my ($storecfg, $vmid, $conf, $drive) = @_;
1634
1635     if (!drive_is_cdrom($drive)) {
1636         my $volid = $drive->{file};
1637         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1638             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1639         }
1640     }
1641 }
1642
1643 sub vmconfig_cleanup_pending {
1644     my ($conf) = @_;
1645
1646     # remove pending changes when nothing changed
1647     my $changes;
1648     foreach my $opt (keys %{$conf->{pending}}) {
1649         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1650             $changes = 1;
1651             delete $conf->{pending}->{$opt};
1652         }
1653     }
1654
1655     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1656     my $pending_delete_hash = {};
1657     while (my ($opt, $force) = each %$current_delete_hash) {
1658         if (defined($conf->{$opt})) {
1659             $pending_delete_hash->{$opt} = $force;
1660         } else {
1661             $changes = 1;
1662         }
1663     }
1664
1665     if (%$pending_delete_hash) {
1666         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1667     } else {
1668         delete $conf->{pending}->{delete};
1669     }
1670
1671     return $changes;
1672 }
1673
1674 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1675 my $smbios1_desc = {
1676     uuid => {
1677         type => 'string',
1678         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1679         format_description => 'UUID',
1680         optional => 1,
1681     },
1682     version => {
1683         type => 'string',
1684         pattern => '\S+',
1685         format_description => 'str',
1686         optional => 1,
1687     },
1688     serial => {
1689         type => 'string',
1690         pattern => '\S+',
1691         format_description => 'str',
1692         optional => 1,
1693     },
1694     manufacturer => {
1695         type => 'string',
1696         pattern => '\S+',
1697         format_description => 'name',
1698         optional => 1,
1699     },
1700     product => {
1701         type => 'string',
1702         pattern => '\S+',
1703         format_description => 'name',
1704         optional => 1,
1705     },
1706     sku => {
1707         type => 'string',
1708         pattern => '\S+',
1709         format_description => 'str',
1710         optional => 1,
1711     },
1712     family => {
1713         type => 'string',
1714         pattern => '\S+',
1715         format_description => 'str',
1716         optional => 1,
1717     },
1718 };
1719
1720 sub parse_smbios1 {
1721     my ($data) = @_;
1722
1723     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1724     warn $@ if $@;
1725     return $res;
1726 }
1727
1728 sub print_smbios1 {
1729     my ($smbios1) = @_;
1730     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1731 }
1732
1733 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1734
1735 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1736 sub verify_bootdisk {
1737     my ($value, $noerr) = @_;
1738
1739     return $value if is_valid_drivename($value);
1740
1741     return undef if $noerr;
1742
1743     die "invalid boot disk '$value'\n";
1744 }
1745
1746 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1747 sub verify_numa {
1748     my ($value, $noerr) = @_;
1749
1750     return $value if parse_numa($value);
1751
1752     return undef if $noerr;
1753
1754     die "unable to parse numa options\n";
1755 }
1756
1757 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1758 sub verify_net {
1759     my ($value, $noerr) = @_;
1760
1761     return $value if parse_net($value);
1762
1763     return undef if $noerr;
1764
1765     die "unable to parse network options\n";
1766 }
1767
1768 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1769 sub verify_hostpci {
1770     my ($value, $noerr) = @_;
1771
1772     return $value if parse_hostpci($value);
1773
1774     return undef if $noerr;
1775
1776     die "unable to parse pci id\n";
1777 }
1778
1779 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1780 sub verify_watchdog {
1781     my ($value, $noerr) = @_;
1782
1783     return $value if parse_watchdog($value);
1784
1785     return undef if $noerr;
1786
1787     die "unable to parse watchdog options\n";
1788 }
1789
1790 sub parse_watchdog {
1791     my ($value) = @_;
1792
1793     return undef if !$value;
1794
1795     my $res = {};
1796
1797     foreach my $p (split(/,/, $value)) {
1798         next if $p =~ m/^\s*$/;
1799
1800         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1801             $res->{model} = $2;
1802         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1803             $res->{action} = $2;
1804         } else {
1805             return undef;
1806         }
1807     }
1808
1809     return $res;
1810 }
1811
1812 sub parse_usb_device {
1813     my ($value) = @_;
1814
1815     return undef if !$value;
1816
1817     my $res = {};
1818     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1819         $res->{vendorid} = $2;
1820         $res->{productid} = $4;
1821     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1822         $res->{hostbus} = $1;
1823         $res->{hostport} = $2;
1824     } elsif ($value =~ m/^spice$/i) {
1825         $res->{spice} = 1;
1826     } else {
1827         return undef;
1828     }
1829
1830     return $res;
1831 }
1832
1833 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1834 sub verify_usb_device {
1835     my ($value, $noerr) = @_;
1836
1837     return $value if parse_usb_device($value);
1838
1839     return undef if $noerr;
1840
1841     die "unable to parse usb device\n";
1842 }
1843
1844 # add JSON properties for create and set function
1845 sub json_config_properties {
1846     my $prop = shift;
1847
1848     foreach my $opt (keys %$confdesc) {
1849         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1850         $prop->{$opt} = $confdesc->{$opt};
1851     }
1852
1853     return $prop;
1854 }
1855
1856 sub check_type {
1857     my ($key, $value) = @_;
1858
1859     die "unknown setting '$key'\n" if !$confdesc->{$key};
1860
1861     my $type = $confdesc->{$key}->{type};
1862
1863     if (!defined($value)) {
1864         die "got undefined value\n";
1865     }
1866
1867     if ($value =~ m/[\n\r]/) {
1868         die "property contains a line feed\n";
1869     }
1870
1871     if ($type eq 'boolean') {
1872         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1873         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1874         die "type check ('boolean') failed - got '$value'\n";
1875     } elsif ($type eq 'integer') {
1876         return int($1) if $value =~ m/^(\d+)$/;
1877         die "type check ('integer') failed - got '$value'\n";
1878     } elsif ($type eq 'number') {
1879         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1880         die "type check ('number') failed - got '$value'\n";
1881     } elsif ($type eq 'string') {
1882         if (my $fmt = $confdesc->{$key}->{format}) {
1883             if ($fmt eq 'pve-qm-drive') {
1884                 # special case - we need to pass $key to parse_drive()
1885                 my $drive = parse_drive($key, $value);
1886                 return $value if $drive;
1887                 die "unable to parse drive options\n";
1888             }
1889             PVE::JSONSchema::check_format($fmt, $value);
1890             return $value;
1891         }
1892         $value =~ s/^\"(.*)\"$/$1/;
1893         return $value;
1894     } else {
1895         die "internal error"
1896     }
1897 }
1898
1899 sub check_iommu_support{
1900     #fixme : need to check IOMMU support
1901     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1902
1903     my $iommu=1;
1904     return $iommu;
1905
1906 }
1907
1908 sub touch_config {
1909     my ($vmid) = @_;
1910
1911     my $conf = PVE::QemuConfig->config_file($vmid);
1912     utime undef, undef, $conf;
1913 }
1914
1915 sub destroy_vm {
1916     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1917
1918     my $conffile = PVE::QemuConfig->config_file($vmid);
1919
1920     my $conf = PVE::QemuConfig->load_config($vmid);
1921
1922     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1923
1924     # only remove disks owned by this VM
1925     foreach_drive($conf, sub {
1926         my ($ds, $drive) = @_;
1927
1928         return if drive_is_cdrom($drive);
1929
1930         my $volid = $drive->{file};
1931
1932         return if !$volid || $volid =~ m|^/|;
1933
1934         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1935         return if !$path || !$owner || ($owner != $vmid);
1936
1937         PVE::Storage::vdisk_free($storecfg, $volid);
1938     });
1939
1940     if ($keep_empty_config) {
1941         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1942     } else {
1943         unlink $conffile;
1944     }
1945
1946     # also remove unused disk
1947     eval {
1948         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1949
1950         eval {
1951             PVE::Storage::foreach_volid($dl, sub {
1952                 my ($volid, $sid, $volname, $d) = @_;
1953                 PVE::Storage::vdisk_free($storecfg, $volid);
1954             });
1955         };
1956         warn $@ if $@;
1957
1958     };
1959     warn $@ if $@;
1960 }
1961
1962 sub parse_vm_config {
1963     my ($filename, $raw) = @_;
1964
1965     return undef if !defined($raw);
1966
1967     my $res = {
1968         digest => Digest::SHA::sha1_hex($raw),
1969         snapshots => {},
1970         pending => {},
1971     };
1972
1973     $filename =~ m|/qemu-server/(\d+)\.conf$|
1974         || die "got strange filename '$filename'";
1975
1976     my $vmid = $1;
1977
1978     my $conf = $res;
1979     my $descr;
1980     my $section = '';
1981
1982     my @lines = split(/\n/, $raw);
1983     foreach my $line (@lines) {
1984         next if $line =~ m/^\s*$/;
1985
1986         if ($line =~ m/^\[PENDING\]\s*$/i) {
1987             $section = 'pending';
1988             if (defined($descr)) {
1989                 $descr =~ s/\s+$//;
1990                 $conf->{description} = $descr;
1991             }
1992             $descr = undef;
1993             $conf = $res->{$section} = {};
1994             next;
1995
1996         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1997             $section = $1;
1998             if (defined($descr)) {
1999                 $descr =~ s/\s+$//;
2000                 $conf->{description} = $descr;
2001             }
2002             $descr = undef;
2003             $conf = $res->{snapshots}->{$section} = {};
2004             next;
2005         }
2006
2007         if ($line =~ m/^\#(.*)\s*$/) {
2008             $descr = '' if !defined($descr);
2009             $descr .= PVE::Tools::decode_text($1) . "\n";
2010             next;
2011         }
2012
2013         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2014             $descr = '' if !defined($descr);
2015             $descr .= PVE::Tools::decode_text($2);
2016         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2017             $conf->{snapstate} = $1;
2018         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2019             my $key = $1;
2020             my $value = $2;
2021             $conf->{$key} = $value;
2022         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2023             my $value = $1;
2024             if ($section eq 'pending') {
2025                 $conf->{delete} = $value; # we parse this later
2026             } else {
2027                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2028             }
2029         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2030             my $key = $1;
2031             my $value = $2;
2032             eval { $value = check_type($key, $value); };
2033             if ($@) {
2034                 warn "vm $vmid - unable to parse value of '$key' - $@";
2035             } else {
2036                 my $fmt = $confdesc->{$key}->{format};
2037                 if ($fmt && $fmt eq 'pve-qm-drive') {
2038                     my $v = parse_drive($key, $value);
2039                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2040                         $v->{file} = $volid;
2041                         $value = print_drive($vmid, $v);
2042                     } else {
2043                         warn "vm $vmid - unable to parse value of '$key'\n";
2044                         next;
2045                     }
2046                 }
2047
2048                 if ($key eq 'cdrom') {
2049                     $conf->{ide2} = $value;
2050                 } else {
2051                     $conf->{$key} = $value;
2052                 }
2053             }
2054         }
2055     }
2056
2057     if (defined($descr)) {
2058         $descr =~ s/\s+$//;
2059         $conf->{description} = $descr;
2060     }
2061     delete $res->{snapstate}; # just to be sure
2062
2063     return $res;
2064 }
2065
2066 sub write_vm_config {
2067     my ($filename, $conf) = @_;
2068
2069     delete $conf->{snapstate}; # just to be sure
2070
2071     if ($conf->{cdrom}) {
2072         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2073         $conf->{ide2} = $conf->{cdrom};
2074         delete $conf->{cdrom};
2075     }
2076
2077     # we do not use 'smp' any longer
2078     if ($conf->{sockets}) {
2079         delete $conf->{smp};
2080     } elsif ($conf->{smp}) {
2081         $conf->{sockets} = $conf->{smp};
2082         delete $conf->{cores};
2083         delete $conf->{smp};
2084     }
2085
2086     my $used_volids = {};
2087
2088     my $cleanup_config = sub {
2089         my ($cref, $pending, $snapname) = @_;
2090
2091         foreach my $key (keys %$cref) {
2092             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2093                 $key eq 'snapstate' || $key eq 'pending';
2094             my $value = $cref->{$key};
2095             if ($key eq 'delete') {
2096                 die "propertry 'delete' is only allowed in [PENDING]\n"
2097                     if !$pending;
2098                 # fixme: check syntax?
2099                 next;
2100             }
2101             eval { $value = check_type($key, $value); };
2102             die "unable to parse value of '$key' - $@" if $@;
2103
2104             $cref->{$key} = $value;
2105
2106             if (!$snapname && is_valid_drivename($key)) {
2107                 my $drive = parse_drive($key, $value);
2108                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2109             }
2110         }
2111     };
2112
2113     &$cleanup_config($conf);
2114
2115     &$cleanup_config($conf->{pending}, 1);
2116
2117     foreach my $snapname (keys %{$conf->{snapshots}}) {
2118         die "internal error" if $snapname eq 'pending';
2119         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2120     }
2121
2122     # remove 'unusedX' settings if we re-add a volume
2123     foreach my $key (keys %$conf) {
2124         my $value = $conf->{$key};
2125         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2126             delete $conf->{$key};
2127         }
2128     }
2129
2130     my $generate_raw_config = sub {
2131         my ($conf, $pending) = @_;
2132
2133         my $raw = '';
2134
2135         # add description as comment to top of file
2136         if (defined(my $descr = $conf->{description})) {
2137             if ($descr) {
2138                 foreach my $cl (split(/\n/, $descr)) {
2139                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2140                 }
2141             } else {
2142                 $raw .= "#\n" if $pending;
2143             }
2144         }
2145
2146         foreach my $key (sort keys %$conf) {
2147             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2148             $raw .= "$key: $conf->{$key}\n";
2149         }
2150         return $raw;
2151     };
2152
2153     my $raw = &$generate_raw_config($conf);
2154
2155     if (scalar(keys %{$conf->{pending}})){
2156         $raw .= "\n[PENDING]\n";
2157         $raw .= &$generate_raw_config($conf->{pending}, 1);
2158     }
2159
2160     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2161         $raw .= "\n[$snapname]\n";
2162         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2163     }
2164
2165     return $raw;
2166 }
2167
2168 sub load_defaults {
2169
2170     my $res = {};
2171
2172     # we use static defaults from our JSON schema configuration
2173     foreach my $key (keys %$confdesc) {
2174         if (defined(my $default = $confdesc->{$key}->{default})) {
2175             $res->{$key} = $default;
2176         }
2177     }
2178
2179     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2180     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2181
2182     return $res;
2183 }
2184
2185 sub config_list {
2186     my $vmlist = PVE::Cluster::get_vmlist();
2187     my $res = {};
2188     return $res if !$vmlist || !$vmlist->{ids};
2189     my $ids = $vmlist->{ids};
2190
2191     foreach my $vmid (keys %$ids) {
2192         my $d = $ids->{$vmid};
2193         next if !$d->{node} || $d->{node} ne $nodename;
2194         next if !$d->{type} || $d->{type} ne 'qemu';
2195         $res->{$vmid}->{exists} = 1;
2196     }
2197     return $res;
2198 }
2199
2200 # test if VM uses local resources (to prevent migration)
2201 sub check_local_resources {
2202     my ($conf, $noerr) = @_;
2203
2204     my $loc_res = 0;
2205
2206     $loc_res = 1 if $conf->{hostusb}; # old syntax
2207     $loc_res = 1 if $conf->{hostpci}; # old syntax
2208
2209     foreach my $k (keys %$conf) {
2210         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2211         # sockets are safe: they will recreated be on the target side post-migrate
2212         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2213         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2214     }
2215
2216     die "VM uses local resources\n" if $loc_res && !$noerr;
2217
2218     return $loc_res;
2219 }
2220
2221 # check if used storages are available on all nodes (use by migrate)
2222 sub check_storage_availability {
2223     my ($storecfg, $conf, $node) = @_;
2224
2225     foreach_drive($conf, sub {
2226         my ($ds, $drive) = @_;
2227
2228         my $volid = $drive->{file};
2229         return if !$volid;
2230
2231         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2232         return if !$sid;
2233
2234         # check if storage is available on both nodes
2235         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2236         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2237    });
2238 }
2239
2240 # list nodes where all VM images are available (used by has_feature API)
2241 sub shared_nodes {
2242     my ($conf, $storecfg) = @_;
2243
2244     my $nodelist = PVE::Cluster::get_nodelist();
2245     my $nodehash = { map { $_ => 1 } @$nodelist };
2246     my $nodename = PVE::INotify::nodename();
2247
2248     foreach_drive($conf, sub {
2249         my ($ds, $drive) = @_;
2250
2251         my $volid = $drive->{file};
2252         return if !$volid;
2253
2254         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2255         if ($storeid) {
2256             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2257             if ($scfg->{disable}) {
2258                 $nodehash = {};
2259             } elsif (my $avail = $scfg->{nodes}) {
2260                 foreach my $node (keys %$nodehash) {
2261                     delete $nodehash->{$node} if !$avail->{$node};
2262                 }
2263             } elsif (!$scfg->{shared}) {
2264                 foreach my $node (keys %$nodehash) {
2265                     delete $nodehash->{$node} if $node ne $nodename
2266                 }
2267             }
2268         }
2269     });
2270
2271     return $nodehash
2272 }
2273
2274 sub check_cmdline {
2275     my ($pidfile, $pid) = @_;
2276
2277     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2278     if (defined($fh)) {
2279         my $line = <$fh>;
2280         $fh->close;
2281         return undef if !$line;
2282         my @param = split(/\0/, $line);
2283
2284         my $cmd = $param[0];
2285         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2286
2287         for (my $i = 0; $i < scalar (@param); $i++) {
2288             my $p = $param[$i];
2289             next if !$p;
2290             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2291                 my $p = $param[$i+1];
2292                 return 1 if $p && ($p eq $pidfile);
2293                 return undef;
2294             }
2295         }
2296     }
2297     return undef;
2298 }
2299
2300 sub check_running {
2301     my ($vmid, $nocheck, $node) = @_;
2302
2303     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2304
2305     die "unable to find configuration file for VM $vmid - no such machine\n"
2306         if !$nocheck && ! -f $filename;
2307
2308     my $pidfile = pidfile_name($vmid);
2309
2310     if (my $fd = IO::File->new("<$pidfile")) {
2311         my $st = stat($fd);
2312         my $line = <$fd>;
2313         close($fd);
2314
2315         my $mtime = $st->mtime;
2316         if ($mtime > time()) {
2317             warn "file '$filename' modified in future\n";
2318         }
2319
2320         if ($line =~ m/^(\d+)$/) {
2321             my $pid = $1;
2322             if (check_cmdline($pidfile, $pid)) {
2323                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2324                     return $pid;
2325                 }
2326             }
2327         }
2328     }
2329
2330     return undef;
2331 }
2332
2333 sub vzlist {
2334
2335     my $vzlist = config_list();
2336
2337     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2338
2339     while (defined(my $de = $fd->read)) {
2340         next if $de !~ m/^(\d+)\.pid$/;
2341         my $vmid = $1;
2342         next if !defined($vzlist->{$vmid});
2343         if (my $pid = check_running($vmid)) {
2344             $vzlist->{$vmid}->{pid} = $pid;
2345         }
2346     }
2347
2348     return $vzlist;
2349 }
2350
2351 sub disksize {
2352     my ($storecfg, $conf) = @_;
2353
2354     my $bootdisk = $conf->{bootdisk};
2355     return undef if !$bootdisk;
2356     return undef if !is_valid_drivename($bootdisk);
2357
2358     return undef if !$conf->{$bootdisk};
2359
2360     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2361     return undef if !defined($drive);
2362
2363     return undef if drive_is_cdrom($drive);
2364
2365     my $volid = $drive->{file};
2366     return undef if !$volid;
2367
2368     return $drive->{size};
2369 }
2370
2371 my $last_proc_pid_stat;
2372
2373 # get VM status information
2374 # This must be fast and should not block ($full == false)
2375 # We only query KVM using QMP if $full == true (this can be slow)
2376 sub vmstatus {
2377     my ($opt_vmid, $full) = @_;
2378
2379     my $res = {};
2380
2381     my $storecfg = PVE::Storage::config();
2382
2383     my $list = vzlist();
2384     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2385
2386     my $cpucount = $cpuinfo->{cpus} || 1;
2387
2388     foreach my $vmid (keys %$list) {
2389         next if $opt_vmid && ($vmid ne $opt_vmid);
2390
2391         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2392         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2393
2394         my $d = {};
2395         $d->{pid} = $list->{$vmid}->{pid};
2396
2397         # fixme: better status?
2398         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2399
2400         my $size = disksize($storecfg, $conf);
2401         if (defined($size)) {
2402             $d->{disk} = 0; # no info available
2403             $d->{maxdisk} = $size;
2404         } else {
2405             $d->{disk} = 0;
2406             $d->{maxdisk} = 0;
2407         }
2408
2409         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2410         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2411         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2412
2413         $d->{name} = $conf->{name} || "VM $vmid";
2414         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2415
2416         if ($conf->{balloon}) {
2417             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2418             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2419         }
2420
2421         $d->{uptime} = 0;
2422         $d->{cpu} = 0;
2423         $d->{mem} = 0;
2424
2425         $d->{netout} = 0;
2426         $d->{netin} = 0;
2427
2428         $d->{diskread} = 0;
2429         $d->{diskwrite} = 0;
2430
2431         $d->{template} = PVE::QemuConfig->is_template($conf);
2432
2433         $res->{$vmid} = $d;
2434     }
2435
2436     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2437     foreach my $dev (keys %$netdev) {
2438         next if $dev !~ m/^tap([1-9]\d*)i/;
2439         my $vmid = $1;
2440         my $d = $res->{$vmid};
2441         next if !$d;
2442
2443         $d->{netout} += $netdev->{$dev}->{receive};
2444         $d->{netin} += $netdev->{$dev}->{transmit};
2445
2446         if ($full) {
2447             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2448             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2449         }
2450
2451     }
2452
2453     my $ctime = gettimeofday;
2454
2455     foreach my $vmid (keys %$list) {
2456
2457         my $d = $res->{$vmid};
2458         my $pid = $d->{pid};
2459         next if !$pid;
2460
2461         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2462         next if !$pstat; # not running
2463
2464         my $used = $pstat->{utime} + $pstat->{stime};
2465
2466         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2467
2468         if ($pstat->{vsize}) {
2469             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2470         }
2471
2472         my $old = $last_proc_pid_stat->{$pid};
2473         if (!$old) {
2474             $last_proc_pid_stat->{$pid} = {
2475                 time => $ctime,
2476                 used => $used,
2477                 cpu => 0,
2478             };
2479             next;
2480         }
2481
2482         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2483
2484         if ($dtime > 1000) {
2485             my $dutime = $used -  $old->{used};
2486
2487             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2488             $last_proc_pid_stat->{$pid} = {
2489                 time => $ctime,
2490                 used => $used,
2491                 cpu => $d->{cpu},
2492             };
2493         } else {
2494             $d->{cpu} = $old->{cpu};
2495         }
2496     }
2497
2498     return $res if !$full;
2499
2500     my $qmpclient = PVE::QMPClient->new();
2501
2502     my $ballooncb = sub {
2503         my ($vmid, $resp) = @_;
2504
2505         my $info = $resp->{'return'};
2506         return if !$info->{max_mem};
2507
2508         my $d = $res->{$vmid};
2509
2510         # use memory assigned to VM
2511         $d->{maxmem} = $info->{max_mem};
2512         $d->{balloon} = $info->{actual};
2513
2514         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2515             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2516             $d->{freemem} = $info->{free_mem};
2517         }
2518
2519         $d->{ballooninfo} = $info;
2520     };
2521
2522     my $blockstatscb = sub {
2523         my ($vmid, $resp) = @_;
2524         my $data = $resp->{'return'} || [];
2525         my $totalrdbytes = 0;
2526         my $totalwrbytes = 0;
2527
2528         for my $blockstat (@$data) {
2529             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2530             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2531
2532             $blockstat->{device} =~ s/drive-//;
2533             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2534         }
2535         $res->{$vmid}->{diskread} = $totalrdbytes;
2536         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2537     };
2538
2539     my $statuscb = sub {
2540         my ($vmid, $resp) = @_;
2541
2542         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2543         # this fails if ballon driver is not loaded, so this must be
2544         # the last commnand (following command are aborted if this fails).
2545         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2546
2547         my $status = 'unknown';
2548         if (!defined($status = $resp->{'return'}->{status})) {
2549             warn "unable to get VM status\n";
2550             return;
2551         }
2552
2553         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2554     };
2555
2556     foreach my $vmid (keys %$list) {
2557         next if $opt_vmid && ($vmid ne $opt_vmid);
2558         next if !$res->{$vmid}->{pid}; # not running
2559         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2560     }
2561
2562     $qmpclient->queue_execute(undef, 1);
2563
2564     foreach my $vmid (keys %$list) {
2565         next if $opt_vmid && ($vmid ne $opt_vmid);
2566         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2567     }
2568
2569     return $res;
2570 }
2571
2572 sub foreach_dimm {
2573     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2574
2575     my $dimm_id = 0;
2576     my $current_size = 1024;
2577     my $dimm_size = 512;
2578     return if $current_size == $memory;
2579
2580     for (my $j = 0; $j < 8; $j++) {
2581         for (my $i = 0; $i < 32; $i++) {
2582             my $name = "dimm${dimm_id}";
2583             $dimm_id++;
2584             my $numanode = $i % $sockets;
2585             $current_size += $dimm_size;
2586             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2587             return  $current_size if $current_size >= $memory;
2588         }
2589         $dimm_size *= 2;
2590     }
2591 }
2592
2593 sub foreach_reverse_dimm {
2594     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2595
2596     my $dimm_id = 253;
2597     my $current_size = 4177920;
2598     my $dimm_size = 65536;
2599     return if $current_size == $memory;
2600
2601     for (my $j = 0; $j < 8; $j++) {
2602         for (my $i = 0; $i < 32; $i++) {
2603             my $name = "dimm${dimm_id}";
2604             $dimm_id--;
2605             my $numanode = $i % $sockets;
2606             $current_size -= $dimm_size;
2607             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2608             return  $current_size if $current_size <= $memory;
2609         }
2610         $dimm_size /= 2;
2611     }
2612 }
2613
2614 sub foreach_drive {
2615     my ($conf, $func) = @_;
2616
2617     foreach my $ds (valid_drive_names()) {
2618         next if !defined($conf->{$ds});
2619
2620         my $drive = parse_drive($ds, $conf->{$ds});
2621         next if !$drive;
2622
2623         &$func($ds, $drive);
2624     }
2625 }
2626
2627 sub foreach_volid {
2628     my ($conf, $func) = @_;
2629
2630     my $volhash = {};
2631
2632     my $test_volid = sub {
2633         my ($volid, $is_cdrom) = @_;
2634
2635         return if !$volid;
2636
2637         $volhash->{$volid} = $is_cdrom || 0;
2638     };
2639
2640     foreach_drive($conf, sub {
2641         my ($ds, $drive) = @_;
2642         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2643     });
2644
2645     foreach my $snapname (keys %{$conf->{snapshots}}) {
2646         my $snap = $conf->{snapshots}->{$snapname};
2647         &$test_volid($snap->{vmstate}, 0);
2648         foreach_drive($snap, sub {
2649             my ($ds, $drive) = @_;
2650             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2651         });
2652     }
2653
2654     foreach my $volid (keys %$volhash) {
2655         &$func($volid, $volhash->{$volid});
2656     }
2657 }
2658
2659 sub vga_conf_has_spice {
2660     my ($vga) = @_;
2661
2662     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2663
2664     return $1 || 1;
2665 }
2666
2667 sub config_to_command {
2668     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2669
2670     my $cmd = [];
2671     my $globalFlags = [];
2672     my $machineFlags = [];
2673     my $rtcFlags = [];
2674     my $cpuFlags = [];
2675     my $devices = [];
2676     my $pciaddr = '';
2677     my $bridges = {};
2678     my $kvmver = kvm_user_version();
2679     my $vernum = 0; # unknown
2680     my $ostype = $conf->{ostype};
2681     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2682         $vernum = $1*1000000+$2*1000;
2683     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2684         $vernum = $1*1000000+$2*1000+$3;
2685     }
2686
2687     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2688
2689     my $have_ovz = -f '/proc/vz/vestat';
2690
2691     my $q35 = machine_type_is_q35($conf);
2692     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2693     my $machine_type = $forcemachine || $conf->{machine};
2694     my $use_old_bios_files = undef;
2695     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2696
2697     my $cpuunits = defined($conf->{cpuunits}) ?
2698             $conf->{cpuunits} : $defaults->{cpuunits};
2699
2700     push @$cmd, '/usr/bin/systemd-run';
2701     push @$cmd, '--scope';
2702     push @$cmd, '--slice', "qemu";
2703     push @$cmd, '--unit', $vmid;
2704     # set KillMode=none, so that systemd don't kill those scopes
2705     # at shutdown (pve-manager service should stop the VMs instead)
2706     push @$cmd, '-p', "KillMode=none";
2707     push @$cmd, '-p', "CPUShares=$cpuunits";
2708     if ($conf->{cpulimit}) {
2709         my $cpulimit = int($conf->{cpulimit} * 100);
2710         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2711     }
2712
2713     push @$cmd, '/usr/bin/kvm';
2714
2715     push @$cmd, '-id', $vmid;
2716
2717     my $use_virtio = 0;
2718
2719     my $qmpsocket = qmp_socket($vmid);
2720     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2721     push @$cmd, '-mon', "chardev=qmp,mode=control";
2722
2723
2724     push @$cmd, '-pidfile' , pidfile_name($vmid);
2725
2726     push @$cmd, '-daemonize';
2727
2728     if ($conf->{smbios1}) {
2729         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2730     }
2731
2732     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2733         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2734         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2735         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2736         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2737         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2738         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2739     }
2740
2741     if ($q35) {
2742         # the q35 chipset support native usb2, so we enable usb controller
2743         # by default for this machine type
2744         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2745     } else {
2746         $pciaddr = print_pci_addr("piix3", $bridges);
2747         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2748
2749         my $use_usb2 = 0;
2750         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2751             next if !$conf->{"usb$i"};
2752             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2753             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2754             $use_usb2 = 1;
2755         }
2756         # include usb device config
2757         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2758     }
2759
2760     # add usb3 controller if needed
2761
2762     my $use_usb3 = 0;
2763     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2764         next if !$conf->{"usb$i"};
2765         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2766         next if !$d || !$d->{usb3};
2767         $use_usb3 = 1;
2768     }
2769
2770     $pciaddr = print_pci_addr("xhci", $bridges);
2771     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2772
2773     my $vga = $conf->{vga};
2774
2775     my $qxlnum = vga_conf_has_spice($vga);
2776     $vga = 'qxl' if $qxlnum;
2777
2778     if (!$vga) {
2779         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2780                                 $conf->{ostype} eq 'win7' ||
2781                                 $conf->{ostype} eq 'w2k8')) {
2782             $vga = 'std';
2783         } else {
2784             $vga = 'cirrus';
2785         }
2786     }
2787
2788     # enable absolute mouse coordinates (needed by vnc)
2789     my $tablet;
2790     if (defined($conf->{tablet})) {
2791         $tablet = $conf->{tablet};
2792     } else {
2793         $tablet = $defaults->{tablet};
2794         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2795         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2796     }
2797
2798     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2799
2800     my $kvm_off = 0;
2801     # host pci devices
2802     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2803         my $d = parse_hostpci($conf->{"hostpci$i"});
2804         next if !$d;
2805
2806         my $pcie = $d->{pcie};
2807         if($pcie){
2808             die "q35 machine model is not enabled" if !$q35;
2809             $pciaddr = print_pcie_addr("hostpci$i");
2810         }else{
2811             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2812         }
2813
2814         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2815         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2816         if ($xvga && $xvga ne '') {
2817             $kvm_off = 1;
2818             $vga = 'none';
2819             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2820                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2821             }
2822             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2823                 $xvga = "";
2824             }
2825         }
2826         my $pcidevices = $d->{pciid};
2827         my $multifunction = 1 if @$pcidevices > 1;
2828
2829         my $j=0;
2830         foreach my $pcidevice (@$pcidevices) {
2831
2832             my $id = "hostpci$i";
2833             $id .= ".$j" if $multifunction;
2834             my $addr = $pciaddr;
2835             $addr .= ".$j" if $multifunction;
2836             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2837
2838             if($j == 0){
2839                 $devicestr .= "$rombar$xvga";
2840                 $devicestr .= ",multifunction=on" if $multifunction;
2841             }
2842
2843             push @$devices, '-device', $devicestr;
2844             $j++;
2845         }
2846     }
2847
2848     # usb devices
2849     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2850         next if !$conf->{"usb$i"};
2851         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2852         next if !$d;
2853
2854         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2855         my $usbbus = '';
2856         if (defined($d->{usb3}) && $d->{usb3}) {
2857             $usbbus = ',bus=xhci.0';
2858         }
2859
2860         if (defined($d->{host})) {
2861             $d = parse_usb_device($d->{host});
2862             if (defined($d->{vendorid}) && defined($d->{productid})) {
2863                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2864             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2865                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2866             } elsif (defined($d->{spice}) && $d->{spice}) {
2867                 # usb redir support for spice, currently no usb3
2868                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2869                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2870             }
2871         }
2872     }
2873
2874     # serial devices
2875     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2876         if (my $path = $conf->{"serial$i"}) {
2877             if ($path eq 'socket') {
2878                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2879                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2880                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2881             } else {
2882                 die "no such serial device\n" if ! -c $path;
2883                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2884                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2885             }
2886         }
2887     }
2888
2889     # parallel devices
2890     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2891         if (my $path = $conf->{"parallel$i"}) {
2892             die "no such parallel device\n" if ! -c $path;
2893             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2894             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2895             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2896         }
2897     }
2898
2899     my $vmname = $conf->{name} || "vm$vmid";
2900
2901     push @$cmd, '-name', $vmname;
2902
2903     my $sockets = 1;
2904     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2905     $sockets = $conf->{sockets} if  $conf->{sockets};
2906
2907     my $cores = $conf->{cores} || 1;
2908
2909     my $maxcpus = $sockets * $cores;
2910
2911     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2912
2913     my $allowed_vcpus = $cpuinfo->{cpus};
2914
2915     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2916         if ($allowed_vcpus < $maxcpus);
2917
2918     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2919
2920     push @$cmd, '-nodefaults';
2921
2922     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2923
2924     my $bootindex_hash = {};
2925     my $i = 1;
2926     foreach my $o (split(//, $bootorder)) {
2927         $bootindex_hash->{$o} = $i*100;
2928         $i++;
2929     }
2930
2931     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2932
2933     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2934
2935     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2936
2937     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2938
2939     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2940         my $socket = vnc_socket($vmid);
2941         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2942     } else {
2943         push @$cmd, '-nographic';
2944     }
2945
2946     # time drift fix
2947     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2948
2949     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2950     my $useLocaltime = $conf->{localtime};
2951
2952     if ($ostype) {
2953         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2954
2955         if ($ostype =~ m/^w/) { # windows
2956             $useLocaltime = 1 if !defined($conf->{localtime});
2957
2958             # use time drift fix when acpi is enabled
2959             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2960                 $tdf = 1 if !defined($conf->{tdf});
2961             }
2962         }
2963
2964         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2965             $ostype eq 'wvista') {
2966             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2967             push @$cmd, '-no-hpet';
2968             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2969                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2970                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2971                 push @$cpuFlags , 'hv_time' if !$nokvm;
2972
2973             } else {
2974                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2975             }
2976         }
2977
2978         if ($ostype eq 'win7' || $ostype eq 'win8') {
2979             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2980         }
2981     }
2982
2983     push @$rtcFlags, 'driftfix=slew' if $tdf;
2984
2985     if ($nokvm) {
2986         push @$machineFlags, 'accel=tcg';
2987     } else {
2988         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2989     }
2990
2991     if ($machine_type) {
2992         push @$machineFlags, "type=${machine_type}";
2993     }
2994
2995     if ($conf->{startdate}) {
2996         push @$rtcFlags, "base=$conf->{startdate}";
2997     } elsif ($useLocaltime) {
2998         push @$rtcFlags, 'base=localtime';
2999     }
3000
3001     my $cpu = $nokvm ? "qemu64" : "kvm64";
3002     if (my $cputype = $conf->{cpu}) {
3003         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
3004             or die "Cannot parse cpu description: $cputype\n";
3005         $cpu = $cpuconf->{cputype};
3006         $kvm_off = 1 if $cpuconf->{hidden};
3007     }
3008
3009     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3010
3011     push @$cpuFlags , '-x2apic'
3012         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3013
3014     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3015
3016     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3017
3018     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3019
3020         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3021         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3022     }
3023
3024     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3025
3026     push @$cpuFlags, 'kvm=off' if $kvm_off;
3027
3028     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3029
3030     push @$cmd, '-cpu', $cpu;
3031
3032     my $memory = $conf->{memory} || $defaults->{memory};
3033     my $static_memory = 0;
3034     my $dimm_memory = 0;
3035
3036     if ($hotplug_features->{memory}) {
3037         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3038         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3039         $static_memory = $STATICMEM;
3040         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3041         $dimm_memory = $memory - $static_memory;
3042         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3043
3044     } else {
3045
3046         $static_memory = $memory;
3047         push @$cmd, '-m', $static_memory;
3048     }
3049
3050     if ($conf->{numa}) {
3051
3052         my $numa_totalmemory = undef;
3053         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3054             next if !$conf->{"numa$i"};
3055             my $numa = parse_numa($conf->{"numa$i"});
3056             next if !$numa;
3057             # memory
3058             die "missing numa node$i memory value\n" if !$numa->{memory};
3059             my $numa_memory = $numa->{memory};
3060             $numa_totalmemory += $numa_memory;
3061             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3062
3063             # cpus
3064             my $cpus_start = $numa->{cpus}->{start};
3065             die "missing numa node$i cpus\n" if !defined($cpus_start);
3066             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3067             my $cpus = $cpus_start;
3068             if (defined($cpus_end)) {
3069                 $cpus .= "-$cpus_end";
3070                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3071             }
3072
3073             # hostnodes
3074             my $hostnodes_start = $numa->{hostnodes}->{start};
3075             if (defined($hostnodes_start)) {
3076                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3077                 my $hostnodes = $hostnodes_start;
3078                 if (defined($hostnodes_end)) {
3079                     $hostnodes .= "-$hostnodes_end";
3080                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3081                 }
3082
3083                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3084                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3085                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3086                 }
3087
3088                 # policy
3089                 my $policy = $numa->{policy};
3090                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3091                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3092             }
3093
3094             push @$cmd, '-object', $numa_object;
3095             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3096         }
3097
3098         die "total memory for NUMA nodes must be equal to vm static memory\n"
3099             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3100
3101         #if no custom tology, we split memory and cores across numa nodes
3102         if(!$numa_totalmemory) {
3103
3104             my $numa_memory = ($static_memory / $sockets) . "M";
3105
3106             for (my $i = 0; $i < $sockets; $i++)  {
3107
3108                 my $cpustart = ($cores * $i);
3109                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3110                 my $cpus = $cpustart;
3111                 $cpus .= "-$cpuend" if $cpuend;
3112
3113                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3114                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3115             }
3116         }
3117     }
3118
3119     if ($hotplug_features->{memory}) {
3120         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3121             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3122             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3123             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3124
3125             #if dimm_memory is not aligned to dimm map
3126             if($current_size > $memory) {
3127                  $conf->{memory} = $current_size;
3128                  PVE::QemuConfig->write_config($vmid, $conf);
3129             }
3130         });
3131     }
3132
3133     push @$cmd, '-S' if $conf->{freeze};
3134
3135     # set keyboard layout
3136     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3137     push @$cmd, '-k', $kb if $kb;
3138
3139     # enable sound
3140     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3141     #push @$cmd, '-soundhw', 'es1370';
3142     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3143
3144     if($conf->{agent}) {
3145         my $qgasocket = qmp_socket($vmid, 1);
3146         my $pciaddr = print_pci_addr("qga0", $bridges);
3147         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3148         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3149         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3150     }
3151
3152     my $spice_port;
3153
3154     if ($qxlnum) {
3155         if ($qxlnum > 1) {
3156             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3157                 for(my $i = 1; $i < $qxlnum; $i++){
3158                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3159                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3160                 }
3161             } else {
3162                 # assume other OS works like Linux
3163                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3164                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3165             }
3166         }
3167
3168         my $pciaddr = print_pci_addr("spice", $bridges);
3169
3170         my $nodename = PVE::INotify::nodename();
3171         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3172         $spice_port = PVE::Tools::next_spice_port($pfamily);
3173
3174         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3175
3176         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3177         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3178         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3179     }
3180
3181     # enable balloon by default, unless explicitly disabled
3182     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3183         $pciaddr = print_pci_addr("balloon0", $bridges);
3184         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3185     }
3186
3187     if ($conf->{watchdog}) {
3188         my $wdopts = parse_watchdog($conf->{watchdog});
3189         $pciaddr = print_pci_addr("watchdog", $bridges);
3190         my $watchdog = $wdopts->{model} || 'i6300esb';
3191         push @$devices, '-device', "$watchdog$pciaddr";
3192         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3193     }
3194
3195     my $vollist = [];
3196     my $scsicontroller = {};
3197     my $ahcicontroller = {};
3198     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3199
3200     # Add iscsi initiator name if available
3201     if (my $initiator = get_initiator_name()) {
3202         push @$devices, '-iscsi', "initiator-name=$initiator";
3203     }
3204
3205     foreach_drive($conf, sub {
3206         my ($ds, $drive) = @_;
3207
3208         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3209             push @$vollist, $drive->{file};
3210         }
3211
3212         $use_virtio = 1 if $ds =~ m/^virtio/;
3213
3214         if (drive_is_cdrom ($drive)) {
3215             if ($bootindex_hash->{d}) {
3216                 $drive->{bootindex} = $bootindex_hash->{d};
3217                 $bootindex_hash->{d} += 1;
3218             }
3219         } else {
3220             if ($bootindex_hash->{c}) {
3221                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3222                 $bootindex_hash->{c} += 1;
3223             }
3224         }
3225
3226         if($drive->{interface} eq 'virtio'){
3227            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3228         }
3229
3230         if ($drive->{interface} eq 'scsi') {
3231
3232             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3233
3234             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3235             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3236
3237             my $iothread = '';
3238             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3239                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3240                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3241             }
3242
3243             my $queues = '';
3244             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3245                 $queues = ",num_queues=$drive->{queues}";
3246             }
3247
3248             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3249             $scsicontroller->{$controller}=1;
3250         }
3251
3252         if ($drive->{interface} eq 'sata') {
3253            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3254            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3255            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3256            $ahcicontroller->{$controller}=1;
3257         }
3258
3259         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3260         push @$devices, '-drive',$drive_cmd;
3261         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3262     });
3263
3264     for (my $i = 0; $i < $MAX_NETS; $i++) {
3265          next if !$conf->{"net$i"};
3266          my $d = parse_net($conf->{"net$i"});
3267          next if !$d;
3268
3269          $use_virtio = 1 if $d->{model} eq 'virtio';
3270
3271          if ($bootindex_hash->{n}) {
3272             $d->{bootindex} = $bootindex_hash->{n};
3273             $bootindex_hash->{n} += 1;
3274          }
3275
3276          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3277          push @$devices, '-netdev', $netdevfull;
3278
3279          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3280          push @$devices, '-device', $netdevicefull;
3281     }
3282
3283     if (!$q35) {
3284         # add pci bridges
3285         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3286            $bridges->{1} = 1;
3287            $bridges->{2} = 1;
3288         }
3289
3290         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3291
3292         while (my ($k, $v) = each %$bridges) {
3293             $pciaddr = print_pci_addr("pci.$k");
3294             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3295         }
3296     }
3297
3298     # add custom args
3299     if ($conf->{args}) {
3300         my $aa = PVE::Tools::split_args($conf->{args});
3301         push @$cmd, @$aa;
3302     }
3303
3304     push @$cmd, @$devices;
3305     push @$cmd, '-rtc', join(',', @$rtcFlags)
3306         if scalar(@$rtcFlags);
3307     push @$cmd, '-machine', join(',', @$machineFlags)
3308         if scalar(@$machineFlags);
3309     push @$cmd, '-global', join(',', @$globalFlags)
3310         if scalar(@$globalFlags);
3311
3312     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3313 }
3314
3315 sub vnc_socket {
3316     my ($vmid) = @_;
3317     return "${var_run_tmpdir}/$vmid.vnc";
3318 }
3319
3320 sub spice_port {
3321     my ($vmid) = @_;
3322
3323     my $res = vm_mon_cmd($vmid, 'query-spice');
3324
3325     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3326 }
3327
3328 sub qmp_socket {
3329     my ($vmid, $qga) = @_;
3330     my $sockettype = $qga ? 'qga' : 'qmp';
3331     return "${var_run_tmpdir}/$vmid.$sockettype";
3332 }
3333
3334 sub pidfile_name {
3335     my ($vmid) = @_;
3336     return "${var_run_tmpdir}/$vmid.pid";
3337 }
3338
3339 sub vm_devices_list {
3340     my ($vmid) = @_;
3341
3342     my $res = vm_mon_cmd($vmid, 'query-pci');
3343     my $devices = {};
3344     foreach my $pcibus (@$res) {
3345         foreach my $device (@{$pcibus->{devices}}) {
3346             next if !$device->{'qdev_id'};
3347             if ($device->{'pci_bridge'}) {
3348                 $devices->{$device->{'qdev_id'}} = 1;
3349                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3350                     next if !$bridge_device->{'qdev_id'};
3351                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3352                     $devices->{$device->{'qdev_id'}}++;
3353                 }
3354             } else {
3355                 $devices->{$device->{'qdev_id'}} = 1;
3356             }
3357         }
3358     }
3359
3360     my $resblock = vm_mon_cmd($vmid, 'query-block');
3361     foreach my $block (@$resblock) {
3362         if($block->{device} =~ m/^drive-(\S+)/){
3363                 $devices->{$1} = 1;
3364         }
3365     }
3366
3367     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3368     foreach my $mice (@$resmice) {
3369         if ($mice->{name} eq 'QEMU HID Tablet') {
3370             $devices->{tablet} = 1;
3371             last;
3372         }
3373     }
3374
3375     return $devices;
3376 }
3377
3378 sub vm_deviceplug {
3379     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3380
3381     my $q35 = machine_type_is_q35($conf);
3382
3383     my $devices_list = vm_devices_list($vmid);
3384     return 1 if defined($devices_list->{$deviceid});
3385
3386     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3387
3388     if ($deviceid eq 'tablet') {
3389
3390         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3391
3392     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3393
3394         qemu_iothread_add($vmid, $deviceid, $device);
3395
3396         qemu_driveadd($storecfg, $vmid, $device);
3397         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3398
3399         qemu_deviceadd($vmid, $devicefull);
3400         eval { qemu_deviceaddverify($vmid, $deviceid); };
3401         if (my $err = $@) {
3402             eval { qemu_drivedel($vmid, $deviceid); };
3403             warn $@ if $@;
3404             die $err;
3405         }
3406
3407     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3408
3409
3410         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3411         my $pciaddr = print_pci_addr($deviceid);
3412         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3413
3414         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3415
3416         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3417             qemu_iothread_add($vmid, $deviceid, $device);
3418             $devicefull .= ",iothread=iothread-$deviceid";
3419         }
3420
3421         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3422             $devicefull .= ",num_queues=$device->{queues}";
3423         }
3424
3425         qemu_deviceadd($vmid, $devicefull);
3426         qemu_deviceaddverify($vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3429
3430         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3431         qemu_driveadd($storecfg, $vmid, $device);
3432
3433         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3434         eval { qemu_deviceadd($vmid, $devicefull); };
3435         if (my $err = $@) {
3436             eval { qemu_drivedel($vmid, $deviceid); };
3437             warn $@ if $@;
3438             die $err;
3439         }
3440
3441     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3442
3443         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3444
3445         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3446         my $use_old_bios_files = undef;
3447         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3448
3449         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3450         qemu_deviceadd($vmid, $netdevicefull);
3451         eval { qemu_deviceaddverify($vmid, $deviceid); };
3452         if (my $err = $@) {
3453             eval { qemu_netdevdel($vmid, $deviceid); };
3454             warn $@ if $@;
3455             die $err;
3456         }
3457
3458     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3459
3460         my $bridgeid = $2;
3461         my $pciaddr = print_pci_addr($deviceid);
3462         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3463
3464         qemu_deviceadd($vmid, $devicefull);
3465         qemu_deviceaddverify($vmid, $deviceid);
3466
3467     } else {
3468         die "can't hotplug device '$deviceid'\n";
3469     }
3470
3471     return 1;
3472 }
3473
3474 # fixme: this should raise exceptions on error!
3475 sub vm_deviceunplug {
3476     my ($vmid, $conf, $deviceid) = @_;
3477
3478     my $devices_list = vm_devices_list($vmid);
3479     return 1 if !defined($devices_list->{$deviceid});
3480
3481     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3482
3483     if ($deviceid eq 'tablet') {
3484
3485         qemu_devicedel($vmid, $deviceid);
3486
3487     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3488
3489         qemu_devicedel($vmid, $deviceid);
3490         qemu_devicedelverify($vmid, $deviceid);
3491         qemu_drivedel($vmid, $deviceid);
3492         qemu_iothread_del($conf, $vmid, $deviceid);
3493
3494     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3495
3496         qemu_devicedel($vmid, $deviceid);
3497         qemu_devicedelverify($vmid, $deviceid);
3498         qemu_iothread_del($conf, $vmid, $deviceid);
3499
3500     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3501
3502         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3503         my $device = parse_drive($deviceid, $conf->{$deviceid});
3504         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3505
3506         qemu_devicedel($vmid, $deviceid);
3507         qemu_drivedel($vmid, $deviceid);
3508         qemu_deletescsihw($conf, $vmid, $deviceid);
3509
3510     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3511
3512         qemu_devicedel($vmid, $deviceid);
3513         qemu_devicedelverify($vmid, $deviceid);
3514         qemu_netdevdel($vmid, $deviceid);
3515
3516     } else {
3517         die "can't unplug device '$deviceid'\n";
3518     }
3519
3520     return 1;
3521 }
3522
3523 sub qemu_deviceadd {
3524     my ($vmid, $devicefull) = @_;
3525
3526     $devicefull = "driver=".$devicefull;
3527     my %options =  split(/[=,]/, $devicefull);
3528
3529     vm_mon_cmd($vmid, "device_add" , %options);
3530 }
3531
3532 sub qemu_devicedel {
3533     my ($vmid, $deviceid) = @_;
3534
3535     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3536 }
3537
3538 sub qemu_iothread_add {
3539     my($vmid, $deviceid, $device) = @_;
3540
3541     if ($device->{iothread}) {
3542         my $iothreads = vm_iothreads_list($vmid);
3543         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3544     }
3545 }
3546
3547 sub qemu_iothread_del {
3548     my($conf, $vmid, $deviceid) = @_;
3549
3550     my $device = parse_drive($deviceid, $conf->{$deviceid});
3551     if ($device->{iothread}) {
3552         my $iothreads = vm_iothreads_list($vmid);
3553         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3554     }
3555 }
3556
3557 sub qemu_objectadd {
3558     my($vmid, $objectid, $qomtype) = @_;
3559
3560     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3561
3562     return 1;
3563 }
3564
3565 sub qemu_objectdel {
3566     my($vmid, $objectid) = @_;
3567
3568     vm_mon_cmd($vmid, "object-del", id => $objectid);
3569
3570     return 1;
3571 }
3572
3573 sub qemu_driveadd {
3574     my ($storecfg, $vmid, $device) = @_;
3575
3576     my $drive = print_drive_full($storecfg, $vmid, $device);
3577     $drive =~ s/\\/\\\\/g;
3578     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3579
3580     # If the command succeeds qemu prints: "OK"
3581     return 1 if $ret =~ m/OK/s;
3582
3583     die "adding drive failed: $ret\n";
3584 }
3585
3586 sub qemu_drivedel {
3587     my($vmid, $deviceid) = @_;
3588
3589     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3590     $ret =~ s/^\s+//;
3591
3592     return 1 if $ret eq "";
3593
3594     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3595     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3596
3597     die "deleting drive $deviceid failed : $ret\n";
3598 }
3599
3600 sub qemu_deviceaddverify {
3601     my ($vmid, $deviceid) = @_;
3602
3603     for (my $i = 0; $i <= 5; $i++) {
3604          my $devices_list = vm_devices_list($vmid);
3605          return 1 if defined($devices_list->{$deviceid});
3606          sleep 1;
3607     }
3608
3609     die "error on hotplug device '$deviceid'\n";
3610 }
3611
3612
3613 sub qemu_devicedelverify {
3614     my ($vmid, $deviceid) = @_;
3615
3616     # need to verify that the device is correctly removed as device_del
3617     # is async and empty return is not reliable
3618
3619     for (my $i = 0; $i <= 5; $i++) {
3620          my $devices_list = vm_devices_list($vmid);
3621          return 1 if !defined($devices_list->{$deviceid});
3622          sleep 1;
3623     }
3624
3625     die "error on hot-unplugging device '$deviceid'\n";
3626 }
3627
3628 sub qemu_findorcreatescsihw {
3629     my ($storecfg, $conf, $vmid, $device) = @_;
3630
3631     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3632
3633     my $scsihwid="$controller_prefix$controller";
3634     my $devices_list = vm_devices_list($vmid);
3635
3636     if(!defined($devices_list->{$scsihwid})) {
3637         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3638     }
3639
3640     return 1;
3641 }
3642
3643 sub qemu_deletescsihw {
3644     my ($conf, $vmid, $opt) = @_;
3645
3646     my $device = parse_drive($opt, $conf->{$opt});
3647
3648     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3649         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3650         return 1;
3651     }
3652
3653     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3654
3655     my $devices_list = vm_devices_list($vmid);
3656     foreach my $opt (keys %{$devices_list}) {
3657         if (PVE::QemuServer::is_valid_drivename($opt)) {
3658             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3659             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3660                 return 1;
3661             }
3662         }
3663     }
3664
3665     my $scsihwid="scsihw$controller";
3666
3667     vm_deviceunplug($vmid, $conf, $scsihwid);
3668
3669     return 1;
3670 }
3671
3672 sub qemu_add_pci_bridge {
3673     my ($storecfg, $conf, $vmid, $device) = @_;
3674
3675     my $bridges = {};
3676
3677     my $bridgeid;
3678
3679     print_pci_addr($device, $bridges);
3680
3681     while (my ($k, $v) = each %$bridges) {
3682         $bridgeid = $k;
3683     }
3684     return 1 if !defined($bridgeid) || $bridgeid < 1;
3685
3686     my $bridge = "pci.$bridgeid";
3687     my $devices_list = vm_devices_list($vmid);
3688
3689     if (!defined($devices_list->{$bridge})) {
3690         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3691     }
3692
3693     return 1;
3694 }
3695
3696 sub qemu_set_link_status {
3697     my ($vmid, $device, $up) = @_;
3698
3699     vm_mon_cmd($vmid, "set_link", name => $device,
3700                up => $up ? JSON::true : JSON::false);
3701 }
3702
3703 sub qemu_netdevadd {
3704     my ($vmid, $conf, $device, $deviceid) = @_;
3705
3706     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3707     my %options =  split(/[=,]/, $netdev);
3708
3709     vm_mon_cmd($vmid, "netdev_add",  %options);
3710     return 1;
3711 }
3712
3713 sub qemu_netdevdel {
3714     my ($vmid, $deviceid) = @_;
3715
3716     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3717 }
3718
3719 sub qemu_cpu_hotplug {
3720     my ($vmid, $conf, $vcpus) = @_;
3721
3722     my $sockets = 1;
3723     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3724     $sockets = $conf->{sockets} if  $conf->{sockets};
3725     my $cores = $conf->{cores} || 1;
3726     my $maxcpus = $sockets * $cores;
3727
3728     $vcpus = $maxcpus if !$vcpus;
3729
3730     die "you can't add more vcpus than maxcpus\n"
3731         if $vcpus > $maxcpus;
3732
3733     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3734     die "online cpu unplug is not yet possible\n"
3735         if $vcpus < $currentvcpus;
3736
3737     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3738     die "vcpus in running vm is different than configuration\n"
3739         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3740
3741     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3742         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3743     }
3744 }
3745
3746 sub qemu_memory_hotplug {
3747     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3748
3749     return $value if !check_running($vmid);
3750
3751     my $memory = $conf->{memory} || $defaults->{memory};
3752     $value = $defaults->{memory} if !$value;
3753     return $value if $value == $memory;
3754
3755     my $static_memory = $STATICMEM;
3756     my $dimm_memory = $memory - $static_memory;
3757
3758     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3759     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3760
3761
3762     my $sockets = 1;
3763     $sockets = $conf->{sockets} if $conf->{sockets};
3764
3765     if($value > $memory) {
3766
3767         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3768             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3769
3770                 return if $current_size <= $conf->{memory};
3771
3772                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3773                 if (my $err = $@) {
3774                     eval { qemu_objectdel($vmid, "mem-$name"); };
3775                     die $err;
3776                 }
3777
3778                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3779                 if (my $err = $@) {
3780                     eval { qemu_objectdel($vmid, "mem-$name"); };
3781                     die $err;
3782                 }
3783                 #update conf after each succesful module hotplug
3784                 $conf->{memory} = $current_size;
3785                 PVE::QemuConfig->write_config($vmid, $conf);
3786         });
3787
3788     } else {
3789
3790         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3791             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3792
3793                 return if $current_size >= $conf->{memory};
3794                 print "try to unplug memory dimm $name\n";
3795
3796                 my $retry = 0;
3797                 while (1) {
3798                     eval { qemu_devicedel($vmid, $name) };
3799                     sleep 3;
3800                     my $dimm_list = qemu_dimm_list($vmid);
3801                     last if !$dimm_list->{$name};
3802                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3803                     $retry++;
3804                 }
3805
3806                 #update conf after each succesful module unplug
3807                 $conf->{memory} = $current_size;
3808
3809                 eval { qemu_objectdel($vmid, "mem-$name"); };
3810                 PVE::QemuConfig->write_config($vmid, $conf);
3811         });
3812     }
3813 }
3814
3815 sub qemu_dimm_list {
3816     my ($vmid) = @_;
3817
3818     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3819     my $dimms = {};
3820
3821     foreach my $dimm (@$dimmarray) {
3822
3823         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3824         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3825         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3826         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3827         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3828     }
3829     return $dimms;
3830 }
3831
3832 sub qemu_block_set_io_throttle {
3833     my ($vmid, $deviceid,
3834         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3835         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3836
3837     return if !check_running($vmid) ;
3838
3839     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3840         bps => int($bps),
3841         bps_rd => int($bps_rd),
3842         bps_wr => int($bps_wr),
3843         iops => int($iops),
3844         iops_rd => int($iops_rd),
3845         iops_wr => int($iops_wr),
3846         bps_max => int($bps_max),
3847         bps_rd_max => int($bps_rd_max),
3848         bps_wr_max => int($bps_wr_max),
3849         iops_max => int($iops_max),
3850         iops_rd_max => int($iops_rd_max),
3851         iops_wr_max => int($iops_wr_max)
3852     );
3853
3854 }
3855
3856 # old code, only used to shutdown old VM after update
3857 sub __read_avail {
3858     my ($fh, $timeout) = @_;
3859
3860     my $sel = new IO::Select;
3861     $sel->add($fh);
3862
3863     my $res = '';
3864     my $buf;
3865
3866     my @ready;
3867     while (scalar (@ready = $sel->can_read($timeout))) {
3868         my $count;
3869         if ($count = $fh->sysread($buf, 8192)) {
3870             if ($buf =~ /^(.*)\(qemu\) $/s) {
3871                 $res .= $1;
3872                 last;
3873             } else {
3874                 $res .= $buf;
3875             }
3876         } else {
3877             if (!defined($count)) {
3878                 die "$!\n";
3879             }
3880             last;
3881         }
3882     }
3883
3884     die "monitor read timeout\n" if !scalar(@ready);
3885
3886     return $res;
3887 }
3888
3889 # old code, only used to shutdown old VM after update
3890 sub vm_monitor_command {
3891     my ($vmid, $cmdstr, $nocheck) = @_;
3892
3893     my $res;
3894
3895     eval {
3896         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3897
3898         my $sname = "${var_run_tmpdir}/$vmid.mon";
3899
3900         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3901             die "unable to connect to VM $vmid socket - $!\n";
3902
3903         my $timeout = 3;
3904
3905         # hack: migrate sometime blocks the monitor (when migrate_downtime
3906         # is set)
3907         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3908             $timeout = 60*60; # 1 hour
3909         }
3910
3911         # read banner;
3912         my $data = __read_avail($sock, $timeout);
3913
3914         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3915             die "got unexpected qemu monitor banner\n";
3916         }
3917
3918         my $sel = new IO::Select;
3919         $sel->add($sock);
3920
3921         if (!scalar(my @ready = $sel->can_write($timeout))) {
3922             die "monitor write error - timeout";
3923         }
3924
3925         my $fullcmd = "$cmdstr\r";
3926
3927         # syslog('info', "VM $vmid monitor command: $cmdstr");
3928
3929         my $b;
3930         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3931             die "monitor write error - $!";
3932         }
3933
3934         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3935
3936         $timeout = 20;
3937
3938         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3939             $timeout = 60*60; # 1 hour
3940         } elsif ($cmdstr =~ m/^(eject|change)/) {
3941             $timeout = 60; # note: cdrom mount command is slow
3942         }
3943         if ($res = __read_avail($sock, $timeout)) {
3944
3945             my @lines = split("\r?\n", $res);
3946
3947             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3948
3949             $res = join("\n", @lines);
3950             $res .= "\n";
3951         }
3952     };
3953
3954     my $err = $@;
3955
3956     if ($err) {
3957         syslog("err", "VM $vmid monitor command failed - $err");
3958         die $err;
3959     }
3960
3961     return $res;
3962 }
3963
3964 sub qemu_block_resize {
3965     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3966
3967     my $running = check_running($vmid);
3968
3969     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3970
3971     return if !$running;
3972
3973     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3974
3975 }
3976
3977 sub qemu_volume_snapshot {
3978     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3979
3980     my $running = check_running($vmid);
3981
3982     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3983         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3984     } else {
3985         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3986     }
3987 }
3988
3989 sub qemu_volume_snapshot_delete {
3990     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3991
3992     my $running = check_running($vmid);
3993
3994     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3995
3996     return if !$running;
3997
3998     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3999 }
4000
4001 sub set_migration_caps {
4002     my ($vmid) = @_;
4003
4004     my $cap_ref = [];
4005
4006     my $enabled_cap = {
4007         "auto-converge" => 1,
4008         "xbzrle" => 1,
4009         "x-rdma-pin-all" => 0,
4010         "zero-blocks" => 0,
4011         "compress" => 0
4012     };
4013
4014     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4015
4016     for my $supported_capability (@$supported_capabilities) {
4017         push @$cap_ref, {
4018             capability => $supported_capability->{capability},
4019             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4020         };
4021     }
4022
4023     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4024 }
4025
4026 my $fast_plug_option = {
4027     'lock' => 1,
4028     'name' => 1,
4029     'onboot' => 1,
4030     'shares' => 1,
4031     'startup' => 1,
4032     'description' => 1,
4033 };
4034
4035 # hotplug changes in [PENDING]
4036 # $selection hash can be used to only apply specified options, for
4037 # example: { cores => 1 } (only apply changed 'cores')
4038 # $errors ref is used to return error messages
4039 sub vmconfig_hotplug_pending {
4040     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4041
4042     my $defaults = load_defaults();
4043
4044     # commit values which do not have any impact on running VM first
4045     # Note: those option cannot raise errors, we we do not care about
4046     # $selection and always apply them.
4047
4048     my $add_error = sub {
4049         my ($opt, $msg) = @_;
4050         $errors->{$opt} = "hotplug problem - $msg";
4051     };
4052
4053     my $changes = 0;
4054     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4055         if ($fast_plug_option->{$opt}) {
4056             $conf->{$opt} = $conf->{pending}->{$opt};
4057             delete $conf->{pending}->{$opt};
4058             $changes = 1;
4059         }
4060     }
4061
4062     if ($changes) {
4063         PVE::QemuConfig->write_config($vmid, $conf);
4064         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4065     }
4066
4067     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4068
4069     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4070     while (my ($opt, $force) = each %$pending_delete_hash) {
4071         next if $selection && !$selection->{$opt};
4072         eval {
4073             if ($opt eq 'hotplug') {
4074                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4075             } elsif ($opt eq 'tablet') {
4076                 die "skip\n" if !$hotplug_features->{usb};
4077                 if ($defaults->{tablet}) {
4078                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4079                 } else {
4080                     vm_deviceunplug($vmid, $conf, $opt);
4081                 }
4082             } elsif ($opt eq 'vcpus') {
4083                 die "skip\n" if !$hotplug_features->{cpu};
4084                 qemu_cpu_hotplug($vmid, $conf, undef);
4085             } elsif ($opt eq 'balloon') {
4086                 # enable balloon device is not hotpluggable
4087                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4088             } elsif ($fast_plug_option->{$opt}) {
4089                 # do nothing
4090             } elsif ($opt =~ m/^net(\d+)$/) {
4091                 die "skip\n" if !$hotplug_features->{network};
4092                 vm_deviceunplug($vmid, $conf, $opt);
4093             } elsif (is_valid_drivename($opt)) {
4094                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4095                 vm_deviceunplug($vmid, $conf, $opt);
4096                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4097             } elsif ($opt =~ m/^memory$/) {
4098                 die "skip\n" if !$hotplug_features->{memory};
4099                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4100             } elsif ($opt eq 'cpuunits') {
4101                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4102             } elsif ($opt eq 'cpulimit') {
4103                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4104             } else {
4105                 die "skip\n";
4106             }
4107         };
4108         if (my $err = $@) {
4109             &$add_error($opt, $err) if $err ne "skip\n";
4110         } else {
4111             # save new config if hotplug was successful
4112             delete $conf->{$opt};
4113             vmconfig_undelete_pending_option($conf, $opt);
4114             PVE::QemuConfig->write_config($vmid, $conf);
4115             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4116         }
4117     }
4118
4119     foreach my $opt (keys %{$conf->{pending}}) {
4120         next if $selection && !$selection->{$opt};
4121         my $value = $conf->{pending}->{$opt};
4122         eval {
4123             if ($opt eq 'hotplug') {
4124                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4125             } elsif ($opt eq 'tablet') {
4126                 die "skip\n" if !$hotplug_features->{usb};
4127                 if ($value == 1) {
4128                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4129                 } elsif ($value == 0) {
4130                     vm_deviceunplug($vmid, $conf, $opt);
4131                 }
4132             } elsif ($opt eq 'vcpus') {
4133                 die "skip\n" if !$hotplug_features->{cpu};
4134                 qemu_cpu_hotplug($vmid, $conf, $value);
4135             } elsif ($opt eq 'balloon') {
4136                 # enable/disable balloning device is not hotpluggable
4137                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4138                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4139                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4140
4141                 # allow manual ballooning if shares is set to zero
4142                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4143                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4144                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4145                 }
4146             } elsif ($opt =~ m/^net(\d+)$/) {
4147                 # some changes can be done without hotplug
4148                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4149                                     $vmid, $opt, $value);
4150             } elsif (is_valid_drivename($opt)) {
4151                 # some changes can be done without hotplug
4152                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4153                                      $vmid, $opt, $value, 1);
4154             } elsif ($opt =~ m/^memory$/) { #dimms
4155                 die "skip\n" if !$hotplug_features->{memory};
4156                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4157             } elsif ($opt eq 'cpuunits') {
4158                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4159             } elsif ($opt eq 'cpulimit') {
4160                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4161                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4162             } else {
4163                 die "skip\n";  # skip non-hot-pluggable options
4164             }
4165         };
4166         if (my $err = $@) {
4167             &$add_error($opt, $err) if $err ne "skip\n";
4168         } else {
4169             # save new config if hotplug was successful
4170             $conf->{$opt} = $value;
4171             delete $conf->{pending}->{$opt};
4172             PVE::QemuConfig->write_config($vmid, $conf);
4173             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4174         }
4175     }
4176 }
4177
4178 sub try_deallocate_drive {
4179     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4180
4181     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4182         my $volid = $drive->{file};
4183         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4184             my $sid = PVE::Storage::parse_volume_id($volid);
4185             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4186
4187             # check if the disk is really unused
4188             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4189                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4190             PVE::Storage::vdisk_free($storecfg, $volid);
4191             return 1;
4192         } else {
4193             # If vm is not owner of this disk remove from config
4194             return 1;
4195         }
4196     }
4197
4198     return undef;
4199 }
4200
4201 sub vmconfig_delete_or_detach_drive {
4202     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4203
4204     my $drive = parse_drive($opt, $conf->{$opt});
4205
4206     my $rpcenv = PVE::RPCEnvironment::get();
4207     my $authuser = $rpcenv->get_user();
4208
4209     if ($force) {
4210         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4211         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4212     } else {
4213         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4214     }
4215 }
4216
4217 sub vmconfig_apply_pending {
4218     my ($vmid, $conf, $storecfg) = @_;
4219
4220     # cold plug
4221
4222     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4223     while (my ($opt, $force) = each %$pending_delete_hash) {
4224         die "internal error" if $opt =~ m/^unused/;
4225         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4226         if (!defined($conf->{$opt})) {
4227             vmconfig_undelete_pending_option($conf, $opt);
4228             PVE::QemuConfig->write_config($vmid, $conf);
4229         } elsif (is_valid_drivename($opt)) {
4230             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4231             vmconfig_undelete_pending_option($conf, $opt);
4232             delete $conf->{$opt};
4233             PVE::QemuConfig->write_config($vmid, $conf);
4234         } else {
4235             vmconfig_undelete_pending_option($conf, $opt);
4236             delete $conf->{$opt};
4237             PVE::QemuConfig->write_config($vmid, $conf);
4238         }
4239     }
4240
4241     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4242
4243     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4244         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4245
4246         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4247             # skip if nothing changed
4248         } elsif (is_valid_drivename($opt)) {
4249             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4250                 if defined($conf->{$opt});
4251             $conf->{$opt} = $conf->{pending}->{$opt};
4252         } else {
4253             $conf->{$opt} = $conf->{pending}->{$opt};
4254         }
4255
4256         delete $conf->{pending}->{$opt};
4257         PVE::QemuConfig->write_config($vmid, $conf);
4258     }
4259 }
4260
4261 my $safe_num_ne = sub {
4262     my ($a, $b) = @_;
4263
4264     return 0 if !defined($a) && !defined($b);
4265     return 1 if !defined($a);
4266     return 1 if !defined($b);
4267
4268     return $a != $b;
4269 };
4270
4271 my $safe_string_ne = sub {
4272     my ($a, $b) = @_;
4273
4274     return 0 if !defined($a) && !defined($b);
4275     return 1 if !defined($a);
4276     return 1 if !defined($b);
4277
4278     return $a ne $b;
4279 };
4280
4281 sub vmconfig_update_net {
4282     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4283
4284     my $newnet = parse_net($value);
4285
4286     if ($conf->{$opt}) {
4287         my $oldnet = parse_net($conf->{$opt});
4288
4289         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4290             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4291             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4292             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4293
4294             # for non online change, we try to hot-unplug
4295             die "skip\n" if !$hotplug;
4296             vm_deviceunplug($vmid, $conf, $opt);
4297         } else {
4298
4299             die "internal error" if $opt !~ m/net(\d+)/;
4300             my $iface = "tap${vmid}i$1";
4301
4302             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4303                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4304                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4305                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4306                 PVE::Network::tap_unplug($iface);
4307                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4308             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4309                 # Rate can be applied on its own but any change above needs to
4310                 # include the rate in tap_plug since OVS resets everything.
4311                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4312             }
4313
4314             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4315                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4316             }
4317
4318             return 1;
4319         }
4320     }
4321
4322     if ($hotplug) {
4323         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4324     } else {
4325         die "skip\n";
4326     }
4327 }
4328
4329 sub vmconfig_update_disk {
4330     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4331
4332     # fixme: do we need force?
4333
4334     my $drive = parse_drive($opt, $value);
4335
4336     if ($conf->{$opt}) {
4337
4338         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4339
4340             my $media = $drive->{media} || 'disk';
4341             my $oldmedia = $old_drive->{media} || 'disk';
4342             die "unable to change media type\n" if $media ne $oldmedia;
4343
4344             if (!drive_is_cdrom($old_drive)) {
4345
4346                 if ($drive->{file} ne $old_drive->{file}) {
4347
4348                     die "skip\n" if !$hotplug;
4349
4350                     # unplug and register as unused
4351                     vm_deviceunplug($vmid, $conf, $opt);
4352                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4353
4354                 } else {
4355                     # update existing disk
4356
4357                     # skip non hotpluggable value
4358                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4359                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4360                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4361                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4362                         die "skip\n";
4363                     }
4364
4365                     # apply throttle
4366                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4367                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4368                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4369                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4370                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4371                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4372                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4373                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4374                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4375                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4376                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4377                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4378
4379                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4380                                                    ($drive->{mbps} || 0)*1024*1024,
4381                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4382                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4383                                                    $drive->{iops} || 0,
4384                                                    $drive->{iops_rd} || 0,
4385                                                    $drive->{iops_wr} || 0,
4386                                                    ($drive->{mbps_max} || 0)*1024*1024,
4387                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4388                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4389                                                    $drive->{iops_max} || 0,
4390                                                    $drive->{iops_rd_max} || 0,
4391                                                    $drive->{iops_wr_max} || 0);
4392
4393                     }
4394
4395                     return 1;
4396                 }
4397
4398             } else { # cdrom
4399
4400                 if ($drive->{file} eq 'none') {
4401                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4402                 } else {
4403                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4404                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4405                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4406                 }
4407
4408                 return 1;
4409             }
4410         }
4411     }
4412
4413     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4414     # hotplug new disks
4415     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4416     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4417 }
4418
4419 sub vm_start {
4420     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4421         $forcemachine, $spice_ticket) = @_;
4422
4423     PVE::QemuConfig->lock_config($vmid, sub {
4424         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4425
4426         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4427
4428         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4429
4430         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4431
4432         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4433             vmconfig_apply_pending($vmid, $conf, $storecfg);
4434             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4435         }
4436
4437         my $defaults = load_defaults();
4438
4439         # set environment variable useful inside network script
4440         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4441
4442         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4443
4444         my $migrate_port = 0;
4445         my $migrate_uri;
4446         if ($statefile) {
4447             if ($statefile eq 'tcp') {
4448                 my $localip = "localhost";
4449                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4450                 my $nodename = PVE::INotify::nodename();
4451                 if ($datacenterconf->{migration_unsecure}) {
4452                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4453                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4454                 }
4455                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4456                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4457                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4458                 push @$cmd, '-incoming', $migrate_uri;
4459                 push @$cmd, '-S';
4460             } else {
4461                 push @$cmd, '-loadstate', $statefile;
4462             }
4463         } elsif ($paused) {
4464             push @$cmd, '-S';
4465         }
4466
4467         # host pci devices
4468         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4469           my $d = parse_hostpci($conf->{"hostpci$i"});
4470           next if !$d;
4471           my $pcidevices = $d->{pciid};
4472           foreach my $pcidevice (@$pcidevices) {
4473                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4474
4475                 my $info = pci_device_info("0000:$pciid");
4476                 die "IOMMU not present\n" if !check_iommu_support();
4477                 die "no pci device info for device '$pciid'\n" if !$info;
4478                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4479                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4480           }
4481         }
4482
4483         PVE::Storage::activate_volumes($storecfg, $vollist);
4484
4485         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4486                     umask => 0077); };
4487
4488         if (my $err = $@) {
4489             # deactivate volumes if start fails
4490             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4491             die "start failed: $err";
4492         }
4493
4494         print "migration listens on $migrate_uri\n" if $migrate_uri;
4495
4496         if ($statefile && $statefile ne 'tcp')  {
4497             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4498             warn $@ if $@;
4499         }
4500
4501         if ($migratedfrom) {
4502
4503             eval {
4504                 set_migration_caps($vmid);
4505             };
4506             warn $@ if $@;
4507
4508             if ($spice_port) {
4509                 print "spice listens on port $spice_port\n";
4510                 if ($spice_ticket) {
4511                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4512                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4513                 }
4514             }
4515
4516         } else {
4517
4518             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4519                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4520                     if $conf->{balloon};
4521             }
4522
4523             foreach my $opt (keys %$conf) {
4524                 next if $opt !~  m/^net\d+$/;
4525                 my $nicconf = parse_net($conf->{$opt});
4526                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4527             }
4528         }
4529
4530         vm_mon_cmd_nocheck($vmid, 'qom-set',
4531                     path => "machine/peripheral/balloon0",
4532                     property => "guest-stats-polling-interval",
4533                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4534
4535     });
4536 }
4537
4538 sub vm_mon_cmd {
4539     my ($vmid, $execute, %params) = @_;
4540
4541     my $cmd = { execute => $execute, arguments => \%params };
4542     vm_qmp_command($vmid, $cmd);
4543 }
4544
4545 sub vm_mon_cmd_nocheck {
4546     my ($vmid, $execute, %params) = @_;
4547
4548     my $cmd = { execute => $execute, arguments => \%params };
4549     vm_qmp_command($vmid, $cmd, 1);
4550 }
4551
4552 sub vm_qmp_command {
4553     my ($vmid, $cmd, $nocheck) = @_;
4554
4555     my $res;
4556
4557     my $timeout;
4558     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4559         $timeout = $cmd->{arguments}->{timeout};
4560         delete $cmd->{arguments}->{timeout};
4561     }
4562
4563     eval {
4564         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4565         my $sname = qmp_socket($vmid);
4566         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4567             my $qmpclient = PVE::QMPClient->new();
4568
4569             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4570         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4571             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4572                 if scalar(%{$cmd->{arguments}});
4573             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4574         } else {
4575             die "unable to open monitor socket\n";
4576         }
4577     };
4578     if (my $err = $@) {
4579         syslog("err", "VM $vmid qmp command failed - $err");
4580         die $err;
4581     }
4582
4583     return $res;
4584 }
4585
4586 sub vm_human_monitor_command {
4587     my ($vmid, $cmdline) = @_;
4588
4589     my $res;
4590
4591     my $cmd = {
4592         execute => 'human-monitor-command',
4593         arguments => { 'command-line' => $cmdline},
4594     };
4595
4596     return vm_qmp_command($vmid, $cmd);
4597 }
4598
4599 sub vm_commandline {
4600     my ($storecfg, $vmid) = @_;
4601
4602     my $conf = PVE::QemuConfig->load_config($vmid);
4603
4604     my $defaults = load_defaults();
4605
4606     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4607
4608     return join(' ', @$cmd);
4609 }
4610
4611 sub vm_reset {
4612     my ($vmid, $skiplock) = @_;
4613
4614     PVE::QemuConfig->lock_config($vmid, sub {
4615
4616         my $conf = PVE::QemuConfig->load_config($vmid);
4617
4618         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4619
4620         vm_mon_cmd($vmid, "system_reset");
4621     });
4622 }
4623
4624 sub get_vm_volumes {
4625     my ($conf) = @_;
4626
4627     my $vollist = [];
4628     foreach_volid($conf, sub {
4629         my ($volid, $is_cdrom) = @_;
4630
4631         return if $volid =~ m|^/|;
4632
4633         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4634         return if !$sid;
4635
4636         push @$vollist, $volid;
4637     });
4638
4639     return $vollist;
4640 }
4641
4642 sub vm_stop_cleanup {
4643     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4644
4645     eval {
4646
4647         if (!$keepActive) {
4648             my $vollist = get_vm_volumes($conf);
4649             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4650         }
4651
4652         foreach my $ext (qw(mon qmp pid vnc qga)) {
4653             unlink "/var/run/qemu-server/${vmid}.$ext";
4654         }
4655
4656         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4657     };
4658     warn $@ if $@; # avoid errors - just warn
4659 }
4660
4661 # Note: use $nockeck to skip tests if VM configuration file exists.
4662 # We need that when migration VMs to other nodes (files already moved)
4663 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4664 sub vm_stop {
4665     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4666
4667     $force = 1 if !defined($force) && !$shutdown;
4668
4669     if ($migratedfrom){
4670         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4671         kill 15, $pid if $pid;
4672         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4673         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4674         return;
4675     }
4676
4677     PVE::QemuConfig->lock_config($vmid, sub {
4678
4679         my $pid = check_running($vmid, $nocheck);
4680         return if !$pid;
4681
4682         my $conf;
4683         if (!$nocheck) {
4684             $conf = PVE::QemuConfig->load_config($vmid);
4685             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4686             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4687                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4688                 $timeout = $opts->{down} if $opts->{down};
4689             }
4690         }
4691
4692         $timeout = 60 if !defined($timeout);
4693
4694         eval {
4695             if ($shutdown) {
4696                 if (defined($conf) && $conf->{agent}) {
4697                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4698                 } else {
4699                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4700                 }
4701             } else {
4702                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4703             }
4704         };
4705         my $err = $@;
4706
4707         if (!$err) {
4708             my $count = 0;
4709             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4710                 $count++;
4711                 sleep 1;
4712             }
4713
4714             if ($count >= $timeout) {
4715                 if ($force) {
4716                     warn "VM still running - terminating now with SIGTERM\n";
4717                     kill 15, $pid;
4718                 } else {
4719                     die "VM quit/powerdown failed - got timeout\n";
4720                 }
4721             } else {
4722                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4723                 return;
4724             }
4725         } else {
4726             if ($force) {
4727                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4728                 kill 15, $pid;
4729             } else {
4730                 die "VM quit/powerdown failed\n";
4731             }
4732         }
4733
4734         # wait again
4735         $timeout = 10;
4736
4737         my $count = 0;
4738         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4739             $count++;
4740             sleep 1;
4741         }
4742
4743         if ($count >= $timeout) {
4744             warn "VM still running - terminating now with SIGKILL\n";
4745             kill 9, $pid;
4746             sleep 1;
4747         }
4748
4749         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4750    });
4751 }
4752
4753 sub vm_suspend {
4754     my ($vmid, $skiplock) = @_;
4755
4756     PVE::QemuConfig->lock_config($vmid, sub {
4757
4758         my $conf = PVE::QemuConfig->load_config($vmid);
4759
4760         PVE::QemuConfig->check_lock($conf)
4761             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4762
4763         vm_mon_cmd($vmid, "stop");
4764     });
4765 }
4766
4767 sub vm_resume {
4768     my ($vmid, $skiplock, $nocheck) = @_;
4769
4770     PVE::QemuConfig->lock_config($vmid, sub {
4771
4772         if (!$nocheck) {
4773
4774             my $conf = PVE::QemuConfig->load_config($vmid);
4775
4776             PVE::QemuConfig->check_lock($conf)
4777                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4778
4779             vm_mon_cmd($vmid, "cont");
4780
4781         } else {
4782             vm_mon_cmd_nocheck($vmid, "cont");
4783         }
4784     });
4785 }
4786
4787 sub vm_sendkey {
4788     my ($vmid, $skiplock, $key) = @_;
4789
4790     PVE::QemuConfig->lock_config($vmid, sub {
4791
4792         my $conf = PVE::QemuConfig->load_config($vmid);
4793
4794         # there is no qmp command, so we use the human monitor command
4795         vm_human_monitor_command($vmid, "sendkey $key");
4796     });
4797 }
4798
4799 sub vm_destroy {
4800     my ($storecfg, $vmid, $skiplock) = @_;
4801
4802     PVE::QemuConfig->lock_config($vmid, sub {
4803
4804         my $conf = PVE::QemuConfig->load_config($vmid);
4805
4806         if (!check_running($vmid)) {
4807             destroy_vm($storecfg, $vmid, undef, $skiplock);
4808         } else {
4809             die "VM $vmid is running - destroy failed\n";
4810         }
4811     });
4812 }
4813
4814 # pci helpers
4815
4816 sub file_write {
4817     my ($filename, $buf) = @_;
4818
4819     my $fh = IO::File->new($filename, "w");
4820     return undef if !$fh;
4821
4822     my $res = print $fh $buf;
4823
4824     $fh->close();
4825
4826     return $res;
4827 }
4828
4829 sub pci_device_info {
4830     my ($name) = @_;
4831
4832     my $res;
4833
4834     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4835     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4836
4837     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4838     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4839
4840     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4841     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4842
4843     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4844     return undef if !defined($product) || $product !~ s/^0x//;
4845
4846     $res = {
4847         name => $name,
4848         vendor => $vendor,
4849         product => $product,
4850         domain => $domain,
4851         bus => $bus,
4852         slot => $slot,
4853         func => $func,
4854         irq => $irq,
4855         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4856     };
4857
4858     return $res;
4859 }
4860
4861 sub pci_dev_reset {
4862     my ($dev) = @_;
4863
4864     my $name = $dev->{name};
4865
4866     my $fn = "$pcisysfs/devices/$name/reset";
4867
4868     return file_write($fn, "1");
4869 }
4870
4871 sub pci_dev_bind_to_vfio {
4872     my ($dev) = @_;
4873
4874     my $name = $dev->{name};
4875
4876     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4877
4878     if (!-d $vfio_basedir) {
4879         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4880     }
4881     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4882
4883     my $testdir = "$vfio_basedir/$name";
4884     return 1 if -d $testdir;
4885
4886     my $data = "$dev->{vendor} $dev->{product}";
4887     return undef if !file_write("$vfio_basedir/new_id", $data);
4888
4889     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4890     if (!file_write($fn, $name)) {
4891         return undef if -f $fn;
4892     }
4893
4894     $fn = "$vfio_basedir/bind";
4895     if (! -d $testdir) {
4896         return undef if !file_write($fn, $name);
4897     }
4898
4899     return -d $testdir;
4900 }
4901
4902 sub pci_dev_group_bind_to_vfio {
4903     my ($pciid) = @_;
4904
4905     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4906
4907     if (!-d $vfio_basedir) {
4908         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4909     }
4910     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4911
4912     # get IOMMU group devices
4913     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4914       my @devs = grep /^0000:/, readdir($D);
4915     closedir($D);
4916
4917     foreach my $pciid (@devs) {
4918         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4919
4920         # pci bridges, switches or root ports are not supported
4921         # they have a pci_bus subdirectory so skip them
4922         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4923
4924         my $info = pci_device_info($1);
4925         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4926     }
4927
4928     return 1;
4929 }
4930
4931 sub print_pci_addr {
4932     my ($id, $bridges) = @_;
4933
4934     my $res = '';
4935     my $devices = {
4936         piix3 => { bus => 0, addr => 1 },
4937         #addr2 : first videocard
4938         balloon0 => { bus => 0, addr => 3 },
4939         watchdog => { bus => 0, addr => 4 },
4940         scsihw0 => { bus => 0, addr => 5 },
4941         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4942         scsihw1 => { bus => 0, addr => 6 },
4943         ahci0 => { bus => 0, addr => 7 },
4944         qga0 => { bus => 0, addr => 8 },
4945         spice => { bus => 0, addr => 9 },
4946         virtio0 => { bus => 0, addr => 10 },
4947         virtio1 => { bus => 0, addr => 11 },
4948         virtio2 => { bus => 0, addr => 12 },
4949         virtio3 => { bus => 0, addr => 13 },
4950         virtio4 => { bus => 0, addr => 14 },
4951         virtio5 => { bus => 0, addr => 15 },
4952         hostpci0 => { bus => 0, addr => 16 },
4953         hostpci1 => { bus => 0, addr => 17 },
4954         net0 => { bus => 0, addr => 18 },
4955         net1 => { bus => 0, addr => 19 },
4956         net2 => { bus => 0, addr => 20 },
4957         net3 => { bus => 0, addr => 21 },
4958         net4 => { bus => 0, addr => 22 },
4959         net5 => { bus => 0, addr => 23 },
4960         vga1 => { bus => 0, addr => 24 },
4961         vga2 => { bus => 0, addr => 25 },
4962         vga3 => { bus => 0, addr => 26 },
4963         hostpci2 => { bus => 0, addr => 27 },
4964         hostpci3 => { bus => 0, addr => 28 },
4965         #addr29 : usb-host (pve-usb.cfg)
4966         'pci.1' => { bus => 0, addr => 30 },
4967         'pci.2' => { bus => 0, addr => 31 },
4968         'net6' => { bus => 1, addr => 1 },
4969         'net7' => { bus => 1, addr => 2 },
4970         'net8' => { bus => 1, addr => 3 },
4971         'net9' => { bus => 1, addr => 4 },
4972         'net10' => { bus => 1, addr => 5 },
4973         'net11' => { bus => 1, addr => 6 },
4974         'net12' => { bus => 1, addr => 7 },
4975         'net13' => { bus => 1, addr => 8 },
4976         'net14' => { bus => 1, addr => 9 },
4977         'net15' => { bus => 1, addr => 10 },
4978         'net16' => { bus => 1, addr => 11 },
4979         'net17' => { bus => 1, addr => 12 },
4980         'net18' => { bus => 1, addr => 13 },
4981         'net19' => { bus => 1, addr => 14 },
4982         'net20' => { bus => 1, addr => 15 },
4983         'net21' => { bus => 1, addr => 16 },
4984         'net22' => { bus => 1, addr => 17 },
4985         'net23' => { bus => 1, addr => 18 },
4986         'net24' => { bus => 1, addr => 19 },
4987         'net25' => { bus => 1, addr => 20 },
4988         'net26' => { bus => 1, addr => 21 },
4989         'net27' => { bus => 1, addr => 22 },
4990         'net28' => { bus => 1, addr => 23 },
4991         'net29' => { bus => 1, addr => 24 },
4992         'net30' => { bus => 1, addr => 25 },
4993         'net31' => { bus => 1, addr => 26 },
4994         'xhci' => { bus => 1, addr => 27 },
4995         'virtio6' => { bus => 2, addr => 1 },
4996         'virtio7' => { bus => 2, addr => 2 },
4997         'virtio8' => { bus => 2, addr => 3 },
4998         'virtio9' => { bus => 2, addr => 4 },
4999         'virtio10' => { bus => 2, addr => 5 },
5000         'virtio11' => { bus => 2, addr => 6 },
5001         'virtio12' => { bus => 2, addr => 7 },
5002         'virtio13' => { bus => 2, addr => 8 },
5003         'virtio14' => { bus => 2, addr => 9 },
5004         'virtio15' => { bus => 2, addr => 10 },
5005         'virtioscsi0' => { bus => 3, addr => 1 },
5006         'virtioscsi1' => { bus => 3, addr => 2 },
5007         'virtioscsi2' => { bus => 3, addr => 3 },
5008         'virtioscsi3' => { bus => 3, addr => 4 },
5009         'virtioscsi4' => { bus => 3, addr => 5 },
5010         'virtioscsi5' => { bus => 3, addr => 6 },
5011         'virtioscsi6' => { bus => 3, addr => 7 },
5012         'virtioscsi7' => { bus => 3, addr => 8 },
5013         'virtioscsi8' => { bus => 3, addr => 9 },
5014         'virtioscsi9' => { bus => 3, addr => 10 },
5015         'virtioscsi10' => { bus => 3, addr => 11 },
5016         'virtioscsi11' => { bus => 3, addr => 12 },
5017         'virtioscsi12' => { bus => 3, addr => 13 },
5018         'virtioscsi13' => { bus => 3, addr => 14 },
5019         'virtioscsi14' => { bus => 3, addr => 15 },
5020         'virtioscsi15' => { bus => 3, addr => 16 },
5021         'virtioscsi16' => { bus => 3, addr => 17 },
5022         'virtioscsi17' => { bus => 3, addr => 18 },
5023         'virtioscsi18' => { bus => 3, addr => 19 },
5024         'virtioscsi19' => { bus => 3, addr => 20 },
5025         'virtioscsi20' => { bus => 3, addr => 21 },
5026         'virtioscsi21' => { bus => 3, addr => 22 },
5027         'virtioscsi22' => { bus => 3, addr => 23 },
5028         'virtioscsi23' => { bus => 3, addr => 24 },
5029         'virtioscsi24' => { bus => 3, addr => 25 },
5030         'virtioscsi25' => { bus => 3, addr => 26 },
5031         'virtioscsi26' => { bus => 3, addr => 27 },
5032         'virtioscsi27' => { bus => 3, addr => 28 },
5033         'virtioscsi28' => { bus => 3, addr => 29 },
5034         'virtioscsi29' => { bus => 3, addr => 30 },
5035         'virtioscsi30' => { bus => 3, addr => 31 },
5036
5037     };
5038
5039     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5040            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5041            my $bus = $devices->{$id}->{bus};
5042            $res = ",bus=pci.$bus,addr=$addr";
5043            $bridges->{$bus} = 1 if $bridges;
5044     }
5045     return $res;
5046
5047 }
5048
5049 sub print_pcie_addr {
5050     my ($id) = @_;
5051
5052     my $res = '';
5053     my $devices = {
5054         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5055         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5056         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5057         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5058     };
5059
5060     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5061            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5062            my $bus = $devices->{$id}->{bus};
5063            $res = ",bus=$bus,addr=$addr";
5064     }
5065     return $res;
5066
5067 }
5068
5069 # vzdump restore implementaion
5070
5071 sub tar_archive_read_firstfile {
5072     my $archive = shift;
5073
5074     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5075
5076     # try to detect archive type first
5077     my $pid = open (TMP, "tar tf '$archive'|") ||
5078         die "unable to open file '$archive'\n";
5079     my $firstfile = <TMP>;
5080     kill 15, $pid;
5081     close TMP;
5082
5083     die "ERROR: archive contaions no data\n" if !$firstfile;
5084     chomp $firstfile;
5085
5086     return $firstfile;
5087 }
5088
5089 sub tar_restore_cleanup {
5090     my ($storecfg, $statfile) = @_;
5091
5092     print STDERR "starting cleanup\n";
5093
5094     if (my $fd = IO::File->new($statfile, "r")) {
5095         while (defined(my $line = <$fd>)) {
5096             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5097                 my $volid = $2;
5098                 eval {
5099                     if ($volid =~ m|^/|) {
5100                         unlink $volid || die 'unlink failed\n';
5101                     } else {
5102                         PVE::Storage::vdisk_free($storecfg, $volid);
5103                     }
5104                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5105                 };
5106                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5107             } else {
5108                 print STDERR "unable to parse line in statfile - $line";
5109             }
5110         }
5111         $fd->close();
5112     }
5113 }
5114
5115 sub restore_archive {
5116     my ($archive, $vmid, $user, $opts) = @_;
5117
5118     my $format = $opts->{format};
5119     my $comp;
5120
5121     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5122         $format = 'tar' if !$format;
5123         $comp = 'gzip';
5124     } elsif ($archive =~ m/\.tar$/) {
5125         $format = 'tar' if !$format;
5126     } elsif ($archive =~ m/.tar.lzo$/) {
5127         $format = 'tar' if !$format;
5128         $comp = 'lzop';
5129     } elsif ($archive =~ m/\.vma$/) {
5130         $format = 'vma' if !$format;
5131     } elsif ($archive =~ m/\.vma\.gz$/) {
5132         $format = 'vma' if !$format;
5133         $comp = 'gzip';
5134     } elsif ($archive =~ m/\.vma\.lzo$/) {
5135         $format = 'vma' if !$format;
5136         $comp = 'lzop';
5137     } else {
5138         $format = 'vma' if !$format; # default
5139     }
5140
5141     # try to detect archive format
5142     if ($format eq 'tar') {
5143         return restore_tar_archive($archive, $vmid, $user, $opts);
5144     } else {
5145         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5146     }
5147 }
5148
5149 sub restore_update_config_line {
5150     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5151
5152     return if $line =~ m/^\#qmdump\#/;
5153     return if $line =~ m/^\#vzdump\#/;
5154     return if $line =~ m/^lock:/;
5155     return if $line =~ m/^unused\d+:/;
5156     return if $line =~ m/^parent:/;
5157     return if $line =~ m/^template:/; # restored VM is never a template
5158
5159     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5160         # try to convert old 1.X settings
5161         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5162         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5163             my ($model, $macaddr) = split(/\=/, $devconfig);
5164             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5165             my $net = {
5166                 model => $model,
5167                 bridge => "vmbr$ind",
5168                 macaddr => $macaddr,
5169             };
5170             my $netstr = print_net($net);
5171
5172             print $outfd "net$cookie->{netcount}: $netstr\n";
5173             $cookie->{netcount}++;
5174         }
5175     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5176         my ($id, $netstr) = ($1, $2);
5177         my $net = parse_net($netstr);
5178         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5179         $netstr = print_net($net);
5180         print $outfd "$id: $netstr\n";
5181     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5182         my $virtdev = $1;
5183         my $value = $3;
5184         my $di = parse_drive($virtdev, $value);
5185         if (defined($di->{backup}) && !$di->{backup}) {
5186             print $outfd "#$line";
5187         } elsif ($map->{$virtdev}) {
5188             delete $di->{format}; # format can change on restore
5189             $di->{file} = $map->{$virtdev};
5190             $value = print_drive($vmid, $di);
5191             print $outfd "$virtdev: $value\n";
5192         } else {
5193             print $outfd $line;
5194         }
5195     } else {
5196         print $outfd $line;
5197     }
5198 }
5199
5200 sub scan_volids {
5201     my ($cfg, $vmid) = @_;
5202
5203     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5204
5205     my $volid_hash = {};
5206     foreach my $storeid (keys %$info) {
5207         foreach my $item (@{$info->{$storeid}}) {
5208             next if !($item->{volid} && $item->{size});
5209             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5210             $volid_hash->{$item->{volid}} = $item;
5211         }
5212     }
5213
5214     return $volid_hash;
5215 }
5216
5217 sub is_volume_in_use {
5218     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5219
5220     my $path = PVE::Storage::path($storecfg, $volid);
5221
5222     my $scan_config = sub {
5223         my ($cref, $snapname) = @_;
5224
5225         foreach my $key (keys %$cref) {
5226             my $value = $cref->{$key};
5227             if (is_valid_drivename($key)) {
5228                 next if $skip_drive && $key eq $skip_drive;
5229                 my $drive = parse_drive($key, $value);
5230                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5231                 return 1 if $volid eq $drive->{file};
5232                 if ($drive->{file} =~ m!^/!) {
5233                     return 1 if $drive->{file} eq $path;
5234                 } else {
5235                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5236                     next if !$storeid;
5237                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5238                     next if !$scfg;
5239                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5240                 }
5241             }
5242         }
5243
5244         return 0;
5245     };
5246
5247     return 1 if &$scan_config($conf);
5248
5249     undef $skip_drive;
5250
5251     foreach my $snapname (keys %{$conf->{snapshots}}) {
5252         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5253     }
5254
5255     return 0;
5256 }
5257
5258 sub update_disksize {
5259     my ($vmid, $conf, $volid_hash) = @_;
5260
5261     my $changes;
5262
5263     my $used = {};
5264
5265     # Note: it is allowed to define multiple storages with same path (alias), so
5266     # we need to check both 'volid' and real 'path' (two different volid can point
5267     # to the same path).
5268
5269     my $usedpath = {};
5270
5271     # update size info
5272     foreach my $opt (keys %$conf) {
5273         if (is_valid_drivename($opt)) {
5274             my $drive = parse_drive($opt, $conf->{$opt});
5275             my $volid = $drive->{file};
5276             next if !$volid;
5277
5278             $used->{$volid} = 1;
5279             if ($volid_hash->{$volid} &&
5280                 (my $path = $volid_hash->{$volid}->{path})) {
5281                 $usedpath->{$path} = 1;
5282             }
5283
5284             next if drive_is_cdrom($drive);
5285             next if !$volid_hash->{$volid};
5286
5287             $drive->{size} = $volid_hash->{$volid}->{size};
5288             my $new = print_drive($vmid, $drive);
5289             if ($new ne $conf->{$opt}) {
5290                 $changes = 1;
5291                 $conf->{$opt} = $new;
5292             }
5293         }
5294     }
5295
5296     # remove 'unusedX' entry if volume is used
5297     foreach my $opt (keys %$conf) {
5298         next if $opt !~ m/^unused\d+$/;
5299         my $volid = $conf->{$opt};
5300         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5301         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5302             $changes = 1;
5303             delete $conf->{$opt};
5304         }
5305     }
5306
5307     foreach my $volid (sort keys %$volid_hash) {
5308         next if $volid =~ m/vm-$vmid-state-/;
5309         next if $used->{$volid};
5310         my $path = $volid_hash->{$volid}->{path};
5311         next if !$path; # just to be sure
5312         next if $usedpath->{$path};
5313         $changes = 1;
5314         PVE::QemuConfig->add_unused_volume($conf, $volid);
5315         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5316     }
5317
5318     return $changes;
5319 }
5320
5321 sub rescan {
5322     my ($vmid, $nolock) = @_;
5323
5324     my $cfg = PVE::Storage::config();
5325
5326     my $volid_hash = scan_volids($cfg, $vmid);
5327
5328     my $updatefn =  sub {
5329         my ($vmid) = @_;
5330
5331         my $conf = PVE::QemuConfig->load_config($vmid);
5332
5333         PVE::QemuConfig->check_lock($conf);
5334
5335         my $vm_volids = {};
5336         foreach my $volid (keys %$volid_hash) {
5337             my $info = $volid_hash->{$volid};
5338             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5339         }
5340
5341         my $changes = update_disksize($vmid, $conf, $vm_volids);
5342
5343         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5344     };
5345
5346     if (defined($vmid)) {
5347         if ($nolock) {
5348             &$updatefn($vmid);
5349         } else {
5350             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5351         }
5352     } else {
5353         my $vmlist = config_list();
5354         foreach my $vmid (keys %$vmlist) {
5355             if ($nolock) {
5356                 &$updatefn($vmid);
5357             } else {
5358                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5359             }
5360         }
5361     }
5362 }
5363
5364 sub restore_vma_archive {
5365     my ($archive, $vmid, $user, $opts, $comp) = @_;
5366
5367     my $input = $archive eq '-' ? "<&STDIN" : undef;
5368     my $readfrom = $archive;
5369
5370     my $uncomp = '';
5371     if ($comp) {
5372         $readfrom = '-';
5373         my $qarchive = PVE::Tools::shellquote($archive);
5374         if ($comp eq 'gzip') {
5375             $uncomp = "zcat $qarchive|";
5376         } elsif ($comp eq 'lzop') {
5377             $uncomp = "lzop -d -c $qarchive|";
5378         } else {
5379             die "unknown compression method '$comp'\n";
5380         }
5381
5382     }
5383
5384     my $tmpdir = "/var/tmp/vzdumptmp$$";
5385     rmtree $tmpdir;
5386
5387     # disable interrupts (always do cleanups)
5388     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5389         warn "got interrupt - ignored\n";
5390     };
5391
5392     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5393     POSIX::mkfifo($mapfifo, 0600);
5394     my $fifofh;
5395
5396     my $openfifo = sub {
5397         open($fifofh, '>', $mapfifo) || die $!;
5398     };
5399
5400     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5401
5402     my $oldtimeout;
5403     my $timeout = 5;
5404
5405     my $devinfo = {};
5406
5407     my $rpcenv = PVE::RPCEnvironment::get();
5408
5409     my $conffile = PVE::QemuConfig->config_file($vmid);
5410     my $tmpfn = "$conffile.$$.tmp";
5411
5412     # Note: $oldconf is undef if VM does not exists
5413     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5414     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5415
5416     my $print_devmap = sub {
5417         my $virtdev_hash = {};
5418
5419         my $cfgfn = "$tmpdir/qemu-server.conf";
5420
5421         # we can read the config - that is already extracted
5422         my $fh = IO::File->new($cfgfn, "r") ||
5423             "unable to read qemu-server.conf - $!\n";
5424
5425         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5426         if (-f $fwcfgfn) {
5427             my $pve_firewall_dir = '/etc/pve/firewall';
5428             mkdir $pve_firewall_dir; # make sure the dir exists
5429             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5430         }
5431
5432         while (defined(my $line = <$fh>)) {
5433             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5434                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5435                 die "archive does not contain data for drive '$virtdev'\n"
5436                     if !$devinfo->{$devname};
5437                 if (defined($opts->{storage})) {
5438                     $storeid = $opts->{storage} || 'local';
5439                 } elsif (!$storeid) {
5440                     $storeid = 'local';
5441                 }
5442                 $format = 'raw' if !$format;
5443                 $devinfo->{$devname}->{devname} = $devname;
5444                 $devinfo->{$devname}->{virtdev} = $virtdev;
5445                 $devinfo->{$devname}->{format} = $format;
5446                 $devinfo->{$devname}->{storeid} = $storeid;
5447
5448                 # check permission on storage
5449                 my $pool = $opts->{pool}; # todo: do we need that?
5450                 if ($user ne 'root@pam') {
5451                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5452                 }
5453
5454                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5455             }
5456         }
5457
5458         foreach my $devname (keys %$devinfo) {
5459             die "found no device mapping information for device '$devname'\n"
5460                 if !$devinfo->{$devname}->{virtdev};
5461         }
5462
5463         my $cfg = PVE::Storage::config();
5464
5465         # create empty/temp config
5466         if ($oldconf) {
5467             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5468             foreach_drive($oldconf, sub {
5469                 my ($ds, $drive) = @_;
5470
5471                 return if drive_is_cdrom($drive);
5472
5473                 my $volid = $drive->{file};
5474
5475                 return if !$volid || $volid =~ m|^/|;
5476
5477                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5478                 return if !$path || !$owner || ($owner != $vmid);
5479
5480                 # Note: only delete disk we want to restore
5481                 # other volumes will become unused
5482                 if ($virtdev_hash->{$ds}) {
5483                     PVE::Storage::vdisk_free($cfg, $volid);
5484                 }
5485             });
5486
5487             # delete vmstate files
5488             # since after the restore we have no snapshots anymore
5489             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5490                 my $snap = $oldconf->{snapshots}->{$snapname};
5491                 if ($snap->{vmstate}) {
5492                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5493                     if (my $err = $@) {
5494                         warn $err;
5495                     }
5496                 }
5497             }
5498         }
5499
5500         my $map = {};
5501         foreach my $virtdev (sort keys %$virtdev_hash) {
5502             my $d = $virtdev_hash->{$virtdev};
5503             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5504             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5505
5506             # test if requested format is supported
5507             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5508             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5509             $d->{format} = $defFormat if !$supported;
5510
5511             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5512                                                   $d->{format}, undef, $alloc_size);
5513             print STDERR "new volume ID is '$volid'\n";
5514             $d->{volid} = $volid;
5515             my $path = PVE::Storage::path($cfg, $volid);
5516
5517             PVE::Storage::activate_volumes($cfg,[$volid]);
5518
5519             my $write_zeros = 1;
5520             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5521                 $write_zeros = 0;
5522             }
5523
5524             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5525
5526             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5527             $map->{$virtdev} = $volid;
5528         }
5529
5530         $fh->seek(0, 0) || die "seek failed - $!\n";
5531
5532         my $outfd = new IO::File ($tmpfn, "w") ||
5533             die "unable to write config for VM $vmid\n";
5534
5535         my $cookie = { netcount => 0 };
5536         while (defined(my $line = <$fh>)) {
5537             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5538         }
5539
5540         $fh->close();
5541         $outfd->close();
5542     };
5543
5544     eval {
5545         # enable interrupts
5546         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5547             die "interrupted by signal\n";
5548         };
5549         local $SIG{ALRM} = sub { die "got timeout\n"; };
5550
5551         $oldtimeout = alarm($timeout);
5552
5553         my $parser = sub {
5554             my $line = shift;
5555
5556             print "$line\n";
5557
5558             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5559                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5560                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5561             } elsif ($line =~ m/^CTIME: /) {
5562                 # we correctly received the vma config, so we can disable
5563                 # the timeout now for disk allocation (set to 10 minutes, so
5564                 # that we always timeout if something goes wrong)
5565                 alarm(600);
5566                 &$print_devmap();
5567                 print $fifofh "done\n";
5568                 my $tmp = $oldtimeout || 0;
5569                 $oldtimeout = undef;
5570                 alarm($tmp);
5571                 close($fifofh);
5572             }
5573         };
5574
5575         print "restore vma archive: $cmd\n";
5576         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5577     };
5578     my $err = $@;
5579
5580     alarm($oldtimeout) if $oldtimeout;
5581
5582     my $vollist = [];
5583     foreach my $devname (keys %$devinfo) {
5584         my $volid = $devinfo->{$devname}->{volid};
5585         push @$vollist, $volid if $volid;
5586     }
5587
5588     my $cfg = PVE::Storage::config();
5589     PVE::Storage::deactivate_volumes($cfg, $vollist);
5590
5591     unlink $mapfifo;
5592
5593     if ($err) {
5594         rmtree $tmpdir;
5595         unlink $tmpfn;
5596
5597         foreach my $devname (keys %$devinfo) {
5598             my $volid = $devinfo->{$devname}->{volid};
5599             next if !$volid;
5600             eval {
5601                 if ($volid =~ m|^/|) {
5602                     unlink $volid || die 'unlink failed\n';
5603                 } else {
5604                     PVE::Storage::vdisk_free($cfg, $volid);
5605                 }
5606                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5607             };
5608             print STDERR "unable to cleanup '$volid' - $@" if $@;
5609         }
5610         die $err;
5611     }
5612
5613     rmtree $tmpdir;
5614
5615     rename($tmpfn, $conffile) ||
5616         die "unable to commit configuration file '$conffile'\n";
5617
5618     PVE::Cluster::cfs_update(); # make sure we read new file
5619
5620     eval { rescan($vmid, 1); };
5621     warn $@ if $@;
5622 }
5623
5624 sub restore_tar_archive {
5625     my ($archive, $vmid, $user, $opts) = @_;
5626
5627     if ($archive ne '-') {
5628         my $firstfile = tar_archive_read_firstfile($archive);
5629         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5630             if $firstfile ne 'qemu-server.conf';
5631     }
5632
5633     my $storecfg = PVE::Storage::config();
5634
5635     # destroy existing data - keep empty config
5636     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5637     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5638
5639     my $tocmd = "/usr/lib/qemu-server/qmextract";
5640
5641     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5642     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5643     $tocmd .= ' --prealloc' if $opts->{prealloc};
5644     $tocmd .= ' --info' if $opts->{info};
5645
5646     # tar option "xf" does not autodetect compression when read from STDIN,
5647     # so we pipe to zcat
5648     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5649         PVE::Tools::shellquote("--to-command=$tocmd");
5650
5651     my $tmpdir = "/var/tmp/vzdumptmp$$";
5652     mkpath $tmpdir;
5653
5654     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5655     local $ENV{VZDUMP_VMID} = $vmid;
5656     local $ENV{VZDUMP_USER} = $user;
5657
5658     my $conffile = PVE::QemuConfig->config_file($vmid);
5659     my $tmpfn = "$conffile.$$.tmp";
5660
5661     # disable interrupts (always do cleanups)
5662     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5663         print STDERR "got interrupt - ignored\n";
5664     };
5665
5666     eval {
5667         # enable interrupts
5668         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5669             die "interrupted by signal\n";
5670         };
5671
5672         if ($archive eq '-') {
5673             print "extracting archive from STDIN\n";
5674             run_command($cmd, input => "<&STDIN");
5675         } else {
5676             print "extracting archive '$archive'\n";
5677             run_command($cmd);
5678         }
5679
5680         return if $opts->{info};
5681
5682         # read new mapping
5683         my $map = {};
5684         my $statfile = "$tmpdir/qmrestore.stat";
5685         if (my $fd = IO::File->new($statfile, "r")) {
5686             while (defined (my $line = <$fd>)) {
5687                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5688                     $map->{$1} = $2 if $1;
5689                 } else {
5690                     print STDERR "unable to parse line in statfile - $line\n";
5691                 }
5692             }
5693             $fd->close();
5694         }
5695
5696         my $confsrc = "$tmpdir/qemu-server.conf";
5697
5698         my $srcfd = new IO::File($confsrc, "r") ||
5699             die "unable to open file '$confsrc'\n";
5700
5701         my $outfd = new IO::File ($tmpfn, "w") ||
5702             die "unable to write config for VM $vmid\n";
5703
5704         my $cookie = { netcount => 0 };
5705         while (defined (my $line = <$srcfd>)) {
5706             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5707         }
5708
5709         $srcfd->close();
5710         $outfd->close();
5711     };
5712     my $err = $@;
5713
5714     if ($err) {
5715
5716         unlink $tmpfn;
5717
5718         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5719
5720         die $err;
5721     }
5722
5723     rmtree $tmpdir;
5724
5725     rename $tmpfn, $conffile ||
5726         die "unable to commit configuration file '$conffile'\n";
5727
5728     PVE::Cluster::cfs_update(); # make sure we read new file
5729
5730     eval { rescan($vmid, 1); };
5731     warn $@ if $@;
5732 };
5733
5734 sub foreach_writable_storage {
5735     my ($conf, $func) = @_;
5736
5737     my $sidhash = {};
5738
5739     foreach my $ds (keys %$conf) {
5740         next if !is_valid_drivename($ds);
5741
5742         my $drive = parse_drive($ds, $conf->{$ds});
5743         next if !$drive;
5744         next if drive_is_cdrom($drive);
5745
5746         my $volid = $drive->{file};
5747
5748         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5749         $sidhash->{$sid} = $sid if $sid;
5750     }
5751
5752     foreach my $sid (sort keys %$sidhash) {
5753         &$func($sid);
5754     }
5755 }
5756
5757 sub do_snapshots_with_qemu {
5758     my ($storecfg, $volid) = @_;
5759
5760     my $storage_name = PVE::Storage::parse_volume_id($volid);
5761
5762     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5763         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5764         return 1;
5765     }
5766
5767     if ($volid =~ m/\.(qcow2|qed)$/){
5768         return 1;
5769     }
5770
5771     return undef;
5772 }
5773
5774 sub qga_check_running {
5775     my ($vmid) = @_;
5776
5777     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5778     if ($@) {
5779         warn "Qemu Guest Agent are not running - $@";
5780         return 0;
5781     }
5782     return 1;
5783 }
5784
5785 sub template_create {
5786     my ($vmid, $conf, $disk) = @_;
5787
5788     my $storecfg = PVE::Storage::config();
5789
5790     foreach_drive($conf, sub {
5791         my ($ds, $drive) = @_;
5792
5793         return if drive_is_cdrom($drive);
5794         return if $disk && $ds ne $disk;
5795
5796         my $volid = $drive->{file};
5797         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5798
5799         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5800         $drive->{file} = $voliddst;
5801         $conf->{$ds} = print_drive($vmid, $drive);
5802         PVE::QemuConfig->write_config($vmid, $conf);
5803     });
5804 }
5805
5806 sub qemu_img_convert {
5807     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5808
5809     my $storecfg = PVE::Storage::config();
5810     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5811     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5812
5813     if ($src_storeid && $dst_storeid) {
5814
5815         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5816
5817         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5818         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5819
5820         my $src_format = qemu_img_format($src_scfg, $src_volname);
5821         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5822
5823         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5824         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5825
5826         my $cmd = [];
5827         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5828         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5829         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5830         if ($is_zero_initialized) {
5831             push @$cmd, "zeroinit:$dst_path";
5832         } else {
5833             push @$cmd, $dst_path;
5834         }
5835
5836         my $parser = sub {
5837             my $line = shift;
5838             if($line =~ m/\((\S+)\/100\%\)/){
5839                 my $percent = $1;
5840                 my $transferred = int($size * $percent / 100);
5841                 my $remaining = $size - $transferred;
5842
5843                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5844             }
5845
5846         };
5847
5848         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5849         my $err = $@;
5850         die "copy failed: $err" if $err;
5851     }
5852 }
5853
5854 sub qemu_img_format {
5855     my ($scfg, $volname) = @_;
5856
5857     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5858         return $1;
5859     } else {
5860         return "raw";
5861     }
5862 }
5863
5864 sub qemu_drive_mirror {
5865     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5866
5867     my $storecfg = PVE::Storage::config();
5868     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5869
5870     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5871
5872     my $format = qemu_img_format($dst_scfg, $dst_volname);
5873
5874     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5875
5876     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5877
5878     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5879     $opts->{format} = $format if $format;
5880
5881     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5882
5883     eval {
5884     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5885         while (1) {
5886             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5887             my $stat = @$stats[0];
5888             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5889             die "error job is not mirroring" if $stat->{type} ne "mirror";
5890
5891             my $busy = $stat->{busy};
5892             my $ready = $stat->{ready};
5893
5894             if (my $total = $stat->{len}) {
5895                 my $transferred = $stat->{offset} || 0;
5896                 my $remaining = $total - $transferred;
5897                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5898
5899                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5900             }
5901
5902
5903             if ($stat->{ready} eq 'true') {
5904
5905                 last if $vmiddst != $vmid;
5906
5907                 # try to switch the disk if source and destination are on the same guest
5908                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5909                 last if !$@;
5910                 die $@ if $@ !~ m/cannot be completed/;
5911             }
5912             sleep 1;
5913         }
5914
5915
5916     };
5917     my $err = $@;
5918
5919     my $cancel_job = sub {
5920         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5921         while (1) {
5922             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5923             my $stat = @$stats[0];
5924             last if !$stat;
5925             sleep 1;
5926         }
5927     };
5928
5929     if ($err) {
5930         eval { &$cancel_job(); };
5931         die "mirroring error: $err";
5932     }
5933
5934     if ($vmiddst != $vmid) {
5935         # if we clone a disk for a new target vm, we don't switch the disk
5936         &$cancel_job(); # so we call block-job-cancel
5937     }
5938 }
5939
5940 sub clone_disk {
5941     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5942         $newvmid, $storage, $format, $full, $newvollist) = @_;
5943
5944     my $newvolid;
5945
5946     if (!$full) {
5947         print "create linked clone of drive $drivename ($drive->{file})\n";
5948         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5949         push @$newvollist, $newvolid;
5950     } else {
5951         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5952         $storeid = $storage if $storage;
5953
5954         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5955         if (!$format) {
5956             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5957             $format = qemu_img_format($scfg, $volname);
5958         }
5959
5960         # test if requested format is supported - else use default
5961         my $supported = grep { $_ eq $format } @$validFormats;
5962         $format = $defFormat if !$supported;
5963
5964         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5965
5966         print "create full clone of drive $drivename ($drive->{file})\n";
5967         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5968         push @$newvollist, $newvolid;
5969
5970         PVE::Storage::activate_volumes($storecfg, $newvollist);
5971
5972         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5973         if (!$running || $snapname) {
5974             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5975         } else {
5976             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5977         }
5978     }
5979
5980     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5981
5982     my $disk = $drive;
5983     $disk->{format} = undef;
5984     $disk->{file} = $newvolid;
5985     $disk->{size} = $size;
5986
5987     return $disk;
5988 }
5989
5990 # this only works if VM is running
5991 sub get_current_qemu_machine {
5992     my ($vmid) = @_;
5993
5994     my $cmd = { execute => 'query-machines', arguments => {} };
5995     my $res = vm_qmp_command($vmid, $cmd);
5996
5997     my ($current, $default);
5998     foreach my $e (@$res) {
5999         $default = $e->{name} if $e->{'is-default'};
6000         $current = $e->{name} if $e->{'is-current'};
6001     }
6002
6003     # fallback to the default machine if current is not supported by qemu
6004     return $current || $default || 'pc';
6005 }
6006
6007 sub qemu_machine_feature_enabled {
6008     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6009
6010     my $current_major;
6011     my $current_minor;
6012
6013     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6014
6015         $current_major = $3;
6016         $current_minor = $4;
6017
6018     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6019
6020         $current_major = $1;
6021         $current_minor = $2;
6022     }
6023
6024     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6025
6026
6027 }
6028
6029 sub qemu_machine_pxe {
6030     my ($vmid, $conf, $machine) = @_;
6031
6032     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6033
6034     foreach my $opt (keys %$conf) {
6035         next if $opt !~ m/^net(\d+)$/;
6036         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6037         next if !$net;
6038         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6039         return $machine.".pxe" if $romfile =~ m/pxe/;
6040         last;
6041     }
6042
6043     return $machine;
6044 }
6045
6046 sub qemu_use_old_bios_files {
6047     my ($machine_type) = @_;
6048
6049     return if !$machine_type;
6050
6051     my $use_old_bios_files = undef;
6052
6053     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6054         $machine_type = $1;
6055         $use_old_bios_files = 1;
6056     } else {
6057         my $kvmver = kvm_user_version();
6058         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6059         # load new efi bios files on migration. So this hack is required to allow
6060         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6061         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6062         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6063     }
6064
6065     return ($use_old_bios_files, $machine_type);
6066 }
6067
6068 sub lspci {
6069
6070     my $devices = {};
6071
6072     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6073             my (undef, $id, $function) = @_;
6074             my $res = { id => $id, function => $function};
6075             push @{$devices->{$id}}, $res;
6076     });
6077
6078     return $devices;
6079 }
6080
6081 sub vm_iothreads_list {
6082     my ($vmid) = @_;
6083
6084     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6085
6086     my $iothreads = {};
6087     foreach my $iothread (@$res) {
6088         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6089     }
6090
6091     return $iothreads;
6092 }
6093
6094 sub scsihw_infos {
6095     my ($conf, $drive) = @_;
6096
6097     my $maxdev = 0;
6098
6099     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6100         $maxdev = 7;
6101     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6102         $maxdev = 1;
6103     } else {
6104         $maxdev = 256;
6105     }
6106
6107     my $controller = int($drive->{index} / $maxdev);
6108     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6109
6110     return ($maxdev, $controller, $controller_prefix);
6111 }
6112
6113 # bash completion helper
6114
6115 sub complete_backup_archives {
6116     my ($cmdname, $pname, $cvalue) = @_;
6117
6118     my $cfg = PVE::Storage::config();
6119
6120     my $storeid;
6121
6122     if ($cvalue =~ m/^([^:]+):/) {
6123         $storeid = $1;
6124     }
6125
6126     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6127
6128     my $res = [];
6129     foreach my $id (keys %$data) {
6130         foreach my $item (@{$data->{$id}}) {
6131             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6132             push @$res, $item->{volid} if defined($item->{volid});
6133         }
6134     }
6135
6136     return $res;
6137 }
6138
6139 my $complete_vmid_full = sub {
6140     my ($running) = @_;
6141
6142     my $idlist = vmstatus();
6143
6144     my $res = [];
6145
6146     foreach my $id (keys %$idlist) {
6147         my $d = $idlist->{$id};
6148         if (defined($running)) {
6149             next if $d->{template};
6150             next if $running && $d->{status} ne 'running';
6151             next if !$running && $d->{status} eq 'running';
6152         }
6153         push @$res, $id;
6154
6155     }
6156     return $res;
6157 };
6158
6159 sub complete_vmid {
6160     return &$complete_vmid_full();
6161 }
6162
6163 sub complete_vmid_stopped {
6164     return &$complete_vmid_full(0);
6165 }
6166
6167 sub complete_vmid_running {
6168     return &$complete_vmid_full(1);
6169 }
6170
6171 sub complete_storage {
6172
6173     my $cfg = PVE::Storage::config();
6174     my $ids = $cfg->{ids};
6175
6176     my $res = [];
6177     foreach my $sid (keys %$ids) {
6178         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6179         next if !$ids->{$sid}->{content}->{images};
6180         push @$res, $sid;
6181     }
6182
6183     return $res;
6184 }
6185
6186 1;