PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use Time::HiRes qw(gettimeofday);
  34 use File::Copy qw(copy);
  35
  36 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  37
  38 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  39
  40 # Note about locking: we use flock on the config file protect
  41 # against concurent actions.
  42 # Aditionaly, we have a 'lock' setting in the config file. This
  43 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  44 # allowed when such lock is set. But you can ignore this kind of
  45 # lock with the --skiplock flag.
  46
  47 cfs_register_file('/qemu-server/',
  48                   \&parse_vm_config,
  49                   \&write_vm_config);
  50
  51 PVE::JSONSchema::register_standard_option('skiplock', {
  52     description => "Ignore locks - only root is allowed to use this option.",
  53     type => 'boolean',
  54     optional => 1,
  55 });
  56
  57 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  58     description => "Some command save/restore state from this location.",
  59     type => 'string',
  60     maxLength => 128,
  61     optional => 1,
  62 });
  63
  64 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  65     description => "The name of the snapshot.",
  66     type => 'string', format => 'pve-configid',
  67     maxLength => 40,
  68 });
  69
  70 #no warnings 'redefine';
  71
  72 sub cgroups_write {
  73    my ($controller, $vmid, $option, $value) = @_;
  74
  75    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  76    PVE::ProcFSTools::write_proc_entry($path, $value);
  77
  78 }
  79
  80 my $nodename = PVE::INotify::nodename();
  81
  82 mkdir "/etc/pve/nodes/$nodename";
  83 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  84 mkdir $confdir;
  85
  86 my $var_run_tmpdir = "/var/run/qemu-server";
  87 mkdir $var_run_tmpdir;
  88
  89 my $lock_dir = "/var/lock/qemu-server";
  90 mkdir $lock_dir;
  91
  92 my $pcisysfs = "/sys/bus/pci";
  93
  94 my $cpudesc = {
  95     cputype => {
  96         description => "Emulated CPU type.",
  97         type => 'string',
  98         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
  99         default => 'kvm64',
 100         default_key => 1,
 101     },
 102     hidden => {
 103         description => "Do not identify as a KVM virtual machine.",
 104         type => 'boolean',
 105         optional => 1,
 106         default => 0
 107     },
 108 };
 109
 110 my $confdesc = {
 111     onboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Specifies whether a VM will be started during system bootup.",
 115         default => 0,
 116     },
 117     autostart => {
 118         optional => 1,
 119         type => 'boolean',
 120         description => "Automatic restart after crash (currently ignored).",
 121         default => 0,
 122     },
 123     hotplug => {
 124         optional => 1,
 125         type => 'string', format => 'pve-hotplug-features',
 126         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 127         default => 'network,disk,usb',
 128     },
 129     reboot => {
 130         optional => 1,
 131         type => 'boolean',
 132         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 133         default => 1,
 134     },
 135     lock => {
 136         optional => 1,
 137         type => 'string',
 138         description => "Lock/unlock the VM.",
 139         enum => [qw(migrate backup snapshot rollback)],
 140     },
 141     cpulimit => {
 142         optional => 1,
 143         type => 'number',
 144         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 145         minimum => 0,
 146         maximum => 128,
 147         default => 0,
 148     },
 149     cpuunits => {
 150         optional => 1,
 151         type => 'integer',
 152         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 153         minimum => 0,
 154         maximum => 500000,
 155         default => 1000,
 156     },
 157     memory => {
 158         optional => 1,
 159         type => 'integer',
 160         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 161         minimum => 16,
 162         default => 512,
 163     },
 164     balloon => {
 165         optional => 1,
 166         type => 'integer',
 167         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 168         minimum => 0,
 169     },
 170     shares => {
 171         optional => 1,
 172         type => 'integer',
 173         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 174         minimum => 0,
 175         maximum => 50000,
 176         default => 1000,
 177     },
 178     keyboard => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 182         enum => PVE::Tools::kvmkeymaplist(),
 183         default => 'en-us',
 184     },
 185     name => {
 186         optional => 1,
 187         type => 'string', format => 'dns-name',
 188         description => "Set a name for the VM. Only used on the configuration web interface.",
 189     },
 190     scsihw => {
 191         optional => 1,
 192         type => 'string',
 193         description => "scsi controller model",
 194         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 195         default => 'lsi',
 196     },
 197     description => {
 198         optional => 1,
 199         type => 'string',
 200         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 201     },
 202     ostype => {
 203         optional => 1,
 204         type => 'string',
 205         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 206         description => <<EODESC,
 207 Used to enable special optimization/features for specific
 208 operating systems:
 209
 210 other  => unspecified OS
 211 wxp    => Microsoft Windows XP
 212 w2k    => Microsoft Windows 2000
 213 w2k3   => Microsoft Windows 2003
 214 w2k8   => Microsoft Windows 2008
 215 wvista => Microsoft Windows Vista
 216 win7   => Microsoft Windows 7
 217 win8   => Microsoft Windows 8/2012
 218 l24    => Linux 2.4 Kernel
 219 l26    => Linux 2.6/3.X Kernel
 220 solaris => solaris/opensolaris/openindiania kernel
 221
 222 other|l24|l26|solaris                       ... no special behaviour
 223 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 224 EODESC
 225     },
 226     boot => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 230         pattern => '[acdn]{1,4}',
 231         default => 'cdn',
 232     },
 233     bootdisk => {
 234         optional => 1,
 235         type => 'string', format => 'pve-qm-bootdisk',
 236         description => "Enable booting from specified disk.",
 237         pattern => '(ide|sata|scsi|virtio)\d+',
 238     },
 239     smp => {
 240         optional => 1,
 241         type => 'integer',
 242         description => "The number of CPUs. Please use option -sockets instead.",
 243         minimum => 1,
 244         default => 1,
 245     },
 246     sockets => {
 247         optional => 1,
 248         type => 'integer',
 249         description => "The number of CPU sockets.",
 250         minimum => 1,
 251         default => 1,
 252     },
 253     cores => {
 254         optional => 1,
 255         type => 'integer',
 256         description => "The number of cores per socket.",
 257         minimum => 1,
 258         default => 1,
 259     },
 260     numa => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Enable/disable Numa.",
 264         default => 0,
 265     },
 266     vcpus => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "Number of hotplugged vcpus.",
 270         minimum => 1,
 271         default => 0,
 272     },
 273     acpi => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable ACPI.",
 277         default => 1,
 278     },
 279     agent => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Enable/disable Qemu GuestAgent.",
 283         default => 0,
 284     },
 285     kvm => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Enable/disable KVM hardware virtualization.",
 289         default => 1,
 290     },
 291     tdf => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Enable/disable time drift fix.",
 295         default => 0,
 296     },
 297     localtime => {
 298         optional => 1,
 299         type => 'boolean',
 300         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 301     },
 302     freeze => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 306     },
 307     vga => {
 308         optional => 1,
 309         type => 'string',
 310         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 311         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 312     },
 313     watchdog => {
 314         optional => 1,
 315         type => 'string', format => 'pve-qm-watchdog',
 316         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 317         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 318     },
 319     startdate => {
 320         optional => 1,
 321         type => 'string',
 322         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 323         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 324         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 325         default => 'now',
 326     },
 327     startup =>  get_standard_option('pve-startup-order'),
 328     template => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Enable/disable Template.",
 332         default => 0,
 333     },
 334     args => {
 335         optional => 1,
 336         type => 'string',
 337         description => <<EODESCR,
 338 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 339
 340 args: -no-reboot -no-hpet
 341 EODESCR
 342     },
 343     tablet => {
 344         optional => 1,
 345         type => 'boolean',
 346         default => 1,
 347         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 348     },
 349     migrate_speed => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 353         minimum => 0,
 354         default => 0,
 355     },
 356     migrate_downtime => {
 357         optional => 1,
 358         type => 'number',
 359         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 360         minimum => 0,
 361         default => 0.1,
 362     },
 363     cdrom => {
 364         optional => 1,
 365         type => 'string', format => 'pve-qm-drive',
 366         typetext => 'volume',
 367         description => "This is an alias for option -ide2",
 368     },
 369     cpu => {
 370         optional => 1,
 371         description => "Emulated CPU type.",
 372         type => 'string',
 373         format => $cpudesc,
 374     },
 375     parent => get_standard_option('pve-snapshot-name', {
 376         optional => 1,
 377         description => "Parent snapshot name. This is used internally, and should not be modified.",
 378     }),
 379     snaptime => {
 380         optional => 1,
 381         description => "Timestamp for snapshots.",
 382         type => 'integer',
 383         minimum => 0,
 384     },
 385     vmstate => {
 386         optional => 1,
 387         type => 'string', format => 'pve-volume-id',
 388         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 389     },
 390     machine => {
 391         description => "Specific the Qemu machine type.",
 392         type => 'string',
 393         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 394         maxLength => 40,
 395         optional => 1,
 396     },
 397     smbios1 => {
 398         description => "Specify SMBIOS type 1 fields.",
 399         type => 'string', format => 'pve-qm-smbios1',
 400         maxLength => 256,
 401         optional => 1,
 402     },
 403     protection => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 407         default => 0,
 408     },
 409     bios => {
 410         optional => 1,
 411         type => 'string',
 412         enum => [ qw(seabios ovmf) ],
 413         description => "Select BIOS implementation.",
 414         default => 'seabios',
 415     },
 416 };
 417
 418 # what about other qemu settings ?
 419 #cpu => 'string',
 420 #machine => 'string',
 421 #fda => 'file',
 422 #fdb => 'file',
 423 #mtdblock => 'file',
 424 #sd => 'file',
 425 #pflash => 'file',
 426 #snapshot => 'bool',
 427 #bootp => 'file',
 428 ##tftp => 'dir',
 429 ##smb => 'dir',
 430 #kernel => 'file',
 431 #append => 'string',
 432 #initrd => 'file',
 433 ##soundhw => 'string',
 434
 435 while (my ($k, $v) = each %$confdesc) {
 436     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 437 }
 438
 439 my $MAX_IDE_DISKS = 4;
 440 my $MAX_SCSI_DISKS = 14;
 441 my $MAX_VIRTIO_DISKS = 16;
 442 my $MAX_SATA_DISKS = 6;
 443 my $MAX_USB_DEVICES = 5;
 444 my $MAX_NETS = 32;
 445 my $MAX_UNUSED_DISKS = 8;
 446 my $MAX_HOSTPCI_DEVICES = 4;
 447 my $MAX_SERIAL_PORTS = 4;
 448 my $MAX_PARALLEL_PORTS = 3;
 449 my $MAX_NUMA = 8;
 450 my $MAX_MEM = 4194304;
 451 my $STATICMEM = 1024;
 452
 453 my $numadesc = {
 454     optional => 1,
 455     type => 'string', format => 'pve-qm-numanode',
 456     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 457     description => "numa topology",
 458 };
 459 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 460
 461 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 462     $confdesc->{"numa$i"} = $numadesc;
 463 }
 464
 465 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 466                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 467                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 468 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 469
 470 my $netdesc = {
 471     optional => 1,
 472     type => 'string', format => 'pve-qm-net',
 473     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,trunks=<vlanid[;vlanid]>][,firewall=0|1],link_down=0|1]",
 474     description => <<EODESCR,
 475 Specify network devices.
 476
 477 MODEL is one of: $nic_model_list_txt
 478
 479 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 480 automatically generated if not specified.
 481
 482 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 483
 484 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 485
 486 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 487
 488 10.0.2.2   Gateway
 489 10.0.2.3   DNS Server
 490 10.0.2.4   SMB Server
 491
 492 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 493
 494 EODESCR
 495 };
 496 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 497
 498 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 499     $confdesc->{"net$i"} = $netdesc;
 500 }
 501
 502 my $drivename_hash;
 503
 504 my %drivedesc_base = (
 505     volume => { alias => 'file' },
 506     file => {
 507         type => 'pve-volume-id',
 508         default_key => 1,
 509         format_description => 'volume',
 510         description => "The drive's backing volume.",
 511     },
 512     media => {
 513         type => 'string',
 514         format_description => 'cdrom|disk',
 515         enum => [qw(cdrom disk)],
 516         description => "The drive's media type.",
 517         default => 'disk',
 518         optional => 1
 519     },
 520     cyls => {
 521         type => 'integer',
 522         format_description => 'count',
 523         description => "Force the drive's physical geometry to have a specific cylinder count.",
 524         optional => 1
 525     },
 526     heads => {
 527         type => 'integer',
 528         format_description => 'count',
 529         description => "Force the drive's physical geometry to have a specific head count.",
 530         optional => 1
 531     },
 532     secs => {
 533         type => 'integer',
 534         format_description => 'count',
 535         description => "Force the drive's physical geometry to have a specific sector count.",
 536         optional => 1
 537     },
 538     trans => {
 539         type => 'string',
 540         format_description => 'none|lba|auto',
 541         enum => [qw(none lba auto)],
 542         description => "Force disk geometry bios translation mode.",
 543         optional => 1,
 544     },
 545     snapshot => {
 546         type => 'boolean',
 547         format_description => 'on|off',
 548         description => "Whether the drive should be included when making snapshots.",
 549         optional => 1,
 550     },
 551     cache => {
 552         type => 'string',
 553         format_description => 'none|writethrough|writeback|unsafe|directsync',
 554         enum => [qw(none writethrough writeback unsafe directsync)],
 555         description => "The drive's cache mode",
 556         optional => 1,
 557     },
 558     format => {
 559         type => 'string',
 560         format_description => 'drive format',
 561         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 562         description => "The drive's backing file's data format.",
 563         optional => 1,
 564     },
 565     size => {
 566         type => 'string',
 567         format => 'disk-size',
 568         description => "Disk size. This is purely informational and has no effect.",
 569         optional => 1,
 570     },
 571     backup => {
 572         type => 'boolean',
 573         format_description => 'on|off',
 574         description => "Whether the drive should be included when making backups.",
 575         optional => 1,
 576     },
 577     werror => {
 578         type => 'string',
 579         format_description => 'enospc|ignore|report|stop',
 580         enum => [qw(enospc ignore report stop)],
 581         description => 'Write error action.',
 582         optional => 1,
 583     },
 584     aio => {
 585         type => 'string',
 586         format_description => 'native|threads',
 587         enum => [qw(native threads)],
 588         description => 'AIO type to use.',
 589         optional => 1,
 590     },
 591     discard => {
 592         type => 'string',
 593         format_description => 'ignore|on',
 594         enum => [qw(ignore on)],
 595         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 596         optional => 1,
 597     },
 598     detect_zeroes => {
 599         type => 'boolean',
 600         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 601         optional => 1,
 602     },
 603     serial => {
 604         type => 'string',
 605         format_description => 'serial',
 606         description => "The drive's reported serial number.",
 607         optional => 1,
 608     }
 609 );
 610
 611 my %rerror_fmt = (
 612     rerror => {
 613         type => 'string',
 614         format_description => 'ignore|report|stop',
 615         enum => [qw(ignore report stop)],
 616         description => 'Read error action.',
 617         optional => 1,
 618     },
 619 );
 620
 621 my %iothread_fmt = ( iothread => {
 622         type => 'boolean',
 623         format_description => 'off|on',
 624         description => "Whether to use iothreads for this drive",
 625         optional => 1,
 626 });
 627
 628 my %model_fmt = (
 629     model => {
 630         type => 'string',
 631         format_description => 'model',
 632         description => "The drive's reported model name.",
 633         optional => 1,
 634     },
 635 );
 636
 637 my %queues_fmt = (
 638     queues => {
 639         type => 'integer',
 640         format_description => 'nbqueues',
 641         description => "Number of queues.",
 642         minimum => 2,
 643         optional => 1
 644     }
 645 );
 646
 647 my $add_throttle_desc = sub {
 648     my ($key, $type, $what, $size, $longsize) = @_;
 649     $drivedesc_base{$key} = {
 650         type => $type,
 651         format_description => $size,
 652         description => "Maximum $what speed in $longsize per second.",
 653         optional => 1,
 654     };
 655 };
 656 # throughput: (leaky bucket)
 657 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 658 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 659 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 660 $add_throttle_desc->('mbps',    'float',   'r/w speed',   'mbps', 'megabytes');
 661 $add_throttle_desc->('mbps_rd', 'float',   'read speed',  'mbps', 'megabytes');
 662 $add_throttle_desc->('mbps_wr', 'float',   'write speed', 'mbps', 'megabytes');
 663 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 664 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 665 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 666
 667 # pools: (pool of IO before throttling starts taking effect)
 668 $add_throttle_desc->('mbps_max',    'float',   'unthrottled r/w pool',       'mbps', 'megabytes');
 669 $add_throttle_desc->('mbps_rd_max', 'float',   'unthrottled read pool',      'mbps', 'megabytes');
 670 $add_throttle_desc->('mbps_wr_max', 'float',   'unthrottled write pool',     'mbps', 'megabytes');
 671 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 672 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 673 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 674
 675 my $ide_fmt = {
 676     %drivedesc_base,
 677     %rerror_fmt,
 678     %model_fmt,
 679 };
 680
 681 my $idedesc = {
 682     optional => 1,
 683     type => 'string', format => $ide_fmt,
 684     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 685 };
 686 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 687
 688 my $scsi_fmt = {
 689     %drivedesc_base,
 690     %iothread_fmt,
 691     %queues_fmt,
 692 };
 693 my $scsidesc = {
 694     optional => 1,
 695     type => 'string', format => $scsi_fmt,
 696     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 697 };
 698 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 699
 700 my $sata_fmt = {
 701     %drivedesc_base,
 702     %rerror_fmt,
 703 };
 704 my $satadesc = {
 705     optional => 1,
 706     type => 'string', format => $sata_fmt,
 707     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 708 };
 709 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 710
 711 my $virtio_fmt = {
 712     %drivedesc_base,
 713     %iothread_fmt,
 714     %rerror_fmt,
 715 };
 716 my $virtiodesc = {
 717     optional => 1,
 718     type => 'string', format => $virtio_fmt,
 719     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 720 };
 721 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 722
 723 my $alldrive_fmt = {
 724     %drivedesc_base,
 725     %rerror_fmt,
 726     %iothread_fmt,
 727     %model_fmt,
 728     %queues_fmt,
 729 };
 730
 731 my $usbformat = {
 732     host => {
 733         default_key => 1,
 734         type => 'string', format => 'pve-qm-usb-device',
 735         format_description => 'HOSTUSBDEVICE|spice',
 736         description => 'The Host USB device or port or the value spice',
 737     },
 738     usb3 => {
 739         optional => 1,
 740         type => 'boolean',
 741         format_description => 'yes|no',
 742         description => 'Specifies whether if given host option is a USB3 device or port',
 743     },
 744 };
 745
 746 my $usbdesc = {
 747     optional => 1,
 748     type => 'string', format => $usbformat,
 749     description => <<EODESCR,
 750 Configure an USB device (n is 0 to 4). This can be used to
 751 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 752
 753 'bus-port(.port)*' (decimal numbers) or
 754 'vendor_id:product_id' (hexadeciaml numbers) or
 755 'spice'
 756
 757 You can use the 'lsusb -t' command to list existing usb devices.
 758
 759 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 760
 761 The value 'spice' can be used to add a usb redirection devices for spice.
 762
 763 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
 764
 765 EODESCR
 766 };
 767 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 768
 769 my $hostpcidesc = {
 770         optional => 1,
 771         type => 'string', format => 'pve-qm-hostpci',
 772         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 773         description => <<EODESCR,
 774 Map host pci devices. HOSTPCIDEVICE syntax is:
 775
 776 'bus:dev.func' (hexadecimal numbers)
 777
 778 You can us the 'lspci' command to list existing pci devices.
 779
 780 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 781
 782 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 783
 784 Experimental: user reported problems with this option.
 785 EODESCR
 786 };
 787 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 788
 789 my $serialdesc = {
 790         optional => 1,
 791         type => 'string',
 792         pattern => '(/dev/.+|socket)',
 793         description =>  <<EODESCR,
 794 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 795
 796 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 797
 798 Experimental: user reported problems with this option.
 799 EODESCR
 800 };
 801
 802 my $paralleldesc= {
 803         optional => 1,
 804         type => 'string',
 805         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 806         description =>  <<EODESCR,
 807 Map host parallel devices (n is 0 to 2).
 808
 809 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 810
 811 Experimental: user reported problems with this option.
 812 EODESCR
 813 };
 814
 815 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 816     $confdesc->{"parallel$i"} = $paralleldesc;
 817 }
 818
 819 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 820     $confdesc->{"serial$i"} = $serialdesc;
 821 }
 822
 823 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 824     $confdesc->{"hostpci$i"} = $hostpcidesc;
 825 }
 826
 827 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 828     $drivename_hash->{"ide$i"} = 1;
 829     $confdesc->{"ide$i"} = $idedesc;
 830 }
 831
 832 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 833     $drivename_hash->{"sata$i"} = 1;
 834     $confdesc->{"sata$i"} = $satadesc;
 835 }
 836
 837 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 838     $drivename_hash->{"scsi$i"} = 1;
 839     $confdesc->{"scsi$i"} = $scsidesc ;
 840 }
 841
 842 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 843     $drivename_hash->{"virtio$i"} = 1;
 844     $confdesc->{"virtio$i"} = $virtiodesc;
 845 }
 846
 847 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 848     $confdesc->{"usb$i"} = $usbdesc;
 849 }
 850
 851 my $unuseddesc = {
 852     optional => 1,
 853     type => 'string', format => 'pve-volume-id',
 854     description => "Reference to unused volumes.",
 855 };
 856
 857 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 858     $confdesc->{"unused$i"} = $unuseddesc;
 859 }
 860
 861 my $kvm_api_version = 0;
 862
 863 sub kvm_version {
 864
 865     return $kvm_api_version if $kvm_api_version;
 866
 867     my $fh = IO::File->new("</dev/kvm") ||
 868         return 0;
 869
 870     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 871         $kvm_api_version = $v;
 872     }
 873
 874     $fh->close();
 875
 876     return  $kvm_api_version;
 877 }
 878
 879 my $kvm_user_version;
 880
 881 sub kvm_user_version {
 882
 883     return $kvm_user_version if $kvm_user_version;
 884
 885     $kvm_user_version = 'unknown';
 886
 887     my $code = sub {
 888         my $line = shift;
 889         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 890             $kvm_user_version = $2;
 891         }
 892     };
 893
 894     eval { run_command("kvm -version", outfunc => $code); };
 895     warn $@ if $@;
 896
 897     return $kvm_user_version;
 898
 899 }
 900
 901 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 902
 903 sub valid_drive_names {
 904     # order is important - used to autoselect boot disk
 905     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 906             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 907             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 908             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 909 }
 910
 911 sub is_valid_drivename {
 912     my $dev = shift;
 913
 914     return defined($drivename_hash->{$dev});
 915 }
 916
 917 sub option_exists {
 918     my $key = shift;
 919     return defined($confdesc->{$key});
 920 }
 921
 922 sub nic_models {
 923     return $nic_model_list;
 924 }
 925
 926 sub os_list_description {
 927
 928     return {
 929         other => 'Other',
 930         wxp => 'Windows XP',
 931         w2k => 'Windows 2000',
 932         w2k3 =>, 'Windows 2003',
 933         w2k8 => 'Windows 2008',
 934         wvista => 'Windows Vista',
 935         win7 => 'Windows 7',
 936         win8 => 'Windows 8/2012',
 937         l24 => 'Linux 2.4',
 938         l26 => 'Linux 2.6',
 939     };
 940 }
 941
 942 my $cdrom_path;
 943
 944 sub get_cdrom_path {
 945
 946     return  $cdrom_path if $cdrom_path;
 947
 948     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 949     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 950     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 951 }
 952
 953 sub get_iso_path {
 954     my ($storecfg, $vmid, $cdrom) = @_;
 955
 956     if ($cdrom eq 'cdrom') {
 957         return get_cdrom_path();
 958     } elsif ($cdrom eq 'none') {
 959         return '';
 960     } elsif ($cdrom =~ m|^/|) {
 961         return $cdrom;
 962     } else {
 963         return PVE::Storage::path($storecfg, $cdrom);
 964     }
 965 }
 966
 967 # try to convert old style file names to volume IDs
 968 sub filename_to_volume_id {
 969     my ($vmid, $file, $media) = @_;
 970
 971     if (!($file eq 'none' || $file eq 'cdrom' ||
 972           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 973
 974         return undef if $file =~ m|/|;
 975
 976         if ($media && $media eq 'cdrom') {
 977             $file = "local:iso/$file";
 978         } else {
 979             $file = "local:$vmid/$file";
 980         }
 981     }
 982
 983     return $file;
 984 }
 985
 986 sub verify_media_type {
 987     my ($opt, $vtype, $media) = @_;
 988
 989     return if !$media;
 990
 991     my $etype;
 992     if ($media eq 'disk') {
 993         $etype = 'images';
 994     } elsif ($media eq 'cdrom') {
 995         $etype = 'iso';
 996     } else {
 997         die "internal error";
 998     }
 999
1000     return if ($vtype eq $etype);
1001
1002     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1003 }
1004
1005 sub cleanup_drive_path {
1006     my ($opt, $storecfg, $drive) = @_;
1007
1008     # try to convert filesystem paths to volume IDs
1009
1010     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1011         ($drive->{file} !~ m|^/dev/.+|) &&
1012         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1013         ($drive->{file} !~ m/^\d+$/)) {
1014         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1015         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1016         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1017         verify_media_type($opt, $vtype, $drive->{media});
1018         $drive->{file} = $volid;
1019     }
1020
1021     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1022 }
1023
1024 sub parse_hotplug_features {
1025     my ($data) = @_;
1026
1027     my $res = {};
1028
1029     return $res if $data eq '0';
1030
1031     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1032
1033     foreach my $feature (PVE::Tools::split_list($data)) {
1034         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1035             $res->{$1} = 1;
1036         } else {
1037             warn "ignoring unknown hotplug feature '$feature'\n";
1038         }
1039     }
1040     return $res;
1041 }
1042
1043 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1044 sub pve_verify_hotplug_features {
1045     my ($value, $noerr) = @_;
1046
1047     return $value if parse_hotplug_features($value);
1048
1049     return undef if $noerr;
1050
1051     die "unable to parse hotplug option\n";
1052 }
1053
1054 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1055 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1056 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1057 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1058 #        [,iothread=on][,serial=serial][,model=model]
1059
1060 sub parse_drive {
1061     my ($key, $data) = @_;
1062
1063     my ($interface, $index);
1064
1065     if ($key =~ m/^([^\d]+)(\d+)$/) {
1066         $interface = $1;
1067         $index = $2;
1068     } else {
1069         return undef;
1070     }
1071
1072     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1073                                      : $confdesc->{$key}->{format};
1074     if (!$desc) {
1075         warn "invalid drive key: $key\n";
1076         return undef;
1077     }
1078     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1079     return undef if !$res;
1080     $res->{interface} = $interface;
1081     $res->{index} = $index;
1082
1083     my $error = 0;
1084     foreach my $opt (qw(bps bps_rd bps_wr)) {
1085         if (my $bps = defined(delete $res->{$opt})) {
1086             if (defined($res->{"m$opt"})) {
1087                 warn "both $opt and m$opt specified\n";
1088                 ++$error;
1089                 next;
1090             }
1091             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1092         }
1093     }
1094     return undef if $error;
1095
1096     return undef if $res->{mbps_rd} && $res->{mbps};
1097     return undef if $res->{mbps_wr} && $res->{mbps};
1098     return undef if $res->{iops_rd} && $res->{iops};
1099     return undef if $res->{iops_wr} && $res->{iops};
1100
1101     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1102         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1103         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1104         return undef if $res->{interface} eq 'virtio';
1105     }
1106
1107     if (my $size = $res->{size}) {
1108         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1109     }
1110
1111     return $res;
1112 }
1113
1114 sub print_drive {
1115     my ($vmid, $drive) = @_;
1116     my $data = { %$drive };
1117     delete $data->{$_} for qw(index interface);
1118     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1119 }
1120
1121 sub scsi_inquiry {
1122     my($fh, $noerr) = @_;
1123
1124     my $SG_IO = 0x2285;
1125     my $SG_GET_VERSION_NUM = 0x2282;
1126
1127     my $versionbuf = "\x00" x 8;
1128     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1129     if (!$ret) {
1130         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1131         return undef;
1132     }
1133     my $version = unpack("I", $versionbuf);
1134     if ($version < 30000) {
1135         die "scsi generic interface too old\n"  if !$noerr;
1136         return undef;
1137     }
1138
1139     my $buf = "\x00" x 36;
1140     my $sensebuf = "\x00" x 8;
1141     my $cmd = pack("C x3 C x1", 0x12, 36);
1142
1143     # see /usr/include/scsi/sg.h
1144     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1145
1146     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1147                       length($sensebuf), 0, length($buf), $buf,
1148                       $cmd, $sensebuf, 6000);
1149
1150     $ret = ioctl($fh, $SG_IO, $packet);
1151     if (!$ret) {
1152         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1153         return undef;
1154     }
1155
1156     my @res = unpack($sg_io_hdr_t, $packet);
1157     if ($res[17] || $res[18]) {
1158         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1159         return undef;
1160     }
1161
1162     my $res = {};
1163     (my $byte0, my $byte1, $res->{vendor},
1164      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1165
1166     $res->{removable} = $byte1 & 128 ? 1 : 0;
1167     $res->{type} = $byte0 & 31;
1168
1169     return $res;
1170 }
1171
1172 sub path_is_scsi {
1173     my ($path) = @_;
1174
1175     my $fh = IO::File->new("+<$path") || return undef;
1176     my $res = scsi_inquiry($fh, 1);
1177     close($fh);
1178
1179     return $res;
1180 }
1181
1182 sub machine_type_is_q35 {
1183     my ($conf) = @_;
1184
1185     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1186 }
1187
1188 sub print_tabletdevice_full {
1189     my ($conf) = @_;
1190
1191     my $q35 = machine_type_is_q35($conf);
1192
1193     # we use uhci for old VMs because tablet driver was buggy in older qemu
1194     my $usbbus = $q35 ? "ehci" : "uhci";
1195
1196     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1197 }
1198
1199 sub print_drivedevice_full {
1200     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1201
1202     my $device = '';
1203     my $maxdev = 0;
1204
1205     if ($drive->{interface} eq 'virtio') {
1206         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1207         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1208         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1209     } elsif ($drive->{interface} eq 'scsi') {
1210
1211         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1212         my $unit = $drive->{index} % $maxdev;
1213         my $devicetype = 'hd';
1214         my $path = '';
1215         if (drive_is_cdrom($drive)) {
1216             $devicetype = 'cd';
1217         } else {
1218             if ($drive->{file} =~ m|^/|) {
1219                 $path = $drive->{file};
1220                 if (my $info = path_is_scsi($path)) {
1221                     if ($info->{type} == 0) {
1222                         $devicetype = 'block';
1223                     } elsif ($info->{type} == 1) { # tape
1224                         $devicetype = 'generic';
1225                     }
1226                 }
1227             } else {
1228                  $path = PVE::Storage::path($storecfg, $drive->{file});
1229             }
1230
1231             if($path =~ m/^iscsi\:\/\//){
1232                 $devicetype = 'generic';
1233             }
1234         }
1235
1236         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1237             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1238         } else {
1239             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1240         }
1241
1242     } elsif ($drive->{interface} eq 'ide'){
1243         $maxdev = 2;
1244         my $controller = int($drive->{index} / $maxdev);
1245         my $unit = $drive->{index} % $maxdev;
1246         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1247
1248         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1249         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1250             $device .= ",model=$model";
1251         }
1252     } elsif ($drive->{interface} eq 'sata'){
1253         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1254         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1255         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1256     } elsif ($drive->{interface} eq 'usb') {
1257         die "implement me";
1258         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1259     } else {
1260         die "unsupported interface type";
1261     }
1262
1263     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1264
1265     return $device;
1266 }
1267
1268 sub get_initiator_name {
1269     my $initiator;
1270
1271     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1272     while (defined(my $line = <$fh>)) {
1273         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1274         $initiator = $1;
1275         last;
1276     }
1277     $fh->close();
1278
1279     return $initiator;
1280 }
1281
1282 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
1283 sub print_drive_full {
1284     my ($storecfg, $vmid, $drive) = @_;
1285
1286     my $path;
1287     my $volid = $drive->{file};
1288     my $format;
1289
1290     if (drive_is_cdrom($drive)) {
1291         $path = get_iso_path($storecfg, $vmid, $volid);
1292     } else {
1293         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1294         if ($storeid) {
1295             $path = PVE::Storage::path($storecfg, $volid);
1296             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1297             $format = qemu_img_format($scfg, $volname);
1298         } else {
1299             $path = $volid;
1300             $format = "raw";
1301         }
1302    }
1303
1304     my $opts = '';
1305     foreach my $o (@qemu_drive_options) {
1306         next if $o eq 'bootindex';
1307         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1308     }
1309
1310     $opts .= ",format=$format" if $format && !$drive->{format};
1311
1312     foreach my $o (qw(bps bps_rd bps_wr)) {
1313         my $v = $drive->{"m$o"};
1314         $opts .= ",$o=" . int($v*1024*1024) if $v;
1315     }
1316
1317     my $cache_direct = 0;
1318
1319     if (my $cache = $drive->{cache}) {
1320         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1321     } elsif (!drive_is_cdrom($drive)) {
1322         $opts .= ",cache=none";
1323         $cache_direct = 1;
1324     }
1325
1326     # aio native works only with O_DIRECT
1327     if (!$drive->{aio}) {
1328         if($cache_direct) {
1329             $opts .= ",aio=native";
1330         } else {
1331             $opts .= ",aio=threads";
1332         }
1333     }
1334
1335     if (!drive_is_cdrom($drive)) {
1336         my $detectzeroes;
1337         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1338             $detectzeroes = 'off';
1339         } elsif ($drive->{discard}) {
1340             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1341         } else {
1342             # This used to be our default with discard not being specified:
1343             $detectzeroes = 'on';
1344         }
1345         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1346     }
1347
1348     my $pathinfo = $path ? "file=$path," : '';
1349
1350     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1351 }
1352
1353 sub print_netdevice_full {
1354     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1355
1356     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1357
1358     my $device = $net->{model};
1359     if ($net->{model} eq 'virtio') {
1360          $device = 'virtio-net-pci';
1361      };
1362
1363     my $pciaddr = print_pci_addr("$netid", $bridges);
1364     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1365     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1366         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1367         my $vectors = $net->{queues} * 2 + 2;
1368         $tmpstr .= ",vectors=$vectors,mq=on";
1369     }
1370     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1371
1372     if ($use_old_bios_files) {
1373         my $romfile;
1374         if ($device eq 'virtio-net-pci') {
1375             $romfile = 'pxe-virtio.rom';
1376         } elsif ($device eq 'e1000') {
1377             $romfile = 'pxe-e1000.rom';
1378         } elsif ($device eq 'ne2k') {
1379             $romfile = 'pxe-ne2k_pci.rom';
1380         } elsif ($device eq 'pcnet') {
1381             $romfile = 'pxe-pcnet.rom';
1382         } elsif ($device eq 'rtl8139') {
1383             $romfile = 'pxe-rtl8139.rom';
1384         }
1385         $tmpstr .= ",romfile=$romfile" if $romfile;
1386     }
1387
1388     return $tmpstr;
1389 }
1390
1391 sub print_netdev_full {
1392     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1393
1394     my $i = '';
1395     if ($netid =~ m/^net(\d+)$/) {
1396         $i = int($1);
1397     }
1398
1399     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1400
1401     my $ifname = "tap${vmid}i$i";
1402
1403     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1404     die "interface name '$ifname' is too long (max 15 character)\n"
1405         if length($ifname) >= 16;
1406
1407     my $vhostparam = '';
1408     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1409
1410     my $vmname = $conf->{name} || "vm$vmid";
1411
1412     my $netdev = "";
1413     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1414
1415     if ($net->{bridge}) {
1416         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1417     } else {
1418         $netdev = "type=user,id=$netid,hostname=$vmname";
1419     }
1420
1421     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1422
1423     return $netdev;
1424 }
1425
1426 sub drive_is_cdrom {
1427     my ($drive) = @_;
1428
1429     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1430
1431 }
1432
1433 sub parse_numa {
1434     my ($data) = @_;
1435
1436     my $res = {};
1437
1438     foreach my $kvp (split(/,/, $data)) {
1439
1440         if ($kvp =~ m/^memory=(\S+)$/) {
1441             $res->{memory} = $1;
1442         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1443             $res->{policy} = $1;
1444         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1445             $res->{cpus}->{start} = $1;
1446             $res->{cpus}->{end} = $3;
1447         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1448             $res->{hostnodes}->{start} = $1;
1449             $res->{hostnodes}->{end} = $3;
1450         } else {
1451             return undef;
1452         }
1453     }
1454
1455     return $res;
1456 }
1457
1458 sub parse_hostpci {
1459     my ($value) = @_;
1460
1461     return undef if !$value;
1462
1463
1464     my @list = split(/,/, $value);
1465     my $found;
1466
1467     my $res = {};
1468     foreach my $kv (@list) {
1469
1470         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1471             $found = 1;
1472             if(defined($4)){
1473                 push @{$res->{pciid}}, { id => $2 , function => $4};
1474
1475             }else{
1476                 my $pcidevices = lspci($2);
1477                 $res->{pciid} = $pcidevices->{$2};
1478             }
1479         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1480             $res->{rombar} = $1;
1481         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1482             $res->{'x-vga'} = $1;
1483         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1484             $res->{pcie} = 1 if $1 == 1;
1485         } else {
1486             warn "unknown hostpci setting '$kv'\n";
1487         }
1488     }
1489
1490     return undef if !$found;
1491
1492     return $res;
1493 }
1494
1495 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1496 sub parse_net {
1497     my ($data) = @_;
1498
1499     my $res = {};
1500
1501     foreach my $kvp (split(/,/, $data)) {
1502
1503         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1504             my $model = lc($1);
1505             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1506             $res->{model} = $model;
1507             $res->{macaddr} = $mac;
1508         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1509             $res->{bridge} = $1;
1510         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1511             $res->{queues} = $1;
1512         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1513             $res->{rate} = $1;
1514         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1515             $res->{tag} = $1;
1516         } elsif ($kvp =~ m/^trunks=([0-9;]+)$/) {
1517             $res->{trunks} = $1;
1518         } elsif ($kvp =~ m/^firewall=([01])$/) {
1519             $res->{firewall} = $1;
1520         } elsif ($kvp =~ m/^link_down=([01])$/) {
1521             $res->{link_down} = $1;
1522         } else {
1523             return undef;
1524         }
1525
1526     }
1527
1528     return undef if !$res->{model};
1529
1530     return $res;
1531 }
1532
1533 sub print_net {
1534     my $net = shift;
1535
1536     my $res = "$net->{model}";
1537     $res .= "=$net->{macaddr}" if $net->{macaddr};
1538     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1539     $res .= ",rate=$net->{rate}" if $net->{rate};
1540     $res .= ",tag=$net->{tag}" if $net->{tag};
1541     $res .= ",trunks=$net->{trunks}" if $net->{trunks};
1542     $res .= ",firewall=1" if $net->{firewall};
1543     $res .= ",link_down=1" if $net->{link_down};
1544     $res .= ",queues=$net->{queues}" if $net->{queues};
1545
1546     return $res;
1547 }
1548
1549 sub add_random_macs {
1550     my ($settings) = @_;
1551
1552     foreach my $opt (keys %$settings) {
1553         next if $opt !~ m/^net(\d+)$/;
1554         my $net = parse_net($settings->{$opt});
1555         next if !$net;
1556         $settings->{$opt} = print_net($net);
1557     }
1558 }
1559
1560 sub vm_is_volid_owner {
1561     my ($storecfg, $vmid, $volid) = @_;
1562
1563     if ($volid !~  m|^/|) {
1564         my ($path, $owner);
1565         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1566         if ($owner && ($owner == $vmid)) {
1567             return 1;
1568         }
1569     }
1570
1571     return undef;
1572 }
1573
1574 sub split_flagged_list {
1575     my $text = shift || '';
1576     $text =~ s/[,;]/ /g;
1577     $text =~ s/^\s+//;
1578     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1579 }
1580
1581 sub join_flagged_list {
1582     my ($how, $lst) = @_;
1583     join $how, map { $lst->{$_} . $_ } keys %$lst;
1584 }
1585
1586 sub vmconfig_delete_pending_option {
1587     my ($conf, $key, $force) = @_;
1588
1589     delete $conf->{pending}->{$key};
1590     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1591     $pending_delete_hash->{$key} = $force ? '!' : '';
1592     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1593 }
1594
1595 sub vmconfig_undelete_pending_option {
1596     my ($conf, $key) = @_;
1597
1598     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1599     delete $pending_delete_hash->{$key};
1600
1601     if (%$pending_delete_hash) {
1602         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1603     } else {
1604         delete $conf->{pending}->{delete};
1605     }
1606 }
1607
1608 sub vmconfig_register_unused_drive {
1609     my ($storecfg, $vmid, $conf, $drive) = @_;
1610
1611     if (!drive_is_cdrom($drive)) {
1612         my $volid = $drive->{file};
1613         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1614             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1615         }
1616     }
1617 }
1618
1619 sub vmconfig_cleanup_pending {
1620     my ($conf) = @_;
1621
1622     # remove pending changes when nothing changed
1623     my $changes;
1624     foreach my $opt (keys %{$conf->{pending}}) {
1625         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1626             $changes = 1;
1627             delete $conf->{pending}->{$opt};
1628         }
1629     }
1630
1631     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1632     my $pending_delete_hash = {};
1633     while (my ($opt, $force) = each %$current_delete_hash) {
1634         if (defined($conf->{$opt})) {
1635             $pending_delete_hash->{$opt} = $force;
1636         } else {
1637             $changes = 1;
1638         }
1639     }
1640
1641     if (%$pending_delete_hash) {
1642         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1643     } else {
1644         delete $conf->{pending}->{delete};
1645     }
1646
1647     return $changes;
1648 }
1649
1650 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1651 my $smbios1_desc = {
1652     uuid => {
1653         type => 'string',
1654         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1655         format_description => 'UUID',
1656         optional => 1,
1657     },
1658     version => {
1659         type => 'string',
1660         pattern => '\S+',
1661         format_description => 'str',
1662         optional => 1,
1663     },
1664     serial => {
1665         type => 'string',
1666         pattern => '\S+',
1667         format_description => 'str',
1668         optional => 1,
1669     },
1670     manufacturer => {
1671         type => 'string',
1672         pattern => '\S+',
1673         format_description => 'name',
1674         optional => 1,
1675     },
1676     product => {
1677         type => 'string',
1678         pattern => '\S+',
1679         format_description => 'name',
1680         optional => 1,
1681     },
1682     sku => {
1683         type => 'string',
1684         pattern => '\S+',
1685         format_description => 'str',
1686         optional => 1,
1687     },
1688     family => {
1689         type => 'string',
1690         pattern => '\S+',
1691         format_description => 'str',
1692         optional => 1,
1693     },
1694 };
1695
1696 sub parse_smbios1 {
1697     my ($data) = @_;
1698
1699     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_desc, $data) };
1700     warn $@ if $@;
1701     return $res;
1702 }
1703
1704 sub print_smbios1 {
1705     my ($smbios1) = @_;
1706     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_desc);
1707 }
1708
1709 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_desc);
1710
1711 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1712 sub verify_bootdisk {
1713     my ($value, $noerr) = @_;
1714
1715     return $value if is_valid_drivename($value);
1716
1717     return undef if $noerr;
1718
1719     die "invalid boot disk '$value'\n";
1720 }
1721
1722 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1723 sub verify_numa {
1724     my ($value, $noerr) = @_;
1725
1726     return $value if parse_numa($value);
1727
1728     return undef if $noerr;
1729
1730     die "unable to parse numa options\n";
1731 }
1732
1733 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1734 sub verify_net {
1735     my ($value, $noerr) = @_;
1736
1737     return $value if parse_net($value);
1738
1739     return undef if $noerr;
1740
1741     die "unable to parse network options\n";
1742 }
1743
1744 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1745 sub verify_hostpci {
1746     my ($value, $noerr) = @_;
1747
1748     return $value if parse_hostpci($value);
1749
1750     return undef if $noerr;
1751
1752     die "unable to parse pci id\n";
1753 }
1754
1755 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1756 sub verify_watchdog {
1757     my ($value, $noerr) = @_;
1758
1759     return $value if parse_watchdog($value);
1760
1761     return undef if $noerr;
1762
1763     die "unable to parse watchdog options\n";
1764 }
1765
1766 sub parse_watchdog {
1767     my ($value) = @_;
1768
1769     return undef if !$value;
1770
1771     my $res = {};
1772
1773     foreach my $p (split(/,/, $value)) {
1774         next if $p =~ m/^\s*$/;
1775
1776         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1777             $res->{model} = $2;
1778         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1779             $res->{action} = $2;
1780         } else {
1781             return undef;
1782         }
1783     }
1784
1785     return $res;
1786 }
1787
1788 sub parse_usb_device {
1789     my ($value) = @_;
1790
1791     return undef if !$value;
1792
1793     my $res = {};
1794     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1795         $res->{vendorid} = $2;
1796         $res->{productid} = $4;
1797     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1798         $res->{hostbus} = $1;
1799         $res->{hostport} = $2;
1800     } elsif ($value =~ m/^spice$/i) {
1801         $res->{spice} = 1;
1802     } else {
1803         return undef;
1804     }
1805
1806     return $res;
1807 }
1808
1809 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1810 sub verify_usb_device {
1811     my ($value, $noerr) = @_;
1812
1813     return $value if parse_usb_device($value);
1814
1815     return undef if $noerr;
1816
1817     die "unable to parse usb device\n";
1818 }
1819
1820 # add JSON properties for create and set function
1821 sub json_config_properties {
1822     my $prop = shift;
1823
1824     foreach my $opt (keys %$confdesc) {
1825         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1826         $prop->{$opt} = $confdesc->{$opt};
1827     }
1828
1829     return $prop;
1830 }
1831
1832 sub check_type {
1833     my ($key, $value) = @_;
1834
1835     die "unknown setting '$key'\n" if !$confdesc->{$key};
1836
1837     my $type = $confdesc->{$key}->{type};
1838
1839     if (!defined($value)) {
1840         die "got undefined value\n";
1841     }
1842
1843     if ($value =~ m/[\n\r]/) {
1844         die "property contains a line feed\n";
1845     }
1846
1847     if ($type eq 'boolean') {
1848         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1849         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1850         die "type check ('boolean') failed - got '$value'\n";
1851     } elsif ($type eq 'integer') {
1852         return int($1) if $value =~ m/^(\d+)$/;
1853         die "type check ('integer') failed - got '$value'\n";
1854     } elsif ($type eq 'number') {
1855         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1856         die "type check ('number') failed - got '$value'\n";
1857     } elsif ($type eq 'string') {
1858         if (my $fmt = $confdesc->{$key}->{format}) {
1859             if ($fmt eq 'pve-qm-drive') {
1860                 # special case - we need to pass $key to parse_drive()
1861                 my $drive = parse_drive($key, $value);
1862                 return $value if $drive;
1863                 die "unable to parse drive options\n";
1864             }
1865             PVE::JSONSchema::check_format($fmt, $value);
1866             return $value;
1867         }
1868         $value =~ s/^\"(.*)\"$/$1/;
1869         return $value;
1870     } else {
1871         die "internal error"
1872     }
1873 }
1874
1875 sub check_iommu_support{
1876     #fixme : need to check IOMMU support
1877     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1878
1879     my $iommu=1;
1880     return $iommu;
1881
1882 }
1883
1884 sub touch_config {
1885     my ($vmid) = @_;
1886
1887     my $conf = PVE::QemuConfig->config_file($vmid);
1888     utime undef, undef, $conf;
1889 }
1890
1891 sub destroy_vm {
1892     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1893
1894     my $conffile = PVE::QemuConfig->config_file($vmid);
1895
1896     my $conf = PVE::QemuConfig->load_config($vmid);
1897
1898     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1899
1900     # only remove disks owned by this VM
1901     foreach_drive($conf, sub {
1902         my ($ds, $drive) = @_;
1903
1904         return if drive_is_cdrom($drive);
1905
1906         my $volid = $drive->{file};
1907
1908         return if !$volid || $volid =~ m|^/|;
1909
1910         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1911         return if !$path || !$owner || ($owner != $vmid);
1912
1913         PVE::Storage::vdisk_free($storecfg, $volid);
1914     });
1915
1916     if ($keep_empty_config) {
1917         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1918     } else {
1919         unlink $conffile;
1920     }
1921
1922     # also remove unused disk
1923     eval {
1924         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1925
1926         eval {
1927             PVE::Storage::foreach_volid($dl, sub {
1928                 my ($volid, $sid, $volname, $d) = @_;
1929                 PVE::Storage::vdisk_free($storecfg, $volid);
1930             });
1931         };
1932         warn $@ if $@;
1933
1934     };
1935     warn $@ if $@;
1936 }
1937
1938 sub parse_vm_config {
1939     my ($filename, $raw) = @_;
1940
1941     return undef if !defined($raw);
1942
1943     my $res = {
1944         digest => Digest::SHA::sha1_hex($raw),
1945         snapshots => {},
1946         pending => {},
1947     };
1948
1949     $filename =~ m|/qemu-server/(\d+)\.conf$|
1950         || die "got strange filename '$filename'";
1951
1952     my $vmid = $1;
1953
1954     my $conf = $res;
1955     my $descr;
1956     my $section = '';
1957
1958     my @lines = split(/\n/, $raw);
1959     foreach my $line (@lines) {
1960         next if $line =~ m/^\s*$/;
1961
1962         if ($line =~ m/^\[PENDING\]\s*$/i) {
1963             $section = 'pending';
1964             if (defined($descr)) {
1965                 $descr =~ s/\s+$//;
1966                 $conf->{description} = $descr;
1967             }
1968             $descr = undef;
1969             $conf = $res->{$section} = {};
1970             next;
1971
1972         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1973             $section = $1;
1974             if (defined($descr)) {
1975                 $descr =~ s/\s+$//;
1976                 $conf->{description} = $descr;
1977             }
1978             $descr = undef;
1979             $conf = $res->{snapshots}->{$section} = {};
1980             next;
1981         }
1982
1983         if ($line =~ m/^\#(.*)\s*$/) {
1984             $descr = '' if !defined($descr);
1985             $descr .= PVE::Tools::decode_text($1) . "\n";
1986             next;
1987         }
1988
1989         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1990             $descr = '' if !defined($descr);
1991             $descr .= PVE::Tools::decode_text($2);
1992         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1993             $conf->{snapstate} = $1;
1994         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1995             my $key = $1;
1996             my $value = $2;
1997             $conf->{$key} = $value;
1998         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1999             my $value = $1;
2000             if ($section eq 'pending') {
2001                 $conf->{delete} = $value; # we parse this later
2002             } else {
2003                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2004             }
2005         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2006             my $key = $1;
2007             my $value = $2;
2008             eval { $value = check_type($key, $value); };
2009             if ($@) {
2010                 warn "vm $vmid - unable to parse value of '$key' - $@";
2011             } else {
2012                 my $fmt = $confdesc->{$key}->{format};
2013                 if ($fmt && $fmt eq 'pve-qm-drive') {
2014                     my $v = parse_drive($key, $value);
2015                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2016                         $v->{file} = $volid;
2017                         $value = print_drive($vmid, $v);
2018                     } else {
2019                         warn "vm $vmid - unable to parse value of '$key'\n";
2020                         next;
2021                     }
2022                 }
2023
2024                 if ($key eq 'cdrom') {
2025                     $conf->{ide2} = $value;
2026                 } else {
2027                     $conf->{$key} = $value;
2028                 }
2029             }
2030         }
2031     }
2032
2033     if (defined($descr)) {
2034         $descr =~ s/\s+$//;
2035         $conf->{description} = $descr;
2036     }
2037     delete $res->{snapstate}; # just to be sure
2038
2039     return $res;
2040 }
2041
2042 sub write_vm_config {
2043     my ($filename, $conf) = @_;
2044
2045     delete $conf->{snapstate}; # just to be sure
2046
2047     if ($conf->{cdrom}) {
2048         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2049         $conf->{ide2} = $conf->{cdrom};
2050         delete $conf->{cdrom};
2051     }
2052
2053     # we do not use 'smp' any longer
2054     if ($conf->{sockets}) {
2055         delete $conf->{smp};
2056     } elsif ($conf->{smp}) {
2057         $conf->{sockets} = $conf->{smp};
2058         delete $conf->{cores};
2059         delete $conf->{smp};
2060     }
2061
2062     my $used_volids = {};
2063
2064     my $cleanup_config = sub {
2065         my ($cref, $pending, $snapname) = @_;
2066
2067         foreach my $key (keys %$cref) {
2068             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2069                 $key eq 'snapstate' || $key eq 'pending';
2070             my $value = $cref->{$key};
2071             if ($key eq 'delete') {
2072                 die "propertry 'delete' is only allowed in [PENDING]\n"
2073                     if !$pending;
2074                 # fixme: check syntax?
2075                 next;
2076             }
2077             eval { $value = check_type($key, $value); };
2078             die "unable to parse value of '$key' - $@" if $@;
2079
2080             $cref->{$key} = $value;
2081
2082             if (!$snapname && is_valid_drivename($key)) {
2083                 my $drive = parse_drive($key, $value);
2084                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2085             }
2086         }
2087     };
2088
2089     &$cleanup_config($conf);
2090
2091     &$cleanup_config($conf->{pending}, 1);
2092
2093     foreach my $snapname (keys %{$conf->{snapshots}}) {
2094         die "internal error" if $snapname eq 'pending';
2095         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2096     }
2097
2098     # remove 'unusedX' settings if we re-add a volume
2099     foreach my $key (keys %$conf) {
2100         my $value = $conf->{$key};
2101         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2102             delete $conf->{$key};
2103         }
2104     }
2105
2106     my $generate_raw_config = sub {
2107         my ($conf, $pending) = @_;
2108
2109         my $raw = '';
2110
2111         # add description as comment to top of file
2112         if (defined(my $descr = $conf->{description})) {
2113             if ($descr) {
2114                 foreach my $cl (split(/\n/, $descr)) {
2115                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2116                 }
2117             } else {
2118                 $raw .= "#\n" if $pending;
2119             }
2120         }
2121
2122         foreach my $key (sort keys %$conf) {
2123             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2124             $raw .= "$key: $conf->{$key}\n";
2125         }
2126         return $raw;
2127     };
2128
2129     my $raw = &$generate_raw_config($conf);
2130
2131     if (scalar(keys %{$conf->{pending}})){
2132         $raw .= "\n[PENDING]\n";
2133         $raw .= &$generate_raw_config($conf->{pending}, 1);
2134     }
2135
2136     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2137         $raw .= "\n[$snapname]\n";
2138         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2139     }
2140
2141     return $raw;
2142 }
2143
2144 sub load_defaults {
2145
2146     my $res = {};
2147
2148     # we use static defaults from our JSON schema configuration
2149     foreach my $key (keys %$confdesc) {
2150         if (defined(my $default = $confdesc->{$key}->{default})) {
2151             $res->{$key} = $default;
2152         }
2153     }
2154
2155     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2156     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2157
2158     return $res;
2159 }
2160
2161 sub config_list {
2162     my $vmlist = PVE::Cluster::get_vmlist();
2163     my $res = {};
2164     return $res if !$vmlist || !$vmlist->{ids};
2165     my $ids = $vmlist->{ids};
2166
2167     foreach my $vmid (keys %$ids) {
2168         my $d = $ids->{$vmid};
2169         next if !$d->{node} || $d->{node} ne $nodename;
2170         next if !$d->{type} || $d->{type} ne 'qemu';
2171         $res->{$vmid}->{exists} = 1;
2172     }
2173     return $res;
2174 }
2175
2176 # test if VM uses local resources (to prevent migration)
2177 sub check_local_resources {
2178     my ($conf, $noerr) = @_;
2179
2180     my $loc_res = 0;
2181
2182     $loc_res = 1 if $conf->{hostusb}; # old syntax
2183     $loc_res = 1 if $conf->{hostpci}; # old syntax
2184
2185     foreach my $k (keys %$conf) {
2186         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2187         # sockets are safe: they will recreated be on the target side post-migrate
2188         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2189         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2190     }
2191
2192     die "VM uses local resources\n" if $loc_res && !$noerr;
2193
2194     return $loc_res;
2195 }
2196
2197 # check if used storages are available on all nodes (use by migrate)
2198 sub check_storage_availability {
2199     my ($storecfg, $conf, $node) = @_;
2200
2201     foreach_drive($conf, sub {
2202         my ($ds, $drive) = @_;
2203
2204         my $volid = $drive->{file};
2205         return if !$volid;
2206
2207         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2208         return if !$sid;
2209
2210         # check if storage is available on both nodes
2211         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2212         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2213    });
2214 }
2215
2216 # list nodes where all VM images are available (used by has_feature API)
2217 sub shared_nodes {
2218     my ($conf, $storecfg) = @_;
2219
2220     my $nodelist = PVE::Cluster::get_nodelist();
2221     my $nodehash = { map { $_ => 1 } @$nodelist };
2222     my $nodename = PVE::INotify::nodename();
2223
2224     foreach_drive($conf, sub {
2225         my ($ds, $drive) = @_;
2226
2227         my $volid = $drive->{file};
2228         return if !$volid;
2229
2230         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2231         if ($storeid) {
2232             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2233             if ($scfg->{disable}) {
2234                 $nodehash = {};
2235             } elsif (my $avail = $scfg->{nodes}) {
2236                 foreach my $node (keys %$nodehash) {
2237                     delete $nodehash->{$node} if !$avail->{$node};
2238                 }
2239             } elsif (!$scfg->{shared}) {
2240                 foreach my $node (keys %$nodehash) {
2241                     delete $nodehash->{$node} if $node ne $nodename
2242                 }
2243             }
2244         }
2245     });
2246
2247     return $nodehash
2248 }
2249
2250 sub check_cmdline {
2251     my ($pidfile, $pid) = @_;
2252
2253     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2254     if (defined($fh)) {
2255         my $line = <$fh>;
2256         $fh->close;
2257         return undef if !$line;
2258         my @param = split(/\0/, $line);
2259
2260         my $cmd = $param[0];
2261         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2262
2263         for (my $i = 0; $i < scalar (@param); $i++) {
2264             my $p = $param[$i];
2265             next if !$p;
2266             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2267                 my $p = $param[$i+1];
2268                 return 1 if $p && ($p eq $pidfile);
2269                 return undef;
2270             }
2271         }
2272     }
2273     return undef;
2274 }
2275
2276 sub check_running {
2277     my ($vmid, $nocheck, $node) = @_;
2278
2279     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2280
2281     die "unable to find configuration file for VM $vmid - no such machine\n"
2282         if !$nocheck && ! -f $filename;
2283
2284     my $pidfile = pidfile_name($vmid);
2285
2286     if (my $fd = IO::File->new("<$pidfile")) {
2287         my $st = stat($fd);
2288         my $line = <$fd>;
2289         close($fd);
2290
2291         my $mtime = $st->mtime;
2292         if ($mtime > time()) {
2293             warn "file '$filename' modified in future\n";
2294         }
2295
2296         if ($line =~ m/^(\d+)$/) {
2297             my $pid = $1;
2298             if (check_cmdline($pidfile, $pid)) {
2299                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2300                     return $pid;
2301                 }
2302             }
2303         }
2304     }
2305
2306     return undef;
2307 }
2308
2309 sub vzlist {
2310
2311     my $vzlist = config_list();
2312
2313     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2314
2315     while (defined(my $de = $fd->read)) {
2316         next if $de !~ m/^(\d+)\.pid$/;
2317         my $vmid = $1;
2318         next if !defined($vzlist->{$vmid});
2319         if (my $pid = check_running($vmid)) {
2320             $vzlist->{$vmid}->{pid} = $pid;
2321         }
2322     }
2323
2324     return $vzlist;
2325 }
2326
2327 sub disksize {
2328     my ($storecfg, $conf) = @_;
2329
2330     my $bootdisk = $conf->{bootdisk};
2331     return undef if !$bootdisk;
2332     return undef if !is_valid_drivename($bootdisk);
2333
2334     return undef if !$conf->{$bootdisk};
2335
2336     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2337     return undef if !defined($drive);
2338
2339     return undef if drive_is_cdrom($drive);
2340
2341     my $volid = $drive->{file};
2342     return undef if !$volid;
2343
2344     return $drive->{size};
2345 }
2346
2347 my $last_proc_pid_stat;
2348
2349 # get VM status information
2350 # This must be fast and should not block ($full == false)
2351 # We only query KVM using QMP if $full == true (this can be slow)
2352 sub vmstatus {
2353     my ($opt_vmid, $full) = @_;
2354
2355     my $res = {};
2356
2357     my $storecfg = PVE::Storage::config();
2358
2359     my $list = vzlist();
2360     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2361
2362     my $cpucount = $cpuinfo->{cpus} || 1;
2363
2364     foreach my $vmid (keys %$list) {
2365         next if $opt_vmid && ($vmid ne $opt_vmid);
2366
2367         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2368         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2369
2370         my $d = {};
2371         $d->{pid} = $list->{$vmid}->{pid};
2372
2373         # fixme: better status?
2374         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2375
2376         my $size = disksize($storecfg, $conf);
2377         if (defined($size)) {
2378             $d->{disk} = 0; # no info available
2379             $d->{maxdisk} = $size;
2380         } else {
2381             $d->{disk} = 0;
2382             $d->{maxdisk} = 0;
2383         }
2384
2385         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2386         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2387         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2388
2389         $d->{name} = $conf->{name} || "VM $vmid";
2390         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2391
2392         if ($conf->{balloon}) {
2393             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2394             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2395         }
2396
2397         $d->{uptime} = 0;
2398         $d->{cpu} = 0;
2399         $d->{mem} = 0;
2400
2401         $d->{netout} = 0;
2402         $d->{netin} = 0;
2403
2404         $d->{diskread} = 0;
2405         $d->{diskwrite} = 0;
2406
2407         $d->{template} = PVE::QemuConfig->is_template($conf);
2408
2409         $res->{$vmid} = $d;
2410     }
2411
2412     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2413     foreach my $dev (keys %$netdev) {
2414         next if $dev !~ m/^tap([1-9]\d*)i/;
2415         my $vmid = $1;
2416         my $d = $res->{$vmid};
2417         next if !$d;
2418
2419         $d->{netout} += $netdev->{$dev}->{receive};
2420         $d->{netin} += $netdev->{$dev}->{transmit};
2421
2422         if ($full) {
2423             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2424             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2425         }
2426
2427     }
2428
2429     my $ctime = gettimeofday;
2430
2431     foreach my $vmid (keys %$list) {
2432
2433         my $d = $res->{$vmid};
2434         my $pid = $d->{pid};
2435         next if !$pid;
2436
2437         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2438         next if !$pstat; # not running
2439
2440         my $used = $pstat->{utime} + $pstat->{stime};
2441
2442         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2443
2444         if ($pstat->{vsize}) {
2445             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2446         }
2447
2448         my $old = $last_proc_pid_stat->{$pid};
2449         if (!$old) {
2450             $last_proc_pid_stat->{$pid} = {
2451                 time => $ctime,
2452                 used => $used,
2453                 cpu => 0,
2454             };
2455             next;
2456         }
2457
2458         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2459
2460         if ($dtime > 1000) {
2461             my $dutime = $used -  $old->{used};
2462
2463             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2464             $last_proc_pid_stat->{$pid} = {
2465                 time => $ctime,
2466                 used => $used,
2467                 cpu => $d->{cpu},
2468             };
2469         } else {
2470             $d->{cpu} = $old->{cpu};
2471         }
2472     }
2473
2474     return $res if !$full;
2475
2476     my $qmpclient = PVE::QMPClient->new();
2477
2478     my $ballooncb = sub {
2479         my ($vmid, $resp) = @_;
2480
2481         my $info = $resp->{'return'};
2482         return if !$info->{max_mem};
2483
2484         my $d = $res->{$vmid};
2485
2486         # use memory assigned to VM
2487         $d->{maxmem} = $info->{max_mem};
2488         $d->{balloon} = $info->{actual};
2489
2490         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2491             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2492             $d->{freemem} = $info->{free_mem};
2493         }
2494
2495         $d->{ballooninfo} = $info;
2496     };
2497
2498     my $blockstatscb = sub {
2499         my ($vmid, $resp) = @_;
2500         my $data = $resp->{'return'} || [];
2501         my $totalrdbytes = 0;
2502         my $totalwrbytes = 0;
2503
2504         for my $blockstat (@$data) {
2505             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2506             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2507
2508             $blockstat->{device} =~ s/drive-//;
2509             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2510         }
2511         $res->{$vmid}->{diskread} = $totalrdbytes;
2512         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2513     };
2514
2515     my $statuscb = sub {
2516         my ($vmid, $resp) = @_;
2517
2518         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2519         # this fails if ballon driver is not loaded, so this must be
2520         # the last commnand (following command are aborted if this fails).
2521         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2522
2523         my $status = 'unknown';
2524         if (!defined($status = $resp->{'return'}->{status})) {
2525             warn "unable to get VM status\n";
2526             return;
2527         }
2528
2529         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2530     };
2531
2532     foreach my $vmid (keys %$list) {
2533         next if $opt_vmid && ($vmid ne $opt_vmid);
2534         next if !$res->{$vmid}->{pid}; # not running
2535         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2536     }
2537
2538     $qmpclient->queue_execute(undef, 1);
2539
2540     foreach my $vmid (keys %$list) {
2541         next if $opt_vmid && ($vmid ne $opt_vmid);
2542         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2543     }
2544
2545     return $res;
2546 }
2547
2548 sub foreach_dimm {
2549     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2550
2551     my $dimm_id = 0;
2552     my $current_size = 1024;
2553     my $dimm_size = 512;
2554     return if $current_size == $memory;
2555
2556     for (my $j = 0; $j < 8; $j++) {
2557         for (my $i = 0; $i < 32; $i++) {
2558             my $name = "dimm${dimm_id}";
2559             $dimm_id++;
2560             my $numanode = $i % $sockets;
2561             $current_size += $dimm_size;
2562             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2563             return  $current_size if $current_size >= $memory;
2564         }
2565         $dimm_size *= 2;
2566     }
2567 }
2568
2569 sub foreach_reverse_dimm {
2570     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2571
2572     my $dimm_id = 253;
2573     my $current_size = 4177920;
2574     my $dimm_size = 65536;
2575     return if $current_size == $memory;
2576
2577     for (my $j = 0; $j < 8; $j++) {
2578         for (my $i = 0; $i < 32; $i++) {
2579             my $name = "dimm${dimm_id}";
2580             $dimm_id--;
2581             my $numanode = $i % $sockets;
2582             $current_size -= $dimm_size;
2583             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2584             return  $current_size if $current_size <= $memory;
2585         }
2586         $dimm_size /= 2;
2587     }
2588 }
2589
2590 sub foreach_drive {
2591     my ($conf, $func) = @_;
2592
2593     foreach my $ds (valid_drive_names()) {
2594         next if !defined($conf->{$ds});
2595
2596         my $drive = parse_drive($ds, $conf->{$ds});
2597         next if !$drive;
2598
2599         &$func($ds, $drive);
2600     }
2601 }
2602
2603 sub foreach_volid {
2604     my ($conf, $func) = @_;
2605
2606     my $volhash = {};
2607
2608     my $test_volid = sub {
2609         my ($volid, $is_cdrom) = @_;
2610
2611         return if !$volid;
2612
2613         $volhash->{$volid} = $is_cdrom || 0;
2614     };
2615
2616     foreach_drive($conf, sub {
2617         my ($ds, $drive) = @_;
2618         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2619     });
2620
2621     foreach my $snapname (keys %{$conf->{snapshots}}) {
2622         my $snap = $conf->{snapshots}->{$snapname};
2623         &$test_volid($snap->{vmstate}, 0);
2624         foreach_drive($snap, sub {
2625             my ($ds, $drive) = @_;
2626             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2627         });
2628     }
2629
2630     foreach my $volid (keys %$volhash) {
2631         &$func($volid, $volhash->{$volid});
2632     }
2633 }
2634
2635 sub vga_conf_has_spice {
2636     my ($vga) = @_;
2637
2638     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2639
2640     return $1 || 1;
2641 }
2642
2643 sub config_to_command {
2644     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2645
2646     my $cmd = [];
2647     my $globalFlags = [];
2648     my $machineFlags = [];
2649     my $rtcFlags = [];
2650     my $cpuFlags = [];
2651     my $devices = [];
2652     my $pciaddr = '';
2653     my $bridges = {};
2654     my $kvmver = kvm_user_version();
2655     my $vernum = 0; # unknown
2656     my $ostype = $conf->{ostype};
2657     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2658         $vernum = $1*1000000+$2*1000;
2659     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2660         $vernum = $1*1000000+$2*1000+$3;
2661     }
2662
2663     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2664
2665     my $have_ovz = -f '/proc/vz/vestat';
2666
2667     my $q35 = machine_type_is_q35($conf);
2668     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2669     my $machine_type = $forcemachine || $conf->{machine};
2670     my $use_old_bios_files = undef;
2671     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2672
2673     my $cpuunits = defined($conf->{cpuunits}) ?
2674             $conf->{cpuunits} : $defaults->{cpuunits};
2675
2676     push @$cmd, '/usr/bin/systemd-run';
2677     push @$cmd, '--scope';
2678     push @$cmd, '--slice', "qemu";
2679     push @$cmd, '--unit', $vmid;
2680     # set KillMode=none, so that systemd don't kill those scopes
2681     # at shutdown (pve-manager service should stop the VMs instead)
2682     push @$cmd, '-p', "KillMode=none";
2683     push @$cmd, '-p', "CPUShares=$cpuunits";
2684     if ($conf->{cpulimit}) {
2685         my $cpulimit = int($conf->{cpulimit} * 100);
2686         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2687     }
2688
2689     push @$cmd, '/usr/bin/kvm';
2690
2691     push @$cmd, '-id', $vmid;
2692
2693     my $use_virtio = 0;
2694
2695     my $qmpsocket = qmp_socket($vmid);
2696     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2697     push @$cmd, '-mon', "chardev=qmp,mode=control";
2698
2699
2700     push @$cmd, '-pidfile' , pidfile_name($vmid);
2701
2702     push @$cmd, '-daemonize';
2703
2704     if ($conf->{smbios1}) {
2705         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2706     }
2707
2708     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2709         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2710         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2711         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2712         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2713         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2714         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2715     }
2716
2717     if ($q35) {
2718         # the q35 chipset support native usb2, so we enable usb controller
2719         # by default for this machine type
2720         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2721     } else {
2722         $pciaddr = print_pci_addr("piix3", $bridges);
2723         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2724
2725         my $use_usb2 = 0;
2726         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2727             next if !$conf->{"usb$i"};
2728             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2729             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2730             $use_usb2 = 1;
2731         }
2732         # include usb device config
2733         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2734     }
2735
2736     # add usb3 controller if needed
2737
2738     my $use_usb3 = 0;
2739     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2740         next if !$conf->{"usb$i"};
2741         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2742         next if !$d || !$d->{usb3};
2743         $use_usb3 = 1;
2744     }
2745
2746     $pciaddr = print_pci_addr("xhci", $bridges);
2747     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2748
2749     my $vga = $conf->{vga};
2750
2751     my $qxlnum = vga_conf_has_spice($vga);
2752     $vga = 'qxl' if $qxlnum;
2753
2754     if (!$vga) {
2755         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2756                                 $conf->{ostype} eq 'win7' ||
2757                                 $conf->{ostype} eq 'w2k8')) {
2758             $vga = 'std';
2759         } else {
2760             $vga = 'cirrus';
2761         }
2762     }
2763
2764     # enable absolute mouse coordinates (needed by vnc)
2765     my $tablet;
2766     if (defined($conf->{tablet})) {
2767         $tablet = $conf->{tablet};
2768     } else {
2769         $tablet = $defaults->{tablet};
2770         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2771         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2772     }
2773
2774     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2775
2776     my $kvm_off = 0;
2777     # host pci devices
2778     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2779         my $d = parse_hostpci($conf->{"hostpci$i"});
2780         next if !$d;
2781
2782         my $pcie = $d->{pcie};
2783         if($pcie){
2784             die "q35 machine model is not enabled" if !$q35;
2785             $pciaddr = print_pcie_addr("hostpci$i");
2786         }else{
2787             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2788         }
2789
2790         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2791         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2792         if ($xvga && $xvga ne '') {
2793             $kvm_off = 1;
2794             $vga = 'none';
2795             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2796                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2797             }
2798             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2799                 $xvga = "";
2800             }
2801         }
2802         my $pcidevices = $d->{pciid};
2803         my $multifunction = 1 if @$pcidevices > 1;
2804
2805         my $j=0;
2806         foreach my $pcidevice (@$pcidevices) {
2807
2808             my $id = "hostpci$i";
2809             $id .= ".$j" if $multifunction;
2810             my $addr = $pciaddr;
2811             $addr .= ".$j" if $multifunction;
2812             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2813
2814             if($j == 0){
2815                 $devicestr .= "$rombar$xvga";
2816                 $devicestr .= ",multifunction=on" if $multifunction;
2817             }
2818
2819             push @$devices, '-device', $devicestr;
2820             $j++;
2821         }
2822     }
2823
2824     # usb devices
2825     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2826         next if !$conf->{"usb$i"};
2827         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2828         next if !$d;
2829
2830         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2831         my $usbbus = '';
2832         if (defined($d->{usb3}) && $d->{usb3}) {
2833             $usbbus = ',bus=xhci.0';
2834         }
2835
2836         if (defined($d->{host})) {
2837             $d = parse_usb_device($d->{host});
2838             if (defined($d->{vendorid}) && defined($d->{productid})) {
2839                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2840             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2841                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2842             } elsif (defined($d->{spice}) && $d->{spice}) {
2843                 # usb redir support for spice, currently no usb3
2844                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2845                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2846             }
2847         }
2848     }
2849
2850     # serial devices
2851     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2852         if (my $path = $conf->{"serial$i"}) {
2853             if ($path eq 'socket') {
2854                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2855                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2856                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2857             } else {
2858                 die "no such serial device\n" if ! -c $path;
2859                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2860                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2861             }
2862         }
2863     }
2864
2865     # parallel devices
2866     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2867         if (my $path = $conf->{"parallel$i"}) {
2868             die "no such parallel device\n" if ! -c $path;
2869             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2870             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2871             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2872         }
2873     }
2874
2875     my $vmname = $conf->{name} || "vm$vmid";
2876
2877     push @$cmd, '-name', $vmname;
2878
2879     my $sockets = 1;
2880     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2881     $sockets = $conf->{sockets} if  $conf->{sockets};
2882
2883     my $cores = $conf->{cores} || 1;
2884
2885     my $maxcpus = $sockets * $cores;
2886
2887     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2888
2889     my $allowed_vcpus = $cpuinfo->{cpus};
2890
2891     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2892         if ($allowed_vcpus < $maxcpus);
2893
2894     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2895
2896     push @$cmd, '-nodefaults';
2897
2898     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2899
2900     my $bootindex_hash = {};
2901     my $i = 1;
2902     foreach my $o (split(//, $bootorder)) {
2903         $bootindex_hash->{$o} = $i*100;
2904         $i++;
2905     }
2906
2907     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2908
2909     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2910
2911     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2912
2913     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2914
2915     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2916         my $socket = vnc_socket($vmid);
2917         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2918     } else {
2919         push @$cmd, '-nographic';
2920     }
2921
2922     # time drift fix
2923     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2924
2925     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2926     my $useLocaltime = $conf->{localtime};
2927
2928     if ($ostype) {
2929         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2930
2931         if ($ostype =~ m/^w/) { # windows
2932             $useLocaltime = 1 if !defined($conf->{localtime});
2933
2934             # use time drift fix when acpi is enabled
2935             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2936                 $tdf = 1 if !defined($conf->{tdf});
2937             }
2938         }
2939
2940         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2941             $ostype eq 'wvista') {
2942             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2943             push @$cmd, '-no-hpet';
2944             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2945                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2946                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2947                 push @$cpuFlags , 'hv_time' if !$nokvm;
2948
2949             } else {
2950                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2951             }
2952         }
2953
2954         if ($ostype eq 'win7' || $ostype eq 'win8') {
2955             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2956         }
2957     }
2958
2959     push @$rtcFlags, 'driftfix=slew' if $tdf;
2960
2961     if ($nokvm) {
2962         push @$machineFlags, 'accel=tcg';
2963     } else {
2964         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2965     }
2966
2967     if ($machine_type) {
2968         push @$machineFlags, "type=${machine_type}";
2969     }
2970
2971     if ($conf->{startdate}) {
2972         push @$rtcFlags, "base=$conf->{startdate}";
2973     } elsif ($useLocaltime) {
2974         push @$rtcFlags, 'base=localtime';
2975     }
2976
2977     my $cpu = $nokvm ? "qemu64" : "kvm64";
2978     if (my $cputype = $conf->{cpu}) {
2979         my $cpuconf = PVE::JSONSchema::parse_property_string($cpudesc, $cputype)
2980             or die "Cannot parse cpu description: $cputype\n";
2981         $cpu = $cpuconf->{cputype};
2982         $kvm_off = 1 if $cpuconf->{hidden};
2983     }
2984
2985     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2986
2987     push @$cpuFlags , '-x2apic'
2988         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2989
2990     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2991
2992     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2993
2994     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2995
2996         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2997         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2998     }
2999
3000     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3001
3002     push @$cpuFlags, 'kvm=off' if $kvm_off;
3003
3004     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3005
3006     push @$cmd, '-cpu', $cpu;
3007
3008     my $memory = $conf->{memory} || $defaults->{memory};
3009     my $static_memory = 0;
3010     my $dimm_memory = 0;
3011
3012     if ($hotplug_features->{memory}) {
3013         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
3014         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3015         $static_memory = $STATICMEM;
3016         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3017         $dimm_memory = $memory - $static_memory;
3018         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3019
3020     } else {
3021
3022         $static_memory = $memory;
3023         push @$cmd, '-m', $static_memory;
3024     }
3025
3026     if ($conf->{numa}) {
3027
3028         my $numa_totalmemory = undef;
3029         for (my $i = 0; $i < $MAX_NUMA; $i++) {
3030             next if !$conf->{"numa$i"};
3031             my $numa = parse_numa($conf->{"numa$i"});
3032             next if !$numa;
3033             # memory
3034             die "missing numa node$i memory value\n" if !$numa->{memory};
3035             my $numa_memory = $numa->{memory};
3036             $numa_totalmemory += $numa_memory;
3037             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3038
3039             # cpus
3040             my $cpus_start = $numa->{cpus}->{start};
3041             die "missing numa node$i cpus\n" if !defined($cpus_start);
3042             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
3043             my $cpus = $cpus_start;
3044             if (defined($cpus_end)) {
3045                 $cpus .= "-$cpus_end";
3046                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3047             }
3048
3049             # hostnodes
3050             my $hostnodes_start = $numa->{hostnodes}->{start};
3051             if (defined($hostnodes_start)) {
3052                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3053                 my $hostnodes = $hostnodes_start;
3054                 if (defined($hostnodes_end)) {
3055                     $hostnodes .= "-$hostnodes_end";
3056                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3057                 }
3058
3059                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3060                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3061                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3062                 }
3063
3064                 # policy
3065                 my $policy = $numa->{policy};
3066                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3067                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3068             }
3069
3070             push @$cmd, '-object', $numa_object;
3071             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3072         }
3073
3074         die "total memory for NUMA nodes must be equal to vm static memory\n"
3075             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3076
3077         #if no custom tology, we split memory and cores across numa nodes
3078         if(!$numa_totalmemory) {
3079
3080             my $numa_memory = ($static_memory / $sockets) . "M";
3081
3082             for (my $i = 0; $i < $sockets; $i++)  {
3083
3084                 my $cpustart = ($cores * $i);
3085                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3086                 my $cpus = $cpustart;
3087                 $cpus .= "-$cpuend" if $cpuend;
3088
3089                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3090                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3091             }
3092         }
3093     }
3094
3095     if ($hotplug_features->{memory}) {
3096         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3097             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3098             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3099             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3100
3101             #if dimm_memory is not aligned to dimm map
3102             if($current_size > $memory) {
3103                  $conf->{memory} = $current_size;
3104                  PVE::QemuConfig->write_config($vmid, $conf);
3105             }
3106         });
3107     }
3108
3109     push @$cmd, '-S' if $conf->{freeze};
3110
3111     # set keyboard layout
3112     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3113     push @$cmd, '-k', $kb if $kb;
3114
3115     # enable sound
3116     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3117     #push @$cmd, '-soundhw', 'es1370';
3118     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3119
3120     if($conf->{agent}) {
3121         my $qgasocket = qmp_socket($vmid, 1);
3122         my $pciaddr = print_pci_addr("qga0", $bridges);
3123         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3124         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3125         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3126     }
3127
3128     my $spice_port;
3129
3130     if ($qxlnum) {
3131         if ($qxlnum > 1) {
3132             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3133                 for(my $i = 1; $i < $qxlnum; $i++){
3134                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3135                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3136                 }
3137             } else {
3138                 # assume other OS works like Linux
3139                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3140                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3141             }
3142         }
3143
3144         my $pciaddr = print_pci_addr("spice", $bridges);
3145
3146         my $nodename = PVE::INotify::nodename();
3147         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3148         $spice_port = PVE::Tools::next_spice_port($pfamily);
3149
3150         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3151
3152         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3153         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3154         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3155     }
3156
3157     # enable balloon by default, unless explicitly disabled
3158     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3159         $pciaddr = print_pci_addr("balloon0", $bridges);
3160         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3161     }
3162
3163     if ($conf->{watchdog}) {
3164         my $wdopts = parse_watchdog($conf->{watchdog});
3165         $pciaddr = print_pci_addr("watchdog", $bridges);
3166         my $watchdog = $wdopts->{model} || 'i6300esb';
3167         push @$devices, '-device', "$watchdog$pciaddr";
3168         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3169     }
3170
3171     my $vollist = [];
3172     my $scsicontroller = {};
3173     my $ahcicontroller = {};
3174     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3175
3176     # Add iscsi initiator name if available
3177     if (my $initiator = get_initiator_name()) {
3178         push @$devices, '-iscsi', "initiator-name=$initiator";
3179     }
3180
3181     foreach_drive($conf, sub {
3182         my ($ds, $drive) = @_;
3183
3184         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3185             push @$vollist, $drive->{file};
3186         }
3187
3188         $use_virtio = 1 if $ds =~ m/^virtio/;
3189
3190         if (drive_is_cdrom ($drive)) {
3191             if ($bootindex_hash->{d}) {
3192                 $drive->{bootindex} = $bootindex_hash->{d};
3193                 $bootindex_hash->{d} += 1;
3194             }
3195         } else {
3196             if ($bootindex_hash->{c}) {
3197                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3198                 $bootindex_hash->{c} += 1;
3199             }
3200         }
3201
3202         if($drive->{interface} eq 'virtio'){
3203            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3204         }
3205
3206         if ($drive->{interface} eq 'scsi') {
3207
3208             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3209
3210             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3211             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3212
3213             my $iothread = '';
3214             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3215                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3216                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3217             }
3218
3219             my $queues = '';
3220             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3221                 $queues = ",num_queues=$drive->{queues}";
3222             }
3223
3224             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3225             $scsicontroller->{$controller}=1;
3226         }
3227
3228         if ($drive->{interface} eq 'sata') {
3229            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3230            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3231            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3232            $ahcicontroller->{$controller}=1;
3233         }
3234
3235         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3236         push @$devices, '-drive',$drive_cmd;
3237         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3238     });
3239
3240     for (my $i = 0; $i < $MAX_NETS; $i++) {
3241          next if !$conf->{"net$i"};
3242          my $d = parse_net($conf->{"net$i"});
3243          next if !$d;
3244
3245          $use_virtio = 1 if $d->{model} eq 'virtio';
3246
3247          if ($bootindex_hash->{n}) {
3248             $d->{bootindex} = $bootindex_hash->{n};
3249             $bootindex_hash->{n} += 1;
3250          }
3251
3252          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3253          push @$devices, '-netdev', $netdevfull;
3254
3255          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3256          push @$devices, '-device', $netdevicefull;
3257     }
3258
3259     if (!$q35) {
3260         # add pci bridges
3261         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3262            $bridges->{1} = 1;
3263            $bridges->{2} = 1;
3264         }
3265
3266         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3267
3268         while (my ($k, $v) = each %$bridges) {
3269             $pciaddr = print_pci_addr("pci.$k");
3270             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3271         }
3272     }
3273
3274     # add custom args
3275     if ($conf->{args}) {
3276         my $aa = PVE::Tools::split_args($conf->{args});
3277         push @$cmd, @$aa;
3278     }
3279
3280     push @$cmd, @$devices;
3281     push @$cmd, '-rtc', join(',', @$rtcFlags)
3282         if scalar(@$rtcFlags);
3283     push @$cmd, '-machine', join(',', @$machineFlags)
3284         if scalar(@$machineFlags);
3285     push @$cmd, '-global', join(',', @$globalFlags)
3286         if scalar(@$globalFlags);
3287
3288     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3289 }
3290
3291 sub vnc_socket {
3292     my ($vmid) = @_;
3293     return "${var_run_tmpdir}/$vmid.vnc";
3294 }
3295
3296 sub spice_port {
3297     my ($vmid) = @_;
3298
3299     my $res = vm_mon_cmd($vmid, 'query-spice');
3300
3301     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3302 }
3303
3304 sub qmp_socket {
3305     my ($vmid, $qga) = @_;
3306     my $sockettype = $qga ? 'qga' : 'qmp';
3307     return "${var_run_tmpdir}/$vmid.$sockettype";
3308 }
3309
3310 sub pidfile_name {
3311     my ($vmid) = @_;
3312     return "${var_run_tmpdir}/$vmid.pid";
3313 }
3314
3315 sub vm_devices_list {
3316     my ($vmid) = @_;
3317
3318     my $res = vm_mon_cmd($vmid, 'query-pci');
3319     my $devices = {};
3320     foreach my $pcibus (@$res) {
3321         foreach my $device (@{$pcibus->{devices}}) {
3322             next if !$device->{'qdev_id'};
3323             if ($device->{'pci_bridge'}) {
3324                 $devices->{$device->{'qdev_id'}} = 1;
3325                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3326                     next if !$bridge_device->{'qdev_id'};
3327                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3328                     $devices->{$device->{'qdev_id'}}++;
3329                 }
3330             } else {
3331                 $devices->{$device->{'qdev_id'}} = 1;
3332             }
3333         }
3334     }
3335
3336     my $resblock = vm_mon_cmd($vmid, 'query-block');
3337     foreach my $block (@$resblock) {
3338         if($block->{device} =~ m/^drive-(\S+)/){
3339                 $devices->{$1} = 1;
3340         }
3341     }
3342
3343     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3344     foreach my $mice (@$resmice) {
3345         if ($mice->{name} eq 'QEMU HID Tablet') {
3346             $devices->{tablet} = 1;
3347             last;
3348         }
3349     }
3350
3351     return $devices;
3352 }
3353
3354 sub vm_deviceplug {
3355     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3356
3357     my $q35 = machine_type_is_q35($conf);
3358
3359     my $devices_list = vm_devices_list($vmid);
3360     return 1 if defined($devices_list->{$deviceid});
3361
3362     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3363
3364     if ($deviceid eq 'tablet') {
3365
3366         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3367
3368     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3369
3370         qemu_iothread_add($vmid, $deviceid, $device);
3371
3372         qemu_driveadd($storecfg, $vmid, $device);
3373         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3374
3375         qemu_deviceadd($vmid, $devicefull);
3376         eval { qemu_deviceaddverify($vmid, $deviceid); };
3377         if (my $err = $@) {
3378             eval { qemu_drivedel($vmid, $deviceid); };
3379             warn $@ if $@;
3380             die $err;
3381         }
3382
3383     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3384
3385
3386         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3387         my $pciaddr = print_pci_addr($deviceid);
3388         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3389
3390         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3391
3392         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3393             qemu_iothread_add($vmid, $deviceid, $device);
3394             $devicefull .= ",iothread=iothread-$deviceid";
3395         }
3396
3397         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3398             $devicefull .= ",num_queues=$device->{queues}";
3399         }
3400
3401         qemu_deviceadd($vmid, $devicefull);
3402         qemu_deviceaddverify($vmid, $deviceid);
3403
3404     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3405
3406         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3407         qemu_driveadd($storecfg, $vmid, $device);
3408
3409         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3410         eval { qemu_deviceadd($vmid, $devicefull); };
3411         if (my $err = $@) {
3412             eval { qemu_drivedel($vmid, $deviceid); };
3413             warn $@ if $@;
3414             die $err;
3415         }
3416
3417     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3418
3419         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3420
3421         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3422         my $use_old_bios_files = undef;
3423         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3424
3425         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3426         qemu_deviceadd($vmid, $netdevicefull);
3427         eval { qemu_deviceaddverify($vmid, $deviceid); };
3428         if (my $err = $@) {
3429             eval { qemu_netdevdel($vmid, $deviceid); };
3430             warn $@ if $@;
3431             die $err;
3432         }
3433
3434     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3435
3436         my $bridgeid = $2;
3437         my $pciaddr = print_pci_addr($deviceid);
3438         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3439
3440         qemu_deviceadd($vmid, $devicefull);
3441         qemu_deviceaddverify($vmid, $deviceid);
3442
3443     } else {
3444         die "can't hotplug device '$deviceid'\n";
3445     }
3446
3447     return 1;
3448 }
3449
3450 # fixme: this should raise exceptions on error!
3451 sub vm_deviceunplug {
3452     my ($vmid, $conf, $deviceid) = @_;
3453
3454     my $devices_list = vm_devices_list($vmid);
3455     return 1 if !defined($devices_list->{$deviceid});
3456
3457     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3458
3459     if ($deviceid eq 'tablet') {
3460
3461         qemu_devicedel($vmid, $deviceid);
3462
3463     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3464
3465         qemu_devicedel($vmid, $deviceid);
3466         qemu_devicedelverify($vmid, $deviceid);
3467         qemu_drivedel($vmid, $deviceid);
3468         qemu_iothread_del($conf, $vmid, $deviceid);
3469
3470     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3471
3472         qemu_devicedel($vmid, $deviceid);
3473         qemu_devicedelverify($vmid, $deviceid);
3474         qemu_iothread_del($conf, $vmid, $deviceid);
3475
3476     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3477
3478         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3479         my $device = parse_drive($deviceid, $conf->{$deviceid});
3480         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3481
3482         qemu_devicedel($vmid, $deviceid);
3483         qemu_drivedel($vmid, $deviceid);
3484         qemu_deletescsihw($conf, $vmid, $deviceid);
3485
3486     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3487
3488         qemu_devicedel($vmid, $deviceid);
3489         qemu_devicedelverify($vmid, $deviceid);
3490         qemu_netdevdel($vmid, $deviceid);
3491
3492     } else {
3493         die "can't unplug device '$deviceid'\n";
3494     }
3495
3496     return 1;
3497 }
3498
3499 sub qemu_deviceadd {
3500     my ($vmid, $devicefull) = @_;
3501
3502     $devicefull = "driver=".$devicefull;
3503     my %options =  split(/[=,]/, $devicefull);
3504
3505     vm_mon_cmd($vmid, "device_add" , %options);
3506 }
3507
3508 sub qemu_devicedel {
3509     my ($vmid, $deviceid) = @_;
3510
3511     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3512 }
3513
3514 sub qemu_iothread_add {
3515     my($vmid, $deviceid, $device) = @_;
3516
3517     if ($device->{iothread}) {
3518         my $iothreads = vm_iothreads_list($vmid);
3519         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3520     }
3521 }
3522
3523 sub qemu_iothread_del {
3524     my($conf, $vmid, $deviceid) = @_;
3525
3526     my $device = parse_drive($deviceid, $conf->{$deviceid});
3527     if ($device->{iothread}) {
3528         my $iothreads = vm_iothreads_list($vmid);
3529         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3530     }
3531 }
3532
3533 sub qemu_objectadd {
3534     my($vmid, $objectid, $qomtype) = @_;
3535
3536     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3537
3538     return 1;
3539 }
3540
3541 sub qemu_objectdel {
3542     my($vmid, $objectid) = @_;
3543
3544     vm_mon_cmd($vmid, "object-del", id => $objectid);
3545
3546     return 1;
3547 }
3548
3549 sub qemu_driveadd {
3550     my ($storecfg, $vmid, $device) = @_;
3551
3552     my $drive = print_drive_full($storecfg, $vmid, $device);
3553     $drive =~ s/\\/\\\\/g;
3554     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3555
3556     # If the command succeeds qemu prints: "OK"
3557     return 1 if $ret =~ m/OK/s;
3558
3559     die "adding drive failed: $ret\n";
3560 }
3561
3562 sub qemu_drivedel {
3563     my($vmid, $deviceid) = @_;
3564
3565     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3566     $ret =~ s/^\s+//;
3567
3568     return 1 if $ret eq "";
3569
3570     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3571     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3572
3573     die "deleting drive $deviceid failed : $ret\n";
3574 }
3575
3576 sub qemu_deviceaddverify {
3577     my ($vmid, $deviceid) = @_;
3578
3579     for (my $i = 0; $i <= 5; $i++) {
3580          my $devices_list = vm_devices_list($vmid);
3581          return 1 if defined($devices_list->{$deviceid});
3582          sleep 1;
3583     }
3584
3585     die "error on hotplug device '$deviceid'\n";
3586 }
3587
3588
3589 sub qemu_devicedelverify {
3590     my ($vmid, $deviceid) = @_;
3591
3592     # need to verify that the device is correctly removed as device_del
3593     # is async and empty return is not reliable
3594
3595     for (my $i = 0; $i <= 5; $i++) {
3596          my $devices_list = vm_devices_list($vmid);
3597          return 1 if !defined($devices_list->{$deviceid});
3598          sleep 1;
3599     }
3600
3601     die "error on hot-unplugging device '$deviceid'\n";
3602 }
3603
3604 sub qemu_findorcreatescsihw {
3605     my ($storecfg, $conf, $vmid, $device) = @_;
3606
3607     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3608
3609     my $scsihwid="$controller_prefix$controller";
3610     my $devices_list = vm_devices_list($vmid);
3611
3612     if(!defined($devices_list->{$scsihwid})) {
3613         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3614     }
3615
3616     return 1;
3617 }
3618
3619 sub qemu_deletescsihw {
3620     my ($conf, $vmid, $opt) = @_;
3621
3622     my $device = parse_drive($opt, $conf->{$opt});
3623
3624     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3625         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3626         return 1;
3627     }
3628
3629     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3630
3631     my $devices_list = vm_devices_list($vmid);
3632     foreach my $opt (keys %{$devices_list}) {
3633         if (PVE::QemuServer::is_valid_drivename($opt)) {
3634             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3635             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3636                 return 1;
3637             }
3638         }
3639     }
3640
3641     my $scsihwid="scsihw$controller";
3642
3643     vm_deviceunplug($vmid, $conf, $scsihwid);
3644
3645     return 1;
3646 }
3647
3648 sub qemu_add_pci_bridge {
3649     my ($storecfg, $conf, $vmid, $device) = @_;
3650
3651     my $bridges = {};
3652
3653     my $bridgeid;
3654
3655     print_pci_addr($device, $bridges);
3656
3657     while (my ($k, $v) = each %$bridges) {
3658         $bridgeid = $k;
3659     }
3660     return 1 if !defined($bridgeid) || $bridgeid < 1;
3661
3662     my $bridge = "pci.$bridgeid";
3663     my $devices_list = vm_devices_list($vmid);
3664
3665     if (!defined($devices_list->{$bridge})) {
3666         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3667     }
3668
3669     return 1;
3670 }
3671
3672 sub qemu_set_link_status {
3673     my ($vmid, $device, $up) = @_;
3674
3675     vm_mon_cmd($vmid, "set_link", name => $device,
3676                up => $up ? JSON::true : JSON::false);
3677 }
3678
3679 sub qemu_netdevadd {
3680     my ($vmid, $conf, $device, $deviceid) = @_;
3681
3682     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3683     my %options =  split(/[=,]/, $netdev);
3684
3685     vm_mon_cmd($vmid, "netdev_add",  %options);
3686     return 1;
3687 }
3688
3689 sub qemu_netdevdel {
3690     my ($vmid, $deviceid) = @_;
3691
3692     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3693 }
3694
3695 sub qemu_cpu_hotplug {
3696     my ($vmid, $conf, $vcpus) = @_;
3697
3698     my $sockets = 1;
3699     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3700     $sockets = $conf->{sockets} if  $conf->{sockets};
3701     my $cores = $conf->{cores} || 1;
3702     my $maxcpus = $sockets * $cores;
3703
3704     $vcpus = $maxcpus if !$vcpus;
3705
3706     die "you can't add more vcpus than maxcpus\n"
3707         if $vcpus > $maxcpus;
3708
3709     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3710     die "online cpu unplug is not yet possible\n"
3711         if $vcpus < $currentvcpus;
3712
3713     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3714     die "vcpus in running vm is different than configuration\n"
3715         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3716
3717     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3718         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3719     }
3720 }
3721
3722 sub qemu_memory_hotplug {
3723     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3724
3725     return $value if !check_running($vmid);
3726
3727     my $memory = $conf->{memory} || $defaults->{memory};
3728     $value = $defaults->{memory} if !$value;
3729     return $value if $value == $memory;
3730
3731     my $static_memory = $STATICMEM;
3732     my $dimm_memory = $memory - $static_memory;
3733
3734     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3735     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3736
3737
3738     my $sockets = 1;
3739     $sockets = $conf->{sockets} if $conf->{sockets};
3740
3741     if($value > $memory) {
3742
3743         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3744             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3745
3746                 return if $current_size <= $conf->{memory};
3747
3748                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3749                 if (my $err = $@) {
3750                     eval { qemu_objectdel($vmid, "mem-$name"); };
3751                     die $err;
3752                 }
3753
3754                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3755                 if (my $err = $@) {
3756                     eval { qemu_objectdel($vmid, "mem-$name"); };
3757                     die $err;
3758                 }
3759                 #update conf after each succesful module hotplug
3760                 $conf->{memory} = $current_size;
3761                 PVE::QemuConfig->write_config($vmid, $conf);
3762         });
3763
3764     } else {
3765
3766         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3767             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3768
3769                 return if $current_size >= $conf->{memory};
3770                 print "try to unplug memory dimm $name\n";
3771
3772                 my $retry = 0;
3773                 while (1) {
3774                     eval { qemu_devicedel($vmid, $name) };
3775                     sleep 3;
3776                     my $dimm_list = qemu_dimm_list($vmid);
3777                     last if !$dimm_list->{$name};
3778                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3779                     $retry++;
3780                 }
3781
3782                 #update conf after each succesful module unplug
3783                 $conf->{memory} = $current_size;
3784
3785                 eval { qemu_objectdel($vmid, "mem-$name"); };
3786                 PVE::QemuConfig->write_config($vmid, $conf);
3787         });
3788     }
3789 }
3790
3791 sub qemu_dimm_list {
3792     my ($vmid) = @_;
3793
3794     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3795     my $dimms = {};
3796
3797     foreach my $dimm (@$dimmarray) {
3798
3799         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3800         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3801         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3802         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3803         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3804     }
3805     return $dimms;
3806 }
3807
3808 sub qemu_block_set_io_throttle {
3809     my ($vmid, $deviceid,
3810         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3811         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3812
3813     return if !check_running($vmid) ;
3814
3815     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3816         bps => int($bps),
3817         bps_rd => int($bps_rd),
3818         bps_wr => int($bps_wr),
3819         iops => int($iops),
3820         iops_rd => int($iops_rd),
3821         iops_wr => int($iops_wr),
3822         bps_max => int($bps_max),
3823         bps_rd_max => int($bps_rd_max),
3824         bps_wr_max => int($bps_wr_max),
3825         iops_max => int($iops_max),
3826         iops_rd_max => int($iops_rd_max),
3827         iops_wr_max => int($iops_wr_max)
3828     );
3829
3830 }
3831
3832 # old code, only used to shutdown old VM after update
3833 sub __read_avail {
3834     my ($fh, $timeout) = @_;
3835
3836     my $sel = new IO::Select;
3837     $sel->add($fh);
3838
3839     my $res = '';
3840     my $buf;
3841
3842     my @ready;
3843     while (scalar (@ready = $sel->can_read($timeout))) {
3844         my $count;
3845         if ($count = $fh->sysread($buf, 8192)) {
3846             if ($buf =~ /^(.*)\(qemu\) $/s) {
3847                 $res .= $1;
3848                 last;
3849             } else {
3850                 $res .= $buf;
3851             }
3852         } else {
3853             if (!defined($count)) {
3854                 die "$!\n";
3855             }
3856             last;
3857         }
3858     }
3859
3860     die "monitor read timeout\n" if !scalar(@ready);
3861
3862     return $res;
3863 }
3864
3865 # old code, only used to shutdown old VM after update
3866 sub vm_monitor_command {
3867     my ($vmid, $cmdstr, $nocheck) = @_;
3868
3869     my $res;
3870
3871     eval {
3872         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3873
3874         my $sname = "${var_run_tmpdir}/$vmid.mon";
3875
3876         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3877             die "unable to connect to VM $vmid socket - $!\n";
3878
3879         my $timeout = 3;
3880
3881         # hack: migrate sometime blocks the monitor (when migrate_downtime
3882         # is set)
3883         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3884             $timeout = 60*60; # 1 hour
3885         }
3886
3887         # read banner;
3888         my $data = __read_avail($sock, $timeout);
3889
3890         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3891             die "got unexpected qemu monitor banner\n";
3892         }
3893
3894         my $sel = new IO::Select;
3895         $sel->add($sock);
3896
3897         if (!scalar(my @ready = $sel->can_write($timeout))) {
3898             die "monitor write error - timeout";
3899         }
3900
3901         my $fullcmd = "$cmdstr\r";
3902
3903         # syslog('info', "VM $vmid monitor command: $cmdstr");
3904
3905         my $b;
3906         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3907             die "monitor write error - $!";
3908         }
3909
3910         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3911
3912         $timeout = 20;
3913
3914         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3915             $timeout = 60*60; # 1 hour
3916         } elsif ($cmdstr =~ m/^(eject|change)/) {
3917             $timeout = 60; # note: cdrom mount command is slow
3918         }
3919         if ($res = __read_avail($sock, $timeout)) {
3920
3921             my @lines = split("\r?\n", $res);
3922
3923             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3924
3925             $res = join("\n", @lines);
3926             $res .= "\n";
3927         }
3928     };
3929
3930     my $err = $@;
3931
3932     if ($err) {
3933         syslog("err", "VM $vmid monitor command failed - $err");
3934         die $err;
3935     }
3936
3937     return $res;
3938 }
3939
3940 sub qemu_block_resize {
3941     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3942
3943     my $running = check_running($vmid);
3944
3945     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3946
3947     return if !$running;
3948
3949     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3950
3951 }
3952
3953 sub qemu_volume_snapshot {
3954     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3955
3956     my $running = check_running($vmid);
3957
3958     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3959         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3960     } else {
3961         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3962     }
3963 }
3964
3965 sub qemu_volume_snapshot_delete {
3966     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3967
3968     my $running = check_running($vmid);
3969
3970     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3971
3972     return if !$running;
3973
3974     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3975 }
3976
3977 sub set_migration_caps {
3978     my ($vmid) = @_;
3979
3980     my $cap_ref = [];
3981
3982     my $enabled_cap = {
3983         "auto-converge" => 1,
3984         "xbzrle" => 1,
3985         "x-rdma-pin-all" => 0,
3986         "zero-blocks" => 0,
3987         "compress" => 0
3988     };
3989
3990     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3991
3992     for my $supported_capability (@$supported_capabilities) {
3993         push @$cap_ref, {
3994             capability => $supported_capability->{capability},
3995             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3996         };
3997     }
3998
3999     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4000 }
4001
4002 my $fast_plug_option = {
4003     'lock' => 1,
4004     'name' => 1,
4005     'onboot' => 1,
4006     'shares' => 1,
4007     'startup' => 1,
4008     'description' => 1,
4009 };
4010
4011 # hotplug changes in [PENDING]
4012 # $selection hash can be used to only apply specified options, for
4013 # example: { cores => 1 } (only apply changed 'cores')
4014 # $errors ref is used to return error messages
4015 sub vmconfig_hotplug_pending {
4016     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4017
4018     my $defaults = load_defaults();
4019
4020     # commit values which do not have any impact on running VM first
4021     # Note: those option cannot raise errors, we we do not care about
4022     # $selection and always apply them.
4023
4024     my $add_error = sub {
4025         my ($opt, $msg) = @_;
4026         $errors->{$opt} = "hotplug problem - $msg";
4027     };
4028
4029     my $changes = 0;
4030     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4031         if ($fast_plug_option->{$opt}) {
4032             $conf->{$opt} = $conf->{pending}->{$opt};
4033             delete $conf->{pending}->{$opt};
4034             $changes = 1;
4035         }
4036     }
4037
4038     if ($changes) {
4039         PVE::QemuConfig->write_config($vmid, $conf);
4040         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4041     }
4042
4043     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4044
4045     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4046     while (my ($opt, $force) = each %$pending_delete_hash) {
4047         next if $selection && !$selection->{$opt};
4048         eval {
4049             if ($opt eq 'hotplug') {
4050                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4051             } elsif ($opt eq 'tablet') {
4052                 die "skip\n" if !$hotplug_features->{usb};
4053                 if ($defaults->{tablet}) {
4054                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4055                 } else {
4056                     vm_deviceunplug($vmid, $conf, $opt);
4057                 }
4058             } elsif ($opt eq 'vcpus') {
4059                 die "skip\n" if !$hotplug_features->{cpu};
4060                 qemu_cpu_hotplug($vmid, $conf, undef);
4061             } elsif ($opt eq 'balloon') {
4062                 # enable balloon device is not hotpluggable
4063                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4064             } elsif ($fast_plug_option->{$opt}) {
4065                 # do nothing
4066             } elsif ($opt =~ m/^net(\d+)$/) {
4067                 die "skip\n" if !$hotplug_features->{network};
4068                 vm_deviceunplug($vmid, $conf, $opt);
4069             } elsif (is_valid_drivename($opt)) {
4070                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4071                 vm_deviceunplug($vmid, $conf, $opt);
4072                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4073             } elsif ($opt =~ m/^memory$/) {
4074                 die "skip\n" if !$hotplug_features->{memory};
4075                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4076             } elsif ($opt eq 'cpuunits') {
4077                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4078             } elsif ($opt eq 'cpulimit') {
4079                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4080             } else {
4081                 die "skip\n";
4082             }
4083         };
4084         if (my $err = $@) {
4085             &$add_error($opt, $err) if $err ne "skip\n";
4086         } else {
4087             # save new config if hotplug was successful
4088             delete $conf->{$opt};
4089             vmconfig_undelete_pending_option($conf, $opt);
4090             PVE::QemuConfig->write_config($vmid, $conf);
4091             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4092         }
4093     }
4094
4095     foreach my $opt (keys %{$conf->{pending}}) {
4096         next if $selection && !$selection->{$opt};
4097         my $value = $conf->{pending}->{$opt};
4098         eval {
4099             if ($opt eq 'hotplug') {
4100                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4101             } elsif ($opt eq 'tablet') {
4102                 die "skip\n" if !$hotplug_features->{usb};
4103                 if ($value == 1) {
4104                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4105                 } elsif ($value == 0) {
4106                     vm_deviceunplug($vmid, $conf, $opt);
4107                 }
4108             } elsif ($opt eq 'vcpus') {
4109                 die "skip\n" if !$hotplug_features->{cpu};
4110                 qemu_cpu_hotplug($vmid, $conf, $value);
4111             } elsif ($opt eq 'balloon') {
4112                 # enable/disable balloning device is not hotpluggable
4113                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4114                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4115                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4116
4117                 # allow manual ballooning if shares is set to zero
4118                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4119                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4120                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4121                 }
4122             } elsif ($opt =~ m/^net(\d+)$/) {
4123                 # some changes can be done without hotplug
4124                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4125                                     $vmid, $opt, $value);
4126             } elsif (is_valid_drivename($opt)) {
4127                 # some changes can be done without hotplug
4128                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4129                                      $vmid, $opt, $value, 1);
4130             } elsif ($opt =~ m/^memory$/) { #dimms
4131                 die "skip\n" if !$hotplug_features->{memory};
4132                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4133             } elsif ($opt eq 'cpuunits') {
4134                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4135             } elsif ($opt eq 'cpulimit') {
4136                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4137                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4138             } else {
4139                 die "skip\n";  # skip non-hot-pluggable options
4140             }
4141         };
4142         if (my $err = $@) {
4143             &$add_error($opt, $err) if $err ne "skip\n";
4144         } else {
4145             # save new config if hotplug was successful
4146             $conf->{$opt} = $value;
4147             delete $conf->{pending}->{$opt};
4148             PVE::QemuConfig->write_config($vmid, $conf);
4149             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4150         }
4151     }
4152 }
4153
4154 sub try_deallocate_drive {
4155     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4156
4157     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4158         my $volid = $drive->{file};
4159         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4160             my $sid = PVE::Storage::parse_volume_id($volid);
4161             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4162
4163             # check if the disk is really unused
4164             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4165                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4166             PVE::Storage::vdisk_free($storecfg, $volid);
4167             return 1;
4168         } else {
4169             # If vm is not owner of this disk remove from config
4170             return 1;
4171         }
4172     }
4173
4174     return undef;
4175 }
4176
4177 sub vmconfig_delete_or_detach_drive {
4178     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4179
4180     my $drive = parse_drive($opt, $conf->{$opt});
4181
4182     my $rpcenv = PVE::RPCEnvironment::get();
4183     my $authuser = $rpcenv->get_user();
4184
4185     if ($force) {
4186         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4187         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4188     } else {
4189         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4190     }
4191 }
4192
4193 sub vmconfig_apply_pending {
4194     my ($vmid, $conf, $storecfg) = @_;
4195
4196     # cold plug
4197
4198     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4199     while (my ($opt, $force) = each %$pending_delete_hash) {
4200         die "internal error" if $opt =~ m/^unused/;
4201         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4202         if (!defined($conf->{$opt})) {
4203             vmconfig_undelete_pending_option($conf, $opt);
4204             PVE::QemuConfig->write_config($vmid, $conf);
4205         } elsif (is_valid_drivename($opt)) {
4206             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4207             vmconfig_undelete_pending_option($conf, $opt);
4208             delete $conf->{$opt};
4209             PVE::QemuConfig->write_config($vmid, $conf);
4210         } else {
4211             vmconfig_undelete_pending_option($conf, $opt);
4212             delete $conf->{$opt};
4213             PVE::QemuConfig->write_config($vmid, $conf);
4214         }
4215     }
4216
4217     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4218
4219     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4220         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4221
4222         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4223             # skip if nothing changed
4224         } elsif (is_valid_drivename($opt)) {
4225             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4226                 if defined($conf->{$opt});
4227             $conf->{$opt} = $conf->{pending}->{$opt};
4228         } else {
4229             $conf->{$opt} = $conf->{pending}->{$opt};
4230         }
4231
4232         delete $conf->{pending}->{$opt};
4233         PVE::QemuConfig->write_config($vmid, $conf);
4234     }
4235 }
4236
4237 my $safe_num_ne = sub {
4238     my ($a, $b) = @_;
4239
4240     return 0 if !defined($a) && !defined($b);
4241     return 1 if !defined($a);
4242     return 1 if !defined($b);
4243
4244     return $a != $b;
4245 };
4246
4247 my $safe_string_ne = sub {
4248     my ($a, $b) = @_;
4249
4250     return 0 if !defined($a) && !defined($b);
4251     return 1 if !defined($a);
4252     return 1 if !defined($b);
4253
4254     return $a ne $b;
4255 };
4256
4257 sub vmconfig_update_net {
4258     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4259
4260     my $newnet = parse_net($value);
4261
4262     if ($conf->{$opt}) {
4263         my $oldnet = parse_net($conf->{$opt});
4264
4265         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4266             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4267             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4268             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4269
4270             # for non online change, we try to hot-unplug
4271             die "skip\n" if !$hotplug;
4272             vm_deviceunplug($vmid, $conf, $opt);
4273         } else {
4274
4275             die "internal error" if $opt !~ m/net(\d+)/;
4276             my $iface = "tap${vmid}i$1";
4277
4278             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4279                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4280                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4281                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4282                 PVE::Network::tap_unplug($iface);
4283                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4284             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4285                 # Rate can be applied on its own but any change above needs to
4286                 # include the rate in tap_plug since OVS resets everything.
4287                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4288             }
4289
4290             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4291                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4292             }
4293
4294             return 1;
4295         }
4296     }
4297
4298     if ($hotplug) {
4299         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4300     } else {
4301         die "skip\n";
4302     }
4303 }
4304
4305 sub vmconfig_update_disk {
4306     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4307
4308     # fixme: do we need force?
4309
4310     my $drive = parse_drive($opt, $value);
4311
4312     if ($conf->{$opt}) {
4313
4314         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4315
4316             my $media = $drive->{media} || 'disk';
4317             my $oldmedia = $old_drive->{media} || 'disk';
4318             die "unable to change media type\n" if $media ne $oldmedia;
4319
4320             if (!drive_is_cdrom($old_drive)) {
4321
4322                 if ($drive->{file} ne $old_drive->{file}) {
4323
4324                     die "skip\n" if !$hotplug;
4325
4326                     # unplug and register as unused
4327                     vm_deviceunplug($vmid, $conf, $opt);
4328                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4329
4330                 } else {
4331                     # update existing disk
4332
4333                     # skip non hotpluggable value
4334                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4335                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4336                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4337                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4338                         die "skip\n";
4339                     }
4340
4341                     # apply throttle
4342                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4343                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4344                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4345                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4346                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4347                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4348                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4349                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4350                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4351                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4352                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4353                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4354
4355                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4356                                                    ($drive->{mbps} || 0)*1024*1024,
4357                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4358                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4359                                                    $drive->{iops} || 0,
4360                                                    $drive->{iops_rd} || 0,
4361                                                    $drive->{iops_wr} || 0,
4362                                                    ($drive->{mbps_max} || 0)*1024*1024,
4363                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4364                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4365                                                    $drive->{iops_max} || 0,
4366                                                    $drive->{iops_rd_max} || 0,
4367                                                    $drive->{iops_wr_max} || 0);
4368
4369                     }
4370
4371                     return 1;
4372                 }
4373
4374             } else { # cdrom
4375
4376                 if ($drive->{file} eq 'none') {
4377                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4378                 } else {
4379                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4380                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4381                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4382                 }
4383
4384                 return 1;
4385             }
4386         }
4387     }
4388
4389     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4390     # hotplug new disks
4391     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4392     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4393 }
4394
4395 sub vm_start {
4396     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4397         $forcemachine, $spice_ticket) = @_;
4398
4399     PVE::QemuConfig->lock_config($vmid, sub {
4400         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4401
4402         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4403
4404         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4405
4406         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4407
4408         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4409             vmconfig_apply_pending($vmid, $conf, $storecfg);
4410             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4411         }
4412
4413         my $defaults = load_defaults();
4414
4415         # set environment variable useful inside network script
4416         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4417
4418         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4419
4420         my $migrate_port = 0;
4421         my $migrate_uri;
4422         if ($statefile) {
4423             if ($statefile eq 'tcp') {
4424                 my $localip = "localhost";
4425                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4426                 my $nodename = PVE::INotify::nodename();
4427                 if ($datacenterconf->{migration_unsecure}) {
4428                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4429                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4430                 }
4431                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4432                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4433                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4434                 push @$cmd, '-incoming', $migrate_uri;
4435                 push @$cmd, '-S';
4436             } else {
4437                 push @$cmd, '-loadstate', $statefile;
4438             }
4439         } elsif ($paused) {
4440             push @$cmd, '-S';
4441         }
4442
4443         # host pci devices
4444         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4445           my $d = parse_hostpci($conf->{"hostpci$i"});
4446           next if !$d;
4447           my $pcidevices = $d->{pciid};
4448           foreach my $pcidevice (@$pcidevices) {
4449                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4450
4451                 my $info = pci_device_info("0000:$pciid");
4452                 die "IOMMU not present\n" if !check_iommu_support();
4453                 die "no pci device info for device '$pciid'\n" if !$info;
4454                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4455                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4456           }
4457         }
4458
4459         PVE::Storage::activate_volumes($storecfg, $vollist);
4460
4461         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4462                     umask => 0077); };
4463
4464         if (my $err = $@) {
4465             # deactivate volumes if start fails
4466             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4467             die "start failed: $err";
4468         }
4469
4470         print "migration listens on $migrate_uri\n" if $migrate_uri;
4471
4472         if ($statefile && $statefile ne 'tcp')  {
4473             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4474             warn $@ if $@;
4475         }
4476
4477         if ($migratedfrom) {
4478
4479             eval {
4480                 set_migration_caps($vmid);
4481             };
4482             warn $@ if $@;
4483
4484             if ($spice_port) {
4485                 print "spice listens on port $spice_port\n";
4486                 if ($spice_ticket) {
4487                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4488                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4489                 }
4490             }
4491
4492         } else {
4493
4494             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4495                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4496                     if $conf->{balloon};
4497             }
4498
4499             foreach my $opt (keys %$conf) {
4500                 next if $opt !~  m/^net\d+$/;
4501                 my $nicconf = parse_net($conf->{$opt});
4502                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4503             }
4504         }
4505
4506         vm_mon_cmd_nocheck($vmid, 'qom-set',
4507                     path => "machine/peripheral/balloon0",
4508                     property => "guest-stats-polling-interval",
4509                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4510
4511     });
4512 }
4513
4514 sub vm_mon_cmd {
4515     my ($vmid, $execute, %params) = @_;
4516
4517     my $cmd = { execute => $execute, arguments => \%params };
4518     vm_qmp_command($vmid, $cmd);
4519 }
4520
4521 sub vm_mon_cmd_nocheck {
4522     my ($vmid, $execute, %params) = @_;
4523
4524     my $cmd = { execute => $execute, arguments => \%params };
4525     vm_qmp_command($vmid, $cmd, 1);
4526 }
4527
4528 sub vm_qmp_command {
4529     my ($vmid, $cmd, $nocheck) = @_;
4530
4531     my $res;
4532
4533     my $timeout;
4534     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4535         $timeout = $cmd->{arguments}->{timeout};
4536         delete $cmd->{arguments}->{timeout};
4537     }
4538
4539     eval {
4540         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4541         my $sname = qmp_socket($vmid);
4542         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4543             my $qmpclient = PVE::QMPClient->new();
4544
4545             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4546         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4547             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4548                 if scalar(%{$cmd->{arguments}});
4549             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4550         } else {
4551             die "unable to open monitor socket\n";
4552         }
4553     };
4554     if (my $err = $@) {
4555         syslog("err", "VM $vmid qmp command failed - $err");
4556         die $err;
4557     }
4558
4559     return $res;
4560 }
4561
4562 sub vm_human_monitor_command {
4563     my ($vmid, $cmdline) = @_;
4564
4565     my $res;
4566
4567     my $cmd = {
4568         execute => 'human-monitor-command',
4569         arguments => { 'command-line' => $cmdline},
4570     };
4571
4572     return vm_qmp_command($vmid, $cmd);
4573 }
4574
4575 sub vm_commandline {
4576     my ($storecfg, $vmid) = @_;
4577
4578     my $conf = PVE::QemuConfig->load_config($vmid);
4579
4580     my $defaults = load_defaults();
4581
4582     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4583
4584     return join(' ', @$cmd);
4585 }
4586
4587 sub vm_reset {
4588     my ($vmid, $skiplock) = @_;
4589
4590     PVE::QemuConfig->lock_config($vmid, sub {
4591
4592         my $conf = PVE::QemuConfig->load_config($vmid);
4593
4594         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4595
4596         vm_mon_cmd($vmid, "system_reset");
4597     });
4598 }
4599
4600 sub get_vm_volumes {
4601     my ($conf) = @_;
4602
4603     my $vollist = [];
4604     foreach_volid($conf, sub {
4605         my ($volid, $is_cdrom) = @_;
4606
4607         return if $volid =~ m|^/|;
4608
4609         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4610         return if !$sid;
4611
4612         push @$vollist, $volid;
4613     });
4614
4615     return $vollist;
4616 }
4617
4618 sub vm_stop_cleanup {
4619     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4620
4621     eval {
4622
4623         if (!$keepActive) {
4624             my $vollist = get_vm_volumes($conf);
4625             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4626         }
4627
4628         foreach my $ext (qw(mon qmp pid vnc qga)) {
4629             unlink "/var/run/qemu-server/${vmid}.$ext";
4630         }
4631
4632         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4633     };
4634     warn $@ if $@; # avoid errors - just warn
4635 }
4636
4637 # Note: use $nockeck to skip tests if VM configuration file exists.
4638 # We need that when migration VMs to other nodes (files already moved)
4639 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4640 sub vm_stop {
4641     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4642
4643     $force = 1 if !defined($force) && !$shutdown;
4644
4645     if ($migratedfrom){
4646         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4647         kill 15, $pid if $pid;
4648         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4649         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4650         return;
4651     }
4652
4653     PVE::QemuConfig->lock_config($vmid, sub {
4654
4655         my $pid = check_running($vmid, $nocheck);
4656         return if !$pid;
4657
4658         my $conf;
4659         if (!$nocheck) {
4660             $conf = PVE::QemuConfig->load_config($vmid);
4661             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4662             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4663                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4664                 $timeout = $opts->{down} if $opts->{down};
4665             }
4666         }
4667
4668         $timeout = 60 if !defined($timeout);
4669
4670         eval {
4671             if ($shutdown) {
4672                 if (defined($conf) && $conf->{agent}) {
4673                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4674                 } else {
4675                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4676                 }
4677             } else {
4678                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4679             }
4680         };
4681         my $err = $@;
4682
4683         if (!$err) {
4684             my $count = 0;
4685             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4686                 $count++;
4687                 sleep 1;
4688             }
4689
4690             if ($count >= $timeout) {
4691                 if ($force) {
4692                     warn "VM still running - terminating now with SIGTERM\n";
4693                     kill 15, $pid;
4694                 } else {
4695                     die "VM quit/powerdown failed - got timeout\n";
4696                 }
4697             } else {
4698                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4699                 return;
4700             }
4701         } else {
4702             if ($force) {
4703                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4704                 kill 15, $pid;
4705             } else {
4706                 die "VM quit/powerdown failed\n";
4707             }
4708         }
4709
4710         # wait again
4711         $timeout = 10;
4712
4713         my $count = 0;
4714         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4715             $count++;
4716             sleep 1;
4717         }
4718
4719         if ($count >= $timeout) {
4720             warn "VM still running - terminating now with SIGKILL\n";
4721             kill 9, $pid;
4722             sleep 1;
4723         }
4724
4725         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4726    });
4727 }
4728
4729 sub vm_suspend {
4730     my ($vmid, $skiplock) = @_;
4731
4732     PVE::QemuConfig->lock_config($vmid, sub {
4733
4734         my $conf = PVE::QemuConfig->load_config($vmid);
4735
4736         PVE::QemuConfig->check_lock($conf)
4737             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4738
4739         vm_mon_cmd($vmid, "stop");
4740     });
4741 }
4742
4743 sub vm_resume {
4744     my ($vmid, $skiplock, $nocheck) = @_;
4745
4746     PVE::QemuConfig->lock_config($vmid, sub {
4747
4748         if (!$nocheck) {
4749
4750             my $conf = PVE::QemuConfig->load_config($vmid);
4751
4752             PVE::QemuConfig->check_lock($conf)
4753                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4754
4755             vm_mon_cmd($vmid, "cont");
4756
4757         } else {
4758             vm_mon_cmd_nocheck($vmid, "cont");
4759         }
4760     });
4761 }
4762
4763 sub vm_sendkey {
4764     my ($vmid, $skiplock, $key) = @_;
4765
4766     PVE::QemuConfig->lock_config($vmid, sub {
4767
4768         my $conf = PVE::QemuConfig->load_config($vmid);
4769
4770         # there is no qmp command, so we use the human monitor command
4771         vm_human_monitor_command($vmid, "sendkey $key");
4772     });
4773 }
4774
4775 sub vm_destroy {
4776     my ($storecfg, $vmid, $skiplock) = @_;
4777
4778     PVE::QemuConfig->lock_config($vmid, sub {
4779
4780         my $conf = PVE::QemuConfig->load_config($vmid);
4781
4782         if (!check_running($vmid)) {
4783             destroy_vm($storecfg, $vmid, undef, $skiplock);
4784         } else {
4785             die "VM $vmid is running - destroy failed\n";
4786         }
4787     });
4788 }
4789
4790 # pci helpers
4791
4792 sub file_write {
4793     my ($filename, $buf) = @_;
4794
4795     my $fh = IO::File->new($filename, "w");
4796     return undef if !$fh;
4797
4798     my $res = print $fh $buf;
4799
4800     $fh->close();
4801
4802     return $res;
4803 }
4804
4805 sub pci_device_info {
4806     my ($name) = @_;
4807
4808     my $res;
4809
4810     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4811     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4812
4813     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4814     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4815
4816     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4817     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4818
4819     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4820     return undef if !defined($product) || $product !~ s/^0x//;
4821
4822     $res = {
4823         name => $name,
4824         vendor => $vendor,
4825         product => $product,
4826         domain => $domain,
4827         bus => $bus,
4828         slot => $slot,
4829         func => $func,
4830         irq => $irq,
4831         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4832     };
4833
4834     return $res;
4835 }
4836
4837 sub pci_dev_reset {
4838     my ($dev) = @_;
4839
4840     my $name = $dev->{name};
4841
4842     my $fn = "$pcisysfs/devices/$name/reset";
4843
4844     return file_write($fn, "1");
4845 }
4846
4847 sub pci_dev_bind_to_vfio {
4848     my ($dev) = @_;
4849
4850     my $name = $dev->{name};
4851
4852     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4853
4854     if (!-d $vfio_basedir) {
4855         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4856     }
4857     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4858
4859     my $testdir = "$vfio_basedir/$name";
4860     return 1 if -d $testdir;
4861
4862     my $data = "$dev->{vendor} $dev->{product}";
4863     return undef if !file_write("$vfio_basedir/new_id", $data);
4864
4865     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4866     if (!file_write($fn, $name)) {
4867         return undef if -f $fn;
4868     }
4869
4870     $fn = "$vfio_basedir/bind";
4871     if (! -d $testdir) {
4872         return undef if !file_write($fn, $name);
4873     }
4874
4875     return -d $testdir;
4876 }
4877
4878 sub pci_dev_group_bind_to_vfio {
4879     my ($pciid) = @_;
4880
4881     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4882
4883     if (!-d $vfio_basedir) {
4884         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4885     }
4886     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4887
4888     # get IOMMU group devices
4889     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4890       my @devs = grep /^0000:/, readdir($D);
4891     closedir($D);
4892
4893     foreach my $pciid (@devs) {
4894         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4895
4896         # pci bridges, switches or root ports are not supported
4897         # they have a pci_bus subdirectory so skip them
4898         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4899
4900         my $info = pci_device_info($1);
4901         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4902     }
4903
4904     return 1;
4905 }
4906
4907 sub print_pci_addr {
4908     my ($id, $bridges) = @_;
4909
4910     my $res = '';
4911     my $devices = {
4912         piix3 => { bus => 0, addr => 1 },
4913         #addr2 : first videocard
4914         balloon0 => { bus => 0, addr => 3 },
4915         watchdog => { bus => 0, addr => 4 },
4916         scsihw0 => { bus => 0, addr => 5 },
4917         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4918         scsihw1 => { bus => 0, addr => 6 },
4919         ahci0 => { bus => 0, addr => 7 },
4920         qga0 => { bus => 0, addr => 8 },
4921         spice => { bus => 0, addr => 9 },
4922         virtio0 => { bus => 0, addr => 10 },
4923         virtio1 => { bus => 0, addr => 11 },
4924         virtio2 => { bus => 0, addr => 12 },
4925         virtio3 => { bus => 0, addr => 13 },
4926         virtio4 => { bus => 0, addr => 14 },
4927         virtio5 => { bus => 0, addr => 15 },
4928         hostpci0 => { bus => 0, addr => 16 },
4929         hostpci1 => { bus => 0, addr => 17 },
4930         net0 => { bus => 0, addr => 18 },
4931         net1 => { bus => 0, addr => 19 },
4932         net2 => { bus => 0, addr => 20 },
4933         net3 => { bus => 0, addr => 21 },
4934         net4 => { bus => 0, addr => 22 },
4935         net5 => { bus => 0, addr => 23 },
4936         vga1 => { bus => 0, addr => 24 },
4937         vga2 => { bus => 0, addr => 25 },
4938         vga3 => { bus => 0, addr => 26 },
4939         hostpci2 => { bus => 0, addr => 27 },
4940         hostpci3 => { bus => 0, addr => 28 },
4941         #addr29 : usb-host (pve-usb.cfg)
4942         'pci.1' => { bus => 0, addr => 30 },
4943         'pci.2' => { bus => 0, addr => 31 },
4944         'net6' => { bus => 1, addr => 1 },
4945         'net7' => { bus => 1, addr => 2 },
4946         'net8' => { bus => 1, addr => 3 },
4947         'net9' => { bus => 1, addr => 4 },
4948         'net10' => { bus => 1, addr => 5 },
4949         'net11' => { bus => 1, addr => 6 },
4950         'net12' => { bus => 1, addr => 7 },
4951         'net13' => { bus => 1, addr => 8 },
4952         'net14' => { bus => 1, addr => 9 },
4953         'net15' => { bus => 1, addr => 10 },
4954         'net16' => { bus => 1, addr => 11 },
4955         'net17' => { bus => 1, addr => 12 },
4956         'net18' => { bus => 1, addr => 13 },
4957         'net19' => { bus => 1, addr => 14 },
4958         'net20' => { bus => 1, addr => 15 },
4959         'net21' => { bus => 1, addr => 16 },
4960         'net22' => { bus => 1, addr => 17 },
4961         'net23' => { bus => 1, addr => 18 },
4962         'net24' => { bus => 1, addr => 19 },
4963         'net25' => { bus => 1, addr => 20 },
4964         'net26' => { bus => 1, addr => 21 },
4965         'net27' => { bus => 1, addr => 22 },
4966         'net28' => { bus => 1, addr => 23 },
4967         'net29' => { bus => 1, addr => 24 },
4968         'net30' => { bus => 1, addr => 25 },
4969         'net31' => { bus => 1, addr => 26 },
4970         'xhci' => { bus => 1, addr => 27 },
4971         'virtio6' => { bus => 2, addr => 1 },
4972         'virtio7' => { bus => 2, addr => 2 },
4973         'virtio8' => { bus => 2, addr => 3 },
4974         'virtio9' => { bus => 2, addr => 4 },
4975         'virtio10' => { bus => 2, addr => 5 },
4976         'virtio11' => { bus => 2, addr => 6 },
4977         'virtio12' => { bus => 2, addr => 7 },
4978         'virtio13' => { bus => 2, addr => 8 },
4979         'virtio14' => { bus => 2, addr => 9 },
4980         'virtio15' => { bus => 2, addr => 10 },
4981         'virtioscsi0' => { bus => 3, addr => 1 },
4982         'virtioscsi1' => { bus => 3, addr => 2 },
4983         'virtioscsi2' => { bus => 3, addr => 3 },
4984         'virtioscsi3' => { bus => 3, addr => 4 },
4985         'virtioscsi4' => { bus => 3, addr => 5 },
4986         'virtioscsi5' => { bus => 3, addr => 6 },
4987         'virtioscsi6' => { bus => 3, addr => 7 },
4988         'virtioscsi7' => { bus => 3, addr => 8 },
4989         'virtioscsi8' => { bus => 3, addr => 9 },
4990         'virtioscsi9' => { bus => 3, addr => 10 },
4991         'virtioscsi10' => { bus => 3, addr => 11 },
4992         'virtioscsi11' => { bus => 3, addr => 12 },
4993         'virtioscsi12' => { bus => 3, addr => 13 },
4994         'virtioscsi13' => { bus => 3, addr => 14 },
4995         'virtioscsi14' => { bus => 3, addr => 15 },
4996         'virtioscsi15' => { bus => 3, addr => 16 },
4997         'virtioscsi16' => { bus => 3, addr => 17 },
4998         'virtioscsi17' => { bus => 3, addr => 18 },
4999         'virtioscsi18' => { bus => 3, addr => 19 },
5000         'virtioscsi19' => { bus => 3, addr => 20 },
5001         'virtioscsi20' => { bus => 3, addr => 21 },
5002         'virtioscsi21' => { bus => 3, addr => 22 },
5003         'virtioscsi22' => { bus => 3, addr => 23 },
5004         'virtioscsi23' => { bus => 3, addr => 24 },
5005         'virtioscsi24' => { bus => 3, addr => 25 },
5006         'virtioscsi25' => { bus => 3, addr => 26 },
5007         'virtioscsi26' => { bus => 3, addr => 27 },
5008         'virtioscsi27' => { bus => 3, addr => 28 },
5009         'virtioscsi28' => { bus => 3, addr => 29 },
5010         'virtioscsi29' => { bus => 3, addr => 30 },
5011         'virtioscsi30' => { bus => 3, addr => 31 },
5012
5013     };
5014
5015     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5016            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5017            my $bus = $devices->{$id}->{bus};
5018            $res = ",bus=pci.$bus,addr=$addr";
5019            $bridges->{$bus} = 1 if $bridges;
5020     }
5021     return $res;
5022
5023 }
5024
5025 sub print_pcie_addr {
5026     my ($id) = @_;
5027
5028     my $res = '';
5029     my $devices = {
5030         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
5031         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
5032         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
5033         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
5034     };
5035
5036     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
5037            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
5038            my $bus = $devices->{$id}->{bus};
5039            $res = ",bus=$bus,addr=$addr";
5040     }
5041     return $res;
5042
5043 }
5044
5045 # vzdump restore implementaion
5046
5047 sub tar_archive_read_firstfile {
5048     my $archive = shift;
5049
5050     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5051
5052     # try to detect archive type first
5053     my $pid = open (TMP, "tar tf '$archive'|") ||
5054         die "unable to open file '$archive'\n";
5055     my $firstfile = <TMP>;
5056     kill 15, $pid;
5057     close TMP;
5058
5059     die "ERROR: archive contaions no data\n" if !$firstfile;
5060     chomp $firstfile;
5061
5062     return $firstfile;
5063 }
5064
5065 sub tar_restore_cleanup {
5066     my ($storecfg, $statfile) = @_;
5067
5068     print STDERR "starting cleanup\n";
5069
5070     if (my $fd = IO::File->new($statfile, "r")) {
5071         while (defined(my $line = <$fd>)) {
5072             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5073                 my $volid = $2;
5074                 eval {
5075                     if ($volid =~ m|^/|) {
5076                         unlink $volid || die 'unlink failed\n';
5077                     } else {
5078                         PVE::Storage::vdisk_free($storecfg, $volid);
5079                     }
5080                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5081                 };
5082                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5083             } else {
5084                 print STDERR "unable to parse line in statfile - $line";
5085             }
5086         }
5087         $fd->close();
5088     }
5089 }
5090
5091 sub restore_archive {
5092     my ($archive, $vmid, $user, $opts) = @_;
5093
5094     my $format = $opts->{format};
5095     my $comp;
5096
5097     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5098         $format = 'tar' if !$format;
5099         $comp = 'gzip';
5100     } elsif ($archive =~ m/\.tar$/) {
5101         $format = 'tar' if !$format;
5102     } elsif ($archive =~ m/.tar.lzo$/) {
5103         $format = 'tar' if !$format;
5104         $comp = 'lzop';
5105     } elsif ($archive =~ m/\.vma$/) {
5106         $format = 'vma' if !$format;
5107     } elsif ($archive =~ m/\.vma\.gz$/) {
5108         $format = 'vma' if !$format;
5109         $comp = 'gzip';
5110     } elsif ($archive =~ m/\.vma\.lzo$/) {
5111         $format = 'vma' if !$format;
5112         $comp = 'lzop';
5113     } else {
5114         $format = 'vma' if !$format; # default
5115     }
5116
5117     # try to detect archive format
5118     if ($format eq 'tar') {
5119         return restore_tar_archive($archive, $vmid, $user, $opts);
5120     } else {
5121         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5122     }
5123 }
5124
5125 sub restore_update_config_line {
5126     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5127
5128     return if $line =~ m/^\#qmdump\#/;
5129     return if $line =~ m/^\#vzdump\#/;
5130     return if $line =~ m/^lock:/;
5131     return if $line =~ m/^unused\d+:/;
5132     return if $line =~ m/^parent:/;
5133     return if $line =~ m/^template:/; # restored VM is never a template
5134
5135     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5136         # try to convert old 1.X settings
5137         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5138         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5139             my ($model, $macaddr) = split(/\=/, $devconfig);
5140             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5141             my $net = {
5142                 model => $model,
5143                 bridge => "vmbr$ind",
5144                 macaddr => $macaddr,
5145             };
5146             my $netstr = print_net($net);
5147
5148             print $outfd "net$cookie->{netcount}: $netstr\n";
5149             $cookie->{netcount}++;
5150         }
5151     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5152         my ($id, $netstr) = ($1, $2);
5153         my $net = parse_net($netstr);
5154         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5155         $netstr = print_net($net);
5156         print $outfd "$id: $netstr\n";
5157     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5158         my $virtdev = $1;
5159         my $value = $3;
5160         my $di = parse_drive($virtdev, $value);
5161         if (defined($di->{backup}) && !$di->{backup}) {
5162             print $outfd "#$line";
5163         } elsif ($map->{$virtdev}) {
5164             delete $di->{format}; # format can change on restore
5165             $di->{file} = $map->{$virtdev};
5166             $value = print_drive($vmid, $di);
5167             print $outfd "$virtdev: $value\n";
5168         } else {
5169             print $outfd $line;
5170         }
5171     } else {
5172         print $outfd $line;
5173     }
5174 }
5175
5176 sub scan_volids {
5177     my ($cfg, $vmid) = @_;
5178
5179     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5180
5181     my $volid_hash = {};
5182     foreach my $storeid (keys %$info) {
5183         foreach my $item (@{$info->{$storeid}}) {
5184             next if !($item->{volid} && $item->{size});
5185             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5186             $volid_hash->{$item->{volid}} = $item;
5187         }
5188     }
5189
5190     return $volid_hash;
5191 }
5192
5193 sub is_volume_in_use {
5194     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5195
5196     my $path = PVE::Storage::path($storecfg, $volid);
5197
5198     my $scan_config = sub {
5199         my ($cref, $snapname) = @_;
5200
5201         foreach my $key (keys %$cref) {
5202             my $value = $cref->{$key};
5203             if (is_valid_drivename($key)) {
5204                 next if $skip_drive && $key eq $skip_drive;
5205                 my $drive = parse_drive($key, $value);
5206                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5207                 return 1 if $volid eq $drive->{file};
5208                 if ($drive->{file} =~ m!^/!) {
5209                     return 1 if $drive->{file} eq $path;
5210                 } else {
5211                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5212                     next if !$storeid;
5213                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5214                     next if !$scfg;
5215                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5216                 }
5217             }
5218         }
5219
5220         return 0;
5221     };
5222
5223     return 1 if &$scan_config($conf);
5224
5225     undef $skip_drive;
5226
5227     foreach my $snapname (keys %{$conf->{snapshots}}) {
5228         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5229     }
5230
5231     return 0;
5232 }
5233
5234 sub update_disksize {
5235     my ($vmid, $conf, $volid_hash) = @_;
5236
5237     my $changes;
5238
5239     my $used = {};
5240
5241     # Note: it is allowed to define multiple storages with same path (alias), so
5242     # we need to check both 'volid' and real 'path' (two different volid can point
5243     # to the same path).
5244
5245     my $usedpath = {};
5246
5247     # update size info
5248     foreach my $opt (keys %$conf) {
5249         if (is_valid_drivename($opt)) {
5250             my $drive = parse_drive($opt, $conf->{$opt});
5251             my $volid = $drive->{file};
5252             next if !$volid;
5253
5254             $used->{$volid} = 1;
5255             if ($volid_hash->{$volid} &&
5256                 (my $path = $volid_hash->{$volid}->{path})) {
5257                 $usedpath->{$path} = 1;
5258             }
5259
5260             next if drive_is_cdrom($drive);
5261             next if !$volid_hash->{$volid};
5262
5263             $drive->{size} = $volid_hash->{$volid}->{size};
5264             my $new = print_drive($vmid, $drive);
5265             if ($new ne $conf->{$opt}) {
5266                 $changes = 1;
5267                 $conf->{$opt} = $new;
5268             }
5269         }
5270     }
5271
5272     # remove 'unusedX' entry if volume is used
5273     foreach my $opt (keys %$conf) {
5274         next if $opt !~ m/^unused\d+$/;
5275         my $volid = $conf->{$opt};
5276         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5277         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5278             $changes = 1;
5279             delete $conf->{$opt};
5280         }
5281     }
5282
5283     foreach my $volid (sort keys %$volid_hash) {
5284         next if $volid =~ m/vm-$vmid-state-/;
5285         next if $used->{$volid};
5286         my $path = $volid_hash->{$volid}->{path};
5287         next if !$path; # just to be sure
5288         next if $usedpath->{$path};
5289         $changes = 1;
5290         PVE::QemuConfig->add_unused_volume($conf, $volid);
5291         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5292     }
5293
5294     return $changes;
5295 }
5296
5297 sub rescan {
5298     my ($vmid, $nolock) = @_;
5299
5300     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5301
5302     my $volid_hash = scan_volids($cfg, $vmid);
5303
5304     my $updatefn =  sub {
5305         my ($vmid) = @_;
5306
5307         my $conf = PVE::QemuConfig->load_config($vmid);
5308
5309         PVE::QemuConfig->check_lock($conf);
5310
5311         my $vm_volids = {};
5312         foreach my $volid (keys %$volid_hash) {
5313             my $info = $volid_hash->{$volid};
5314             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5315         }
5316
5317         my $changes = update_disksize($vmid, $conf, $vm_volids);
5318
5319         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5320     };
5321
5322     if (defined($vmid)) {
5323         if ($nolock) {
5324             &$updatefn($vmid);
5325         } else {
5326             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5327         }
5328     } else {
5329         my $vmlist = config_list();
5330         foreach my $vmid (keys %$vmlist) {
5331             if ($nolock) {
5332                 &$updatefn($vmid);
5333             } else {
5334                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5335             }
5336         }
5337     }
5338 }
5339
5340 sub restore_vma_archive {
5341     my ($archive, $vmid, $user, $opts, $comp) = @_;
5342
5343     my $input = $archive eq '-' ? "<&STDIN" : undef;
5344     my $readfrom = $archive;
5345
5346     my $uncomp = '';
5347     if ($comp) {
5348         $readfrom = '-';
5349         my $qarchive = PVE::Tools::shellquote($archive);
5350         if ($comp eq 'gzip') {
5351             $uncomp = "zcat $qarchive|";
5352         } elsif ($comp eq 'lzop') {
5353             $uncomp = "lzop -d -c $qarchive|";
5354         } else {
5355             die "unknown compression method '$comp'\n";
5356         }
5357
5358     }
5359
5360     my $tmpdir = "/var/tmp/vzdumptmp$$";
5361     rmtree $tmpdir;
5362
5363     # disable interrupts (always do cleanups)
5364     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5365         warn "got interrupt - ignored\n";
5366     };
5367
5368     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5369     POSIX::mkfifo($mapfifo, 0600);
5370     my $fifofh;
5371
5372     my $openfifo = sub {
5373         open($fifofh, '>', $mapfifo) || die $!;
5374     };
5375
5376     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5377
5378     my $oldtimeout;
5379     my $timeout = 5;
5380
5381     my $devinfo = {};
5382
5383     my $rpcenv = PVE::RPCEnvironment::get();
5384
5385     my $conffile = PVE::QemuConfig->config_file($vmid);
5386     my $tmpfn = "$conffile.$$.tmp";
5387
5388     # Note: $oldconf is undef if VM does not exists
5389     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5390     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5391
5392     my $print_devmap = sub {
5393         my $virtdev_hash = {};
5394
5395         my $cfgfn = "$tmpdir/qemu-server.conf";
5396
5397         # we can read the config - that is already extracted
5398         my $fh = IO::File->new($cfgfn, "r") ||
5399             "unable to read qemu-server.conf - $!\n";
5400
5401         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5402         if (-f $fwcfgfn) {
5403             my $pve_firewall_dir = '/etc/pve/firewall';
5404             mkdir $pve_firewall_dir; # make sure the dir exists
5405             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5406         }
5407
5408         while (defined(my $line = <$fh>)) {
5409             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5410                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5411                 die "archive does not contain data for drive '$virtdev'\n"
5412                     if !$devinfo->{$devname};
5413                 if (defined($opts->{storage})) {
5414                     $storeid = $opts->{storage} || 'local';
5415                 } elsif (!$storeid) {
5416                     $storeid = 'local';
5417                 }
5418                 $format = 'raw' if !$format;
5419                 $devinfo->{$devname}->{devname} = $devname;
5420                 $devinfo->{$devname}->{virtdev} = $virtdev;
5421                 $devinfo->{$devname}->{format} = $format;
5422                 $devinfo->{$devname}->{storeid} = $storeid;
5423
5424                 # check permission on storage
5425                 my $pool = $opts->{pool}; # todo: do we need that?
5426                 if ($user ne 'root@pam') {
5427                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5428                 }
5429
5430                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5431             }
5432         }
5433
5434         foreach my $devname (keys %$devinfo) {
5435             die "found no device mapping information for device '$devname'\n"
5436                 if !$devinfo->{$devname}->{virtdev};
5437         }
5438
5439         my $cfg = cfs_read_file('storage.cfg');
5440
5441         # create empty/temp config
5442         if ($oldconf) {
5443             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5444             foreach_drive($oldconf, sub {
5445                 my ($ds, $drive) = @_;
5446
5447                 return if drive_is_cdrom($drive);
5448
5449                 my $volid = $drive->{file};
5450
5451                 return if !$volid || $volid =~ m|^/|;
5452
5453                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5454                 return if !$path || !$owner || ($owner != $vmid);
5455
5456                 # Note: only delete disk we want to restore
5457                 # other volumes will become unused
5458                 if ($virtdev_hash->{$ds}) {
5459                     PVE::Storage::vdisk_free($cfg, $volid);
5460                 }
5461             });
5462
5463             # delete vmstate files
5464             # since after the restore we have no snapshots anymore
5465             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5466                 my $snap = $oldconf->{snapshots}->{$snapname};
5467                 if ($snap->{vmstate}) {
5468                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5469                     if (my $err = $@) {
5470                         warn $err;
5471                     }
5472                 }
5473             }
5474         }
5475
5476         my $map = {};
5477         foreach my $virtdev (sort keys %$virtdev_hash) {
5478             my $d = $virtdev_hash->{$virtdev};
5479             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5480             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5481
5482             # test if requested format is supported
5483             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5484             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5485             $d->{format} = $defFormat if !$supported;
5486
5487             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5488                                                   $d->{format}, undef, $alloc_size);
5489             print STDERR "new volume ID is '$volid'\n";
5490             $d->{volid} = $volid;
5491             my $path = PVE::Storage::path($cfg, $volid);
5492
5493             PVE::Storage::activate_volumes($cfg,[$volid]);
5494
5495             my $write_zeros = 1;
5496             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5497                 $write_zeros = 0;
5498             }
5499
5500             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5501
5502             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5503             $map->{$virtdev} = $volid;
5504         }
5505
5506         $fh->seek(0, 0) || die "seek failed - $!\n";
5507
5508         my $outfd = new IO::File ($tmpfn, "w") ||
5509             die "unable to write config for VM $vmid\n";
5510
5511         my $cookie = { netcount => 0 };
5512         while (defined(my $line = <$fh>)) {
5513             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5514         }
5515
5516         $fh->close();
5517         $outfd->close();
5518     };
5519
5520     eval {
5521         # enable interrupts
5522         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5523             die "interrupted by signal\n";
5524         };
5525         local $SIG{ALRM} = sub { die "got timeout\n"; };
5526
5527         $oldtimeout = alarm($timeout);
5528
5529         my $parser = sub {
5530             my $line = shift;
5531
5532             print "$line\n";
5533
5534             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5535                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5536                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5537             } elsif ($line =~ m/^CTIME: /) {
5538                 # we correctly received the vma config, so we can disable
5539                 # the timeout now for disk allocation (set to 10 minutes, so
5540                 # that we always timeout if something goes wrong)
5541                 alarm(600);
5542                 &$print_devmap();
5543                 print $fifofh "done\n";
5544                 my $tmp = $oldtimeout || 0;
5545                 $oldtimeout = undef;
5546                 alarm($tmp);
5547                 close($fifofh);
5548             }
5549         };
5550
5551         print "restore vma archive: $cmd\n";
5552         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5553     };
5554     my $err = $@;
5555
5556     alarm($oldtimeout) if $oldtimeout;
5557
5558     my $vollist = [];
5559     foreach my $devname (keys %$devinfo) {
5560         my $volid = $devinfo->{$devname}->{volid};
5561         push @$vollist, $volid if $volid;
5562     }
5563
5564     my $cfg = cfs_read_file('storage.cfg');
5565     PVE::Storage::deactivate_volumes($cfg, $vollist);
5566
5567     unlink $mapfifo;
5568
5569     if ($err) {
5570         rmtree $tmpdir;
5571         unlink $tmpfn;
5572
5573         foreach my $devname (keys %$devinfo) {
5574             my $volid = $devinfo->{$devname}->{volid};
5575             next if !$volid;
5576             eval {
5577                 if ($volid =~ m|^/|) {
5578                     unlink $volid || die 'unlink failed\n';
5579                 } else {
5580                     PVE::Storage::vdisk_free($cfg, $volid);
5581                 }
5582                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5583             };
5584             print STDERR "unable to cleanup '$volid' - $@" if $@;
5585         }
5586         die $err;
5587     }
5588
5589     rmtree $tmpdir;
5590
5591     rename($tmpfn, $conffile) ||
5592         die "unable to commit configuration file '$conffile'\n";
5593
5594     PVE::Cluster::cfs_update(); # make sure we read new file
5595
5596     eval { rescan($vmid, 1); };
5597     warn $@ if $@;
5598 }
5599
5600 sub restore_tar_archive {
5601     my ($archive, $vmid, $user, $opts) = @_;
5602
5603     if ($archive ne '-') {
5604         my $firstfile = tar_archive_read_firstfile($archive);
5605         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5606             if $firstfile ne 'qemu-server.conf';
5607     }
5608
5609     my $storecfg = cfs_read_file('storage.cfg');
5610
5611     # destroy existing data - keep empty config
5612     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5613     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5614
5615     my $tocmd = "/usr/lib/qemu-server/qmextract";
5616
5617     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5618     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5619     $tocmd .= ' --prealloc' if $opts->{prealloc};
5620     $tocmd .= ' --info' if $opts->{info};
5621
5622     # tar option "xf" does not autodetect compression when read from STDIN,
5623     # so we pipe to zcat
5624     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5625         PVE::Tools::shellquote("--to-command=$tocmd");
5626
5627     my $tmpdir = "/var/tmp/vzdumptmp$$";
5628     mkpath $tmpdir;
5629
5630     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5631     local $ENV{VZDUMP_VMID} = $vmid;
5632     local $ENV{VZDUMP_USER} = $user;
5633
5634     my $conffile = PVE::QemuConfig->config_file($vmid);
5635     my $tmpfn = "$conffile.$$.tmp";
5636
5637     # disable interrupts (always do cleanups)
5638     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5639         print STDERR "got interrupt - ignored\n";
5640     };
5641
5642     eval {
5643         # enable interrupts
5644         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5645             die "interrupted by signal\n";
5646         };
5647
5648         if ($archive eq '-') {
5649             print "extracting archive from STDIN\n";
5650             run_command($cmd, input => "<&STDIN");
5651         } else {
5652             print "extracting archive '$archive'\n";
5653             run_command($cmd);
5654         }
5655
5656         return if $opts->{info};
5657
5658         # read new mapping
5659         my $map = {};
5660         my $statfile = "$tmpdir/qmrestore.stat";
5661         if (my $fd = IO::File->new($statfile, "r")) {
5662             while (defined (my $line = <$fd>)) {
5663                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5664                     $map->{$1} = $2 if $1;
5665                 } else {
5666                     print STDERR "unable to parse line in statfile - $line\n";
5667                 }
5668             }
5669             $fd->close();
5670         }
5671
5672         my $confsrc = "$tmpdir/qemu-server.conf";
5673
5674         my $srcfd = new IO::File($confsrc, "r") ||
5675             die "unable to open file '$confsrc'\n";
5676
5677         my $outfd = new IO::File ($tmpfn, "w") ||
5678             die "unable to write config for VM $vmid\n";
5679
5680         my $cookie = { netcount => 0 };
5681         while (defined (my $line = <$srcfd>)) {
5682             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5683         }
5684
5685         $srcfd->close();
5686         $outfd->close();
5687     };
5688     my $err = $@;
5689
5690     if ($err) {
5691
5692         unlink $tmpfn;
5693
5694         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5695
5696         die $err;
5697     }
5698
5699     rmtree $tmpdir;
5700
5701     rename $tmpfn, $conffile ||
5702         die "unable to commit configuration file '$conffile'\n";
5703
5704     PVE::Cluster::cfs_update(); # make sure we read new file
5705
5706     eval { rescan($vmid, 1); };
5707     warn $@ if $@;
5708 };
5709
5710 sub foreach_writable_storage {
5711     my ($conf, $func) = @_;
5712
5713     my $sidhash = {};
5714
5715     foreach my $ds (keys %$conf) {
5716         next if !is_valid_drivename($ds);
5717
5718         my $drive = parse_drive($ds, $conf->{$ds});
5719         next if !$drive;
5720         next if drive_is_cdrom($drive);
5721
5722         my $volid = $drive->{file};
5723
5724         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5725         $sidhash->{$sid} = $sid if $sid;
5726     }
5727
5728     foreach my $sid (sort keys %$sidhash) {
5729         &$func($sid);
5730     }
5731 }
5732
5733 sub do_snapshots_with_qemu {
5734     my ($storecfg, $volid) = @_;
5735
5736     my $storage_name = PVE::Storage::parse_volume_id($volid);
5737
5738     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5739         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5740         return 1;
5741     }
5742
5743     if ($volid =~ m/\.(qcow2|qed)$/){
5744         return 1;
5745     }
5746
5747     return undef;
5748 }
5749
5750 sub qga_check_running {
5751     my ($vmid) = @_;
5752
5753     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5754     if ($@) {
5755         warn "Qemu Guest Agent are not running - $@";
5756         return 0;
5757     }
5758     return 1;
5759 }
5760
5761 sub template_create {
5762     my ($vmid, $conf, $disk) = @_;
5763
5764     my $storecfg = PVE::Storage::config();
5765
5766     foreach_drive($conf, sub {
5767         my ($ds, $drive) = @_;
5768
5769         return if drive_is_cdrom($drive);
5770         return if $disk && $ds ne $disk;
5771
5772         my $volid = $drive->{file};
5773         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5774
5775         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5776         $drive->{file} = $voliddst;
5777         $conf->{$ds} = print_drive($vmid, $drive);
5778         PVE::QemuConfig->write_config($vmid, $conf);
5779     });
5780 }
5781
5782 sub qemu_img_convert {
5783     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5784
5785     my $storecfg = PVE::Storage::config();
5786     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5787     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5788
5789     if ($src_storeid && $dst_storeid) {
5790
5791         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5792
5793         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5794         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5795
5796         my $src_format = qemu_img_format($src_scfg, $src_volname);
5797         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5798
5799         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5800         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5801
5802         my $cmd = [];
5803         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5804         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5805         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5806
5807         my $parser = sub {
5808             my $line = shift;
5809             if($line =~ m/\((\S+)\/100\%\)/){
5810                 my $percent = $1;
5811                 my $transferred = int($size * $percent / 100);
5812                 my $remaining = $size - $transferred;
5813
5814                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5815             }
5816
5817         };
5818
5819         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5820         my $err = $@;
5821         die "copy failed: $err" if $err;
5822     }
5823 }
5824
5825 sub qemu_img_format {
5826     my ($scfg, $volname) = @_;
5827
5828     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5829         return $1;
5830     } else {
5831         return "raw";
5832     }
5833 }
5834
5835 sub qemu_drive_mirror {
5836     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
5837
5838     my $storecfg = PVE::Storage::config();
5839     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5840
5841     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5842
5843     my $format = qemu_img_format($dst_scfg, $dst_volname);
5844
5845     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5846
5847     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
5848     $opts->{format} = $format if $format;
5849
5850     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5851
5852     eval {
5853     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5854         while (1) {
5855             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5856             my $stat = @$stats[0];
5857             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5858             die "error job is not mirroring" if $stat->{type} ne "mirror";
5859
5860             my $busy = $stat->{busy};
5861             my $ready = $stat->{ready};
5862
5863             if (my $total = $stat->{len}) {
5864                 my $transferred = $stat->{offset} || 0;
5865                 my $remaining = $total - $transferred;
5866                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5867
5868                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5869             }
5870
5871
5872             if ($stat->{ready} eq 'true') {
5873
5874                 last if $vmiddst != $vmid;
5875
5876                 # try to switch the disk if source and destination are on the same guest
5877                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5878                 last if !$@;
5879                 die $@ if $@ !~ m/cannot be completed/;
5880             }
5881             sleep 1;
5882         }
5883
5884
5885     };
5886     my $err = $@;
5887
5888     my $cancel_job = sub {
5889         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5890         while (1) {
5891             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5892             my $stat = @$stats[0];
5893             last if !$stat;
5894             sleep 1;
5895         }
5896     };
5897
5898     if ($err) {
5899         eval { &$cancel_job(); };
5900         die "mirroring error: $err";
5901     }
5902
5903     if ($vmiddst != $vmid) {
5904         # if we clone a disk for a new target vm, we don't switch the disk
5905         &$cancel_job(); # so we call block-job-cancel
5906     }
5907 }
5908
5909 sub clone_disk {
5910     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5911         $newvmid, $storage, $format, $full, $newvollist) = @_;
5912
5913     my $newvolid;
5914
5915     if (!$full) {
5916         print "create linked clone of drive $drivename ($drive->{file})\n";
5917         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5918         push @$newvollist, $newvolid;
5919     } else {
5920         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5921         $storeid = $storage if $storage;
5922
5923         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5924         if (!$format) {
5925             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5926             $format = qemu_img_format($scfg, $volname);
5927         }
5928
5929         # test if requested format is supported - else use default
5930         my $supported = grep { $_ eq $format } @$validFormats;
5931         $format = $defFormat if !$supported;
5932
5933         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5934
5935         print "create full clone of drive $drivename ($drive->{file})\n";
5936         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5937         push @$newvollist, $newvolid;
5938
5939         PVE::Storage::activate_volumes($storecfg, $newvollist);
5940
5941         if (!$running || $snapname) {
5942             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
5943         } else {
5944             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
5945         }
5946     }
5947
5948     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5949
5950     my $disk = $drive;
5951     $disk->{format} = undef;
5952     $disk->{file} = $newvolid;
5953     $disk->{size} = $size;
5954
5955     return $disk;
5956 }
5957
5958 # this only works if VM is running
5959 sub get_current_qemu_machine {
5960     my ($vmid) = @_;
5961
5962     my $cmd = { execute => 'query-machines', arguments => {} };
5963     my $res = vm_qmp_command($vmid, $cmd);
5964
5965     my ($current, $default);
5966     foreach my $e (@$res) {
5967         $default = $e->{name} if $e->{'is-default'};
5968         $current = $e->{name} if $e->{'is-current'};
5969     }
5970
5971     # fallback to the default machine if current is not supported by qemu
5972     return $current || $default || 'pc';
5973 }
5974
5975 sub qemu_machine_feature_enabled {
5976     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5977
5978     my $current_major;
5979     my $current_minor;
5980
5981     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5982
5983         $current_major = $3;
5984         $current_minor = $4;
5985
5986     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5987
5988         $current_major = $1;
5989         $current_minor = $2;
5990     }
5991
5992     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5993
5994
5995 }
5996
5997 sub qemu_machine_pxe {
5998     my ($vmid, $conf, $machine) = @_;
5999
6000     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6001
6002     foreach my $opt (keys %$conf) {
6003         next if $opt !~ m/^net(\d+)$/;
6004         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6005         next if !$net;
6006         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6007         return $machine.".pxe" if $romfile =~ m/pxe/;
6008         last;
6009     }
6010
6011     return $machine;
6012 }
6013
6014 sub qemu_use_old_bios_files {
6015     my ($machine_type) = @_;
6016
6017     return if !$machine_type;
6018
6019     my $use_old_bios_files = undef;
6020
6021     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6022         $machine_type = $1;
6023         $use_old_bios_files = 1;
6024     } else {
6025         my $kvmver = kvm_user_version();
6026         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6027         # load new efi bios files on migration. So this hack is required to allow
6028         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6029         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6030         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6031     }
6032
6033     return ($use_old_bios_files, $machine_type);
6034 }
6035
6036 sub lspci {
6037
6038     my $devices = {};
6039
6040     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6041             my (undef, $id, $function) = @_;
6042             my $res = { id => $id, function => $function};
6043             push @{$devices->{$id}}, $res;
6044     });
6045
6046     return $devices;
6047 }
6048
6049 sub vm_iothreads_list {
6050     my ($vmid) = @_;
6051
6052     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6053
6054     my $iothreads = {};
6055     foreach my $iothread (@$res) {
6056         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6057     }
6058
6059     return $iothreads;
6060 }
6061
6062 sub scsihw_infos {
6063     my ($conf, $drive) = @_;
6064
6065     my $maxdev = 0;
6066
6067     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6068         $maxdev = 7;
6069     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6070         $maxdev = 1;
6071     } else {
6072         $maxdev = 256;
6073     }
6074
6075     my $controller = int($drive->{index} / $maxdev);
6076     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6077
6078     return ($maxdev, $controller, $controller_prefix);
6079 }
6080
6081 # bash completion helper
6082
6083 sub complete_backup_archives {
6084     my ($cmdname, $pname, $cvalue) = @_;
6085
6086     my $cfg = PVE::Storage::config();
6087
6088     my $storeid;
6089
6090     if ($cvalue =~ m/^([^:]+):/) {
6091         $storeid = $1;
6092     }
6093
6094     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6095
6096     my $res = [];
6097     foreach my $id (keys %$data) {
6098         foreach my $item (@{$data->{$id}}) {
6099             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6100             push @$res, $item->{volid} if defined($item->{volid});
6101         }
6102     }
6103
6104     return $res;
6105 }
6106
6107 my $complete_vmid_full = sub {
6108     my ($running) = @_;
6109
6110     my $idlist = vmstatus();
6111
6112     my $res = [];
6113
6114     foreach my $id (keys %$idlist) {
6115         my $d = $idlist->{$id};
6116         if (defined($running)) {
6117             next if $d->{template};
6118             next if $running && $d->{status} ne 'running';
6119             next if !$running && $d->{status} eq 'running';
6120         }
6121         push @$res, $id;
6122
6123     }
6124     return $res;
6125 };
6126
6127 sub complete_vmid {
6128     return &$complete_vmid_full();
6129 }
6130
6131 sub complete_vmid_stopped {
6132     return &$complete_vmid_full(0);
6133 }
6134
6135 sub complete_vmid_running {
6136     return &$complete_vmid_full(1);
6137 }
6138
6139 sub complete_storage {
6140
6141     my $cfg = PVE::Storage::config();
6142     my $ids = $cfg->{ids};
6143
6144     my $res = [];
6145     foreach my $sid (keys %$ids) {
6146         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6147         next if !$ids->{$sid}->{content}->{images};
6148         push @$res, $sid;
6149     }
6150
6151     return $res;
6152 }
6153
6154 1;